def parse(self, value: str, messages: RPSLParserMessages, strict_validation=True) -> Optional[RPSLFieldParseResult]: if '-' not in value: messages.error( f'Invalid AS range: {value}: does not contain a hyphen') return None as1_raw, as2_raw = map(str.strip, value.split('-', 1)) try: as1_str, as1_int = parse_as_number(as1_raw) as2_str, as2_int = parse_as_number(as2_raw) except ValidationError as ve: messages.error(str(ve)) return None if as1_int > as2_int: # type: ignore messages.error( f'Invalid AS range: {value}: first AS is higher then second AS' ) return None parsed_value = f'{as1_str} - {as2_str}' if parsed_value != value: messages.info( f'AS range {value} was reformatted as {parsed_value}') return RPSLFieldParseResult(parsed_value, asn_first=as1_int, asn_last=as2_int)
def _recursive_set_resolve(self, members: Set[str], sets_seen=None, root_source: Optional[str] = None) -> Set[str]: """ Resolve all members of a number of sets, recursively. For each set in members, determines whether it has been seen already (to prevent infinite recursion), ignores it if already seen, and then either adds it directly or adds it to a set that requires further resolving. If root_source is set, the root object is only looked for in that source - resolving is then continued using the currently set sources. """ if not sets_seen: sets_seen = set() if all([member in sets_seen for member in members]): return set() sets_seen.update(members) set_members = set() resolved_as_members = set() sub_members, leaf_members = self._find_set_members( members, limit_source=root_source) for sub_member in sub_members: if self._current_set_root_object_class is None or self._current_set_root_object_class == 'route-set': try: IP(sub_member.split('^')[0]) set_members.add(sub_member) continue except ValueError: pass # AS numbers are permitted in route-sets and as-sets, per RFC 2622 5.3. # When an AS number is encountered as part of route-set resolving, # the prefixes originating from that AS should be added to the response. try: as_number_formatted, _ = parse_as_number(sub_member) if self._current_set_root_object_class == 'route-set': set_members.update( self.preloader.routes_for_origins( [as_number_formatted], self.sources)) resolved_as_members.add(sub_member) else: set_members.add(sub_member) continue except ValueError: pass self._current_set_maximum_depth -= 1 if self._current_set_maximum_depth == 0: return set_members | sub_members | leaf_members further_resolving_required = sub_members - set_members - sets_seen - resolved_as_members - self._current_excluded_sets new_members = self._recursive_set_resolve(further_resolving_required, sets_seen) set_members.update(new_members) return set_members
def parse(self, value: str, messages: RPSLParserMessages, strict_validation=True) -> Optional[RPSLFieldParseResult]: try: parsed_str, parsed_int = parse_as_number(value) except ValidationError as ve: messages.error(str(ve)) return None if parsed_str and parsed_str.upper() != value.upper(): messages.info(f"AS number {value} was reformatted as {parsed_str}") return RPSLFieldParseResult(parsed_str, asn_first=parsed_int, asn_last=parsed_int)
def parse_set_name(prefixes: List[str], value: str, messages: RPSLParserMessages, strict_validation=True) -> Optional[RPSLFieldParseResult]: assert all([prefix in reserved_prefixes for prefix in prefixes]) input_components = value.split(':') output_components: List[str] = [] prefix_display = '/'.join(prefixes) if strict_validation and len(input_components) > 5: messages.error(f'Set names can have a maximum of five components.') return None if strict_validation and not any( [c.upper().startswith(tuple(prefixes)) for c in input_components]): messages.error( f'Invalid set name {value}: at least one component must be ' f'an actual set name (i.e. start with {prefix_display})') return None for component in input_components: if strict_validation and component.upper() in reserved_words: messages.error( f'Invalid set name {value}: component {component} is a reserved word' ) return None parsed_as_number = None try: parsed_as_number, _ = parse_as_number(component) except ValidationError: pass if not re_generic_name.match( component.upper()) and not parsed_as_number: messages.error( f'Invalid set {value}: component {component} is not a valid AS number nor a valid set name' ) return None if strict_validation and not parsed_as_number and not component.upper( ).startswith(tuple(prefixes)): messages.error( f'Invalid set {value}: component {component} is not a valid AS number, ' f'nor does it start with {prefix_display}') return None if parsed_as_number: output_components.append(parsed_as_number) else: output_components.append(component) parsed_value = ':'.join(output_components) if parsed_value != value: messages.info(f'Set name {value} was reformatted as {parsed_value}') return RPSLFieldParseResult(parsed_value)
def _recursive_set_resolve(self, members: Set[str], sets_seen=None) -> Set[str]: """ Resolve all members of a number of sets, recursively. For each set in members, determines whether it has been seen already (to prevent infinite recursion), ignores it if already seen, and then either adds it directly or adds it to a set that requires further resolving. """ if not sets_seen: sets_seen = set() if all([member in sets_seen for member in members]): return set() sets_seen.update(members) set_members = set() sub_members, leaf_members = self._find_set_members(members) set_members.update(leaf_members) for sub_member in sub_members: try: IP(sub_member) set_members.add(sub_member) continue except ValueError: pass try: parse_as_number(sub_member) set_members.add(sub_member) continue except ValueError: pass further_resolving_required = sub_members - set_members - sets_seen new_members = self._recursive_set_resolve(further_resolving_required, sets_seen) set_members.update(new_members) return set_members
def _routes_for_origin(self, origin: str, ip_version: Optional[int]=None) -> str: """ Resolve all route(6)s prefixes for an origin, returning a space-separated list of all originating prefixes, not including duplicates. """ try: origin_formatted, _ = parse_as_number(origin) except ValidationError as ve: raise WhoisQueryParserException(str(ve)) prefixes = self.preloader.routes_for_origins([origin_formatted], ip_version=ip_version) return ' '.join(prefixes)
def parse(self, value: str, messages: RPSLParserMessages, strict_validation=True) -> Optional[RPSLFieldParseResult]: if '^' in value: address, range_operator = value.split('^', maxsplit=1) if not range_operator: messages.error(f'Missing range operator in value: {value}') return None else: address = value range_operator = '' parse_set_result_messages = RPSLParserMessages() parse_set_result = parse_set_name(['RS-', 'AS-'], address, parse_set_result_messages, strict_validation) if parse_set_result and not parse_set_result_messages.errors(): result_value = parse_set_result.value if range_operator: result_value += '^' + range_operator if result_value != value: messages.info(f'Route set member {value} was reformatted as {result_value}') return RPSLFieldParseResult(value=result_value) try: parsed_str, parsed_int = parse_as_number(address) result_value = parsed_str if range_operator: result_value += '^' + range_operator if result_value != value: messages.info(f'Route set member {value} was reformatted as {result_value}') return RPSLFieldParseResult(value=result_value) except ValidationError: pass try: ip_version = self.ip_version if self.ip_version else 0 ip = IP(address, ipversion=ip_version) except ValueError as ve: clean_error = clean_ip_value_error(ve) messages.error(f'Value is neither a valid set name nor a valid prefix: {address}: {clean_error}') return None if range_operator and not re_range_operator.match(range_operator): messages.error(f'Invalid range operator {range_operator} in value: {value}') return None parsed_ip_str = str(ip) if ip.version() == 4 and ip.prefixlen() == 32: parsed_ip_str += '/32' if ip.version() == 6 and ip.prefixlen() == 128: parsed_ip_str += '/128' if range_operator: parsed_ip_str += '^' + range_operator if parsed_ip_str != value: messages.info(f'Route set member {value} was reformatted as {parsed_ip_str}') return RPSLFieldParseResult(parsed_ip_str)
def _routes_for_origin(self, object_class: str, origin: str) -> str: """ Resolve all route(6)s for an origin, returning a space-separated list of all originating prefixes, not including duplicates. """ try: _, asn = parse_as_number(origin) except ValidationError as ve: raise WhoisQueryParserException(str(ve)) query = self._prepare_query().object_classes([object_class]).asn(asn) query_result = self.database_handler.execute_query(query) prefixes = [r['parsed_data'][object_class] for r in query_result] return ' '.join(OrderedSet(prefixes))
def text_search(self, value: str, extract_asn_ip=True): """ Search the database for a specific free text. In order, this attempts: - If the value is a valid AS number, return all as-block, as-set, aut-num objects relating or including that AS number. - If the value is a valid IP address or network, return all objects that relate to that resource and any less specifics. - Otherwise, return all objects where the RPSL primary key is exactly this value, or it matches part of a person/role name (not nic-hdl, their actual person/role attribute value). If extract_asn_ip is False, the first two steps are skipped. """ self._check_query_frozen() if extract_asn_ip: try: _, asn = parse_as_number(value) return self.object_classes(['as-block', 'as-set', 'aut-num']).asn_less_specific(asn) except ValidationError: pass try: ip = IP(value) return self.ip_less_specific(ip) except ValueError: pass counter = self._lookup_attr_counter self._lookup_attr_counter += 1 fltr = sa.or_( self.columns.rpsl_pk == value.upper(), sa.and_( self.columns.object_class == 'person', sa.text( f"parsed_data->>'person' ILIKE :lookup_attr_text_search{counter}" )), sa.and_( self.columns.object_class == 'role', sa.text( f"parsed_data->>'role' ILIKE :lookup_attr_text_search{counter}" )), ) self.statement = self.statement.where(fltr).params( **{f'lookup_attr_text_search{counter}': '%' + value + '%'}) return self
def __init__(self, prefix: IP, asn: str, max_length: str, trust_anchor: str): try: self.prefix = prefix self.prefix_str = str(prefix) _, self.asn = parse_as_number(asn) self.max_length = int(max_length) self.trust_anchor = trust_anchor except ValueError as ve: msg = f'Invalid value in ROA: {ve}' logger.error(msg) raise ROAParserException(msg) if self.max_length < self.prefix.prefixlen(): msg = f'Invalid ROA: prefix size {self.prefix.prefixlen()} is smaller than max length {max_length} in ' \ f'ROA for {self.prefix} / AS{self.asn}' logger.error(msg) raise ROAParserException(msg)
def __init__(self, rpki_json_str: str, slurm_json_str: Optional[str], database_handler: DatabaseHandler): self.roa_objs: List[ROA] = [] self._filtered_asns: Set[int] = set() self._filtered_prefixes: IPSet = IPSet() self._filtered_combined: Dict[int, IPSet] = defaultdict(IPSet) self._load_roa_dicts(rpki_json_str) if slurm_json_str: self._load_slurm(slurm_json_str) scopefilter_validator = ScopeFilterValidator() for roa_dict in self._roa_dicts: try: _, asn = parse_as_number(roa_dict['asn'], permit_plain=True) prefix = IP(roa_dict['prefix']) ta = roa_dict['ta'] if ta != SLURM_TRUST_ANCHOR: if asn in self._filtered_asns: continue if any([prefix in self._filtered_prefixes]): continue if any([prefix in self._filtered_combined.get(asn, [])]): continue roa_obj = ROA(prefix, asn, roa_dict['maxLength'], ta) except KeyError as ke: msg = f'Unable to parse ROA record: missing key {ke} -- full record: {roa_dict}' logger.error(msg) raise ROAParserException(msg) except ValueError as ve: msg = f'Invalid value in ROA or SLURM: {ve}' logger.error(msg) raise ROAParserException(msg) roa_obj.save(database_handler, scopefilter_validator) self.roa_objs.append(roa_obj)
def parse_set_name(prefix: str, value: str, messages: RPSLParserMessages, strict_validation=True) -> Optional[RPSLFieldParseResult]: assert prefix in reserved_prefixes input_components = value.split(":") output_components: List[str] = [] if strict_validation and not any([c.upper().startswith(prefix) for c in input_components]): messages.error(f"Invalid set name {value}: at least one component must be " f"an actual set name (i.e. start with {prefix})") return None for component in input_components: if strict_validation and component.upper() in reserved_words: messages.error(f"Invalid set name {value}: component {component} is a reserved word") return None parsed_as_number = None try: parsed_as_number, _ = parse_as_number(component) except ValidationError: pass if not re_generic_name.match(component.upper()) and not parsed_as_number: messages.error( f"Invalid set {value}: component {component} is not a valid AS number nor a valid set name" ) return None if strict_validation and not parsed_as_number and not component.upper().startswith(prefix): messages.error(f"Invalid set {value}: component {component} is not a valid AS number, " f"nor does it start with {prefix}") return None if parsed_as_number: output_components.append(parsed_as_number) else: output_components.append(component) parsed_value = ":".join(output_components) if parsed_value != value: messages.info(f"Set name {value} was reformatted as {parsed_value}") return RPSLFieldParseResult(parsed_value)