def parse(self,
value: str,
messages: RPSLParserMessages,
strict_validation=True) -> Optional[RPSLFieldParseResult]:
if '-' not in value:
messages.error(
f'Invalid AS range: {value}: does not contain a hyphen')
return None
as1_raw, as2_raw = map(str.strip, value.split('-', 1))
try:
as1_str, as1_int = parse_as_number(as1_raw)
as2_str, as2_int = parse_as_number(as2_raw)
except ValidationError as ve:
messages.error(str(ve))
return None
if as1_int > as2_int: # type: ignore
messages.error(
f'Invalid AS range: {value}: first AS is higher then second AS'
)
return None
parsed_value = f'{as1_str} - {as2_str}'
if parsed_value != value:
messages.info(
f'AS range {value} was reformatted as {parsed_value}')
return RPSLFieldParseResult(parsed_value,
asn_first=as1_int,
asn_last=as2_int)
def _recursive_set_resolve(self,
members: Set[str],
sets_seen=None,
root_source: Optional[str] = None) -> Set[str]:
"""
Resolve all members of a number of sets, recursively.
For each set in members, determines whether it has been seen already (to prevent
infinite recursion), ignores it if already seen, and then either adds
it directly or adds it to a set that requires further resolving.
If root_source is set, the root object is only looked for in that source -
resolving is then continued using the currently set sources.
"""
if not sets_seen:
sets_seen = set()
if all([member in sets_seen for member in members]):
return set()
sets_seen.update(members)
set_members = set()
resolved_as_members = set()
sub_members, leaf_members = self._find_set_members(
members, limit_source=root_source)
for sub_member in sub_members:
if self._current_set_root_object_class is None or self._current_set_root_object_class == 'route-set':
try:
IP(sub_member.split('^')[0])
set_members.add(sub_member)
continue
except ValueError:
pass
# AS numbers are permitted in route-sets and as-sets, per RFC 2622 5.3.
# When an AS number is encountered as part of route-set resolving,
# the prefixes originating from that AS should be added to the response.
try:
as_number_formatted, _ = parse_as_number(sub_member)
if self._current_set_root_object_class == 'route-set':
set_members.update(
self.preloader.routes_for_origins(
[as_number_formatted], self.sources))
resolved_as_members.add(sub_member)
else:
set_members.add(sub_member)
continue
except ValueError:
pass
self._current_set_maximum_depth -= 1
if self._current_set_maximum_depth == 0:
return set_members | sub_members | leaf_members
further_resolving_required = sub_members - set_members - sets_seen - resolved_as_members - self._current_excluded_sets
new_members = self._recursive_set_resolve(further_resolving_required,
sets_seen)
set_members.update(new_members)
return set_members
def parse(self, value: str, messages: RPSLParserMessages, strict_validation=True) -> Optional[RPSLFieldParseResult]:
try:
parsed_str, parsed_int = parse_as_number(value)
except ValidationError as ve:
messages.error(str(ve))
return None
if parsed_str and parsed_str.upper() != value.upper():
messages.info(f"AS number {value} was reformatted as {parsed_str}")
return RPSLFieldParseResult(parsed_str, asn_first=parsed_int, asn_last=parsed_int)
def parse_set_name(prefixes: List[str],
value: str,
messages: RPSLParserMessages,
strict_validation=True) -> Optional[RPSLFieldParseResult]:
assert all([prefix in reserved_prefixes for prefix in prefixes])
input_components = value.split(':')
output_components: List[str] = []
prefix_display = '/'.join(prefixes)
if strict_validation and len(input_components) > 5:
messages.error(f'Set names can have a maximum of five components.')
return None
if strict_validation and not any(
[c.upper().startswith(tuple(prefixes)) for c in input_components]):
messages.error(
f'Invalid set name {value}: at least one component must be '
f'an actual set name (i.e. start with {prefix_display})')
return None
for component in input_components:
if strict_validation and component.upper() in reserved_words:
messages.error(
f'Invalid set name {value}: component {component} is a reserved word'
)
return None
parsed_as_number = None
try:
parsed_as_number, _ = parse_as_number(component)
except ValidationError:
pass
if not re_generic_name.match(
component.upper()) and not parsed_as_number:
messages.error(
f'Invalid set {value}: component {component} is not a valid AS number nor a valid set name'
)
return None
if strict_validation and not parsed_as_number and not component.upper(
).startswith(tuple(prefixes)):
messages.error(
f'Invalid set {value}: component {component} is not a valid AS number, '
f'nor does it start with {prefix_display}')
return None
if parsed_as_number:
output_components.append(parsed_as_number)
else:
output_components.append(component)
parsed_value = ':'.join(output_components)
if parsed_value != value:
messages.info(f'Set name {value} was reformatted as {parsed_value}')
return RPSLFieldParseResult(parsed_value)
def _recursive_set_resolve(self,
members: Set[str],
sets_seen=None) -> Set[str]:
"""
Resolve all members of a number of sets, recursively.
For each set in members, determines whether it has been seen already (to prevent
infinite recursion), ignores it if already seen, and then either adds
it directly or adds it to a set that requires further resolving.
"""
if not sets_seen:
sets_seen = set()
if all([member in sets_seen for member in members]):
return set()
sets_seen.update(members)
set_members = set()
sub_members, leaf_members = self._find_set_members(members)
set_members.update(leaf_members)
for sub_member in sub_members:
try:
IP(sub_member)
set_members.add(sub_member)
continue
except ValueError:
pass
try:
parse_as_number(sub_member)
set_members.add(sub_member)
continue
except ValueError:
pass
further_resolving_required = sub_members - set_members - sets_seen
new_members = self._recursive_set_resolve(further_resolving_required,
sets_seen)
set_members.update(new_members)
return set_members
def _routes_for_origin(self, origin: str, ip_version: Optional[int]=None) -> str:
"""
Resolve all route(6)s prefixes for an origin, returning a space-separated list
of all originating prefixes, not including duplicates.
"""
try:
origin_formatted, _ = parse_as_number(origin)
except ValidationError as ve:
raise WhoisQueryParserException(str(ve))
prefixes = self.preloader.routes_for_origins([origin_formatted], ip_version=ip_version)
return ' '.join(prefixes)
def parse(self, value: str, messages: RPSLParserMessages, strict_validation=True) -> Optional[RPSLFieldParseResult]:
if '^' in value:
address, range_operator = value.split('^', maxsplit=1)
if not range_operator:
messages.error(f'Missing range operator in value: {value}')
return None
else:
address = value
range_operator = ''
parse_set_result_messages = RPSLParserMessages()
parse_set_result = parse_set_name(['RS-', 'AS-'], address, parse_set_result_messages, strict_validation)
if parse_set_result and not parse_set_result_messages.errors():
result_value = parse_set_result.value
if range_operator:
result_value += '^' + range_operator
if result_value != value:
messages.info(f'Route set member {value} was reformatted as {result_value}')
return RPSLFieldParseResult(value=result_value)
try:
parsed_str, parsed_int = parse_as_number(address)
result_value = parsed_str
if range_operator:
result_value += '^' + range_operator
if result_value != value:
messages.info(f'Route set member {value} was reformatted as {result_value}')
return RPSLFieldParseResult(value=result_value)
except ValidationError:
pass
try:
ip_version = self.ip_version if self.ip_version else 0
ip = IP(address, ipversion=ip_version)
except ValueError as ve:
clean_error = clean_ip_value_error(ve)
messages.error(f'Value is neither a valid set name nor a valid prefix: {address}: {clean_error}')
return None
if range_operator and not re_range_operator.match(range_operator):
messages.error(f'Invalid range operator {range_operator} in value: {value}')
return None
parsed_ip_str = str(ip)
if ip.version() == 4 and ip.prefixlen() == 32:
parsed_ip_str += '/32'
if ip.version() == 6 and ip.prefixlen() == 128:
parsed_ip_str += '/128'
if range_operator:
parsed_ip_str += '^' + range_operator
if parsed_ip_str != value:
messages.info(f'Route set member {value} was reformatted as {parsed_ip_str}')
return RPSLFieldParseResult(parsed_ip_str)
def _routes_for_origin(self, object_class: str, origin: str) -> str:
"""
Resolve all route(6)s for an origin, returning a space-separated list
of all originating prefixes, not including duplicates.
"""
try:
_, asn = parse_as_number(origin)
except ValidationError as ve:
raise WhoisQueryParserException(str(ve))
query = self._prepare_query().object_classes([object_class]).asn(asn)
query_result = self.database_handler.execute_query(query)
prefixes = [r['parsed_data'][object_class] for r in query_result]
return ' '.join(OrderedSet(prefixes))
def text_search(self, value: str, extract_asn_ip=True):
"""
Search the database for a specific free text.
In order, this attempts:
- If the value is a valid AS number, return all as-block, as-set, aut-num objects
relating or including that AS number.
- If the value is a valid IP address or network, return all objects that relate to
that resource and any less specifics.
- Otherwise, return all objects where the RPSL primary key is exactly this value,
or it matches part of a person/role name (not nic-hdl, their
actual person/role attribute value).
If extract_asn_ip is False, the first two steps are skipped.
"""
self._check_query_frozen()
if extract_asn_ip:
try:
_, asn = parse_as_number(value)
return self.object_classes(['as-block', 'as-set',
'aut-num']).asn_less_specific(asn)
except ValidationError:
pass
try:
ip = IP(value)
return self.ip_less_specific(ip)
except ValueError:
pass
counter = self._lookup_attr_counter
self._lookup_attr_counter += 1
fltr = sa.or_(
self.columns.rpsl_pk == value.upper(),
sa.and_(
self.columns.object_class == 'person',
sa.text(
f"parsed_data->>'person' ILIKE :lookup_attr_text_search{counter}"
)),
sa.and_(
self.columns.object_class == 'role',
sa.text(
f"parsed_data->>'role' ILIKE :lookup_attr_text_search{counter}"
)),
)
self.statement = self.statement.where(fltr).params(
**{f'lookup_attr_text_search{counter}': '%' + value + '%'})
return self
def __init__(self, prefix: IP, asn: str, max_length: str,
trust_anchor: str):
try:
self.prefix = prefix
self.prefix_str = str(prefix)
_, self.asn = parse_as_number(asn)
self.max_length = int(max_length)
self.trust_anchor = trust_anchor
except ValueError as ve:
msg = f'Invalid value in ROA: {ve}'
logger.error(msg)
raise ROAParserException(msg)
if self.max_length < self.prefix.prefixlen():
msg = f'Invalid ROA: prefix size {self.prefix.prefixlen()} is smaller than max length {max_length} in ' \
f'ROA for {self.prefix} / AS{self.asn}'
logger.error(msg)
raise ROAParserException(msg)
def __init__(self, rpki_json_str: str, slurm_json_str: Optional[str],
database_handler: DatabaseHandler):
self.roa_objs: List[ROA] = []
self._filtered_asns: Set[int] = set()
self._filtered_prefixes: IPSet = IPSet()
self._filtered_combined: Dict[int, IPSet] = defaultdict(IPSet)
self._load_roa_dicts(rpki_json_str)
if slurm_json_str:
self._load_slurm(slurm_json_str)
scopefilter_validator = ScopeFilterValidator()
for roa_dict in self._roa_dicts:
try:
_, asn = parse_as_number(roa_dict['asn'], permit_plain=True)
prefix = IP(roa_dict['prefix'])
ta = roa_dict['ta']
if ta != SLURM_TRUST_ANCHOR:
if asn in self._filtered_asns:
continue
if any([prefix in self._filtered_prefixes]):
continue
if any([prefix in self._filtered_combined.get(asn, [])]):
continue
roa_obj = ROA(prefix, asn, roa_dict['maxLength'], ta)
except KeyError as ke:
msg = f'Unable to parse ROA record: missing key {ke} -- full record: {roa_dict}'
logger.error(msg)
raise ROAParserException(msg)
except ValueError as ve:
msg = f'Invalid value in ROA or SLURM: {ve}'
logger.error(msg)
raise ROAParserException(msg)
roa_obj.save(database_handler, scopefilter_validator)
self.roa_objs.append(roa_obj)
def parse_set_name(prefix: str, value: str, messages: RPSLParserMessages, strict_validation=True) -> Optional[RPSLFieldParseResult]:
assert prefix in reserved_prefixes
input_components = value.split(":")
output_components: List[str] = []
if strict_validation and not any([c.upper().startswith(prefix) for c in input_components]):
messages.error(f"Invalid set name {value}: at least one component must be "
f"an actual set name (i.e. start with {prefix})")
return None
for component in input_components:
if strict_validation and component.upper() in reserved_words:
messages.error(f"Invalid set name {value}: component {component} is a reserved word")
return None
parsed_as_number = None
try:
parsed_as_number, _ = parse_as_number(component)
except ValidationError:
pass
if not re_generic_name.match(component.upper()) and not parsed_as_number:
messages.error(
f"Invalid set {value}: component {component} is not a valid AS number nor a valid set name"
)
return None
if strict_validation and not parsed_as_number and not component.upper().startswith(prefix):
messages.error(f"Invalid set {value}: component {component} is not a valid AS number, "
f"nor does it start with {prefix}")
return None
if parsed_as_number:
output_components.append(parsed_as_number)
else:
output_components.append(component)
parsed_value = ":".join(output_components)
if parsed_value != value:
messages.info(f"Set name {value} was reformatted as {parsed_value}")
return RPSLFieldParseResult(parsed_value)
