def init_detection(self) -> NoReturn:
"""
Updates rules everytime it receives a new configuration.
"""
log.info("Initiating detection...")
log.info("Starting building detection prefix tree...")
self.prefix_tree = {
"v4": pytricia.PyTricia(32),
"v6": pytricia.PyTricia(128),
}
raw_prefix_count = 0
for rule in self.rules:
try:
rule_translated_origin_asn_set = set()
for asn in rule["origin_asns"]:
this_translated_asn_list = flatten(translate_asn_range(asn))
rule_translated_origin_asn_set.update(
set(this_translated_asn_list)
)
rule["origin_asns"] = list(rule_translated_origin_asn_set)
rule_translated_neighbor_set = set()
for asn in rule["neighbors"]:
this_translated_asn_list = flatten(translate_asn_range(asn))
rule_translated_neighbor_set.update(
set(this_translated_asn_list)
)
rule["neighbors"] = list(rule_translated_neighbor_set)
conf_obj = {
"origin_asns": rule["origin_asns"],
"neighbors": rule["neighbors"],
"policies": set(rule["policies"]),
"community_annotations": rule["community_annotations"],
}
for prefix in rule["prefixes"]:
for translated_prefix in translate_rfc2622(prefix):
ip_version = get_ip_version(translated_prefix)
if self.prefix_tree[ip_version].has_key(translated_prefix):
node = self.prefix_tree[ip_version][translated_prefix]
else:
node = {
"prefix": translated_prefix,
"data": {"confs": []},
}
self.prefix_tree[ip_version].insert(
translated_prefix, node
)
node["data"]["confs"].append(conf_obj)
raw_prefix_count += 1
except Exception:
log.exception("Exception")
log.info(
"{} prefixes integrated in detection prefix tree in total".format(
raw_prefix_count
)
)
log.info("Finished building detection prefix tree.")
log.info("Detection initiated, configured and running.")
def build_prefix_tree(self):
log.info("Starting building autoignore prefix tree...")
self.prefix_tree = {
"v4": pytricia.PyTricia(32),
"v6": pytricia.PyTricia(128),
}
raw_prefix_count = 0
for key in self.autoignore_rules:
try:
rule = self.autoignore_rules[key]
for prefix in rule["prefixes"]:
for translated_prefix in translate_rfc2622(prefix):
ip_version = get_ip_version(translated_prefix)
if self.prefix_tree[ip_version].has_key(
translated_prefix):
node = self.prefix_tree[ip_version][
translated_prefix]
else:
node = {
"prefix": translated_prefix,
"rule_key": key
}
self.prefix_tree[ip_version].insert(
translated_prefix, node)
raw_prefix_count += 1
except Exception:
log.exception("Exception")
log.info(
"{} prefixes integrated in autoignore prefix tree in total".
format(raw_prefix_count))
log.info("Finished building autoignore prefix tree.")
def handle_mitigation_request(self, message):
message.ack()
hijack_event = message.payload
ip_version = get_ip_version(hijack_event["prefix"])
if hijack_event["prefix"] in self.prefix_tree[ip_version]:
prefix_node = self.prefix_tree[ip_version][
hijack_event["prefix"]]
mitigation_action = prefix_node["data"]["mitigation"][0]
if mitigation_action == "manual":
log.info("starting manual mitigation of hijack {}".format(
hijack_event))
else:
log.info(
"starting custom mitigation of hijack {} using '{}' script"
.format(hijack_event, mitigation_action))
hijack_event_str = json.dumps(hijack_event)
subprocess.Popen(
[mitigation_action, "-i", hijack_event_str],
shell=False,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
# do something
mit_started = {"key": hijack_event["key"], "time": time.time()}
self.producer.publish(
mit_started,
exchange=self.mitigation_exchange,
routing_key="mit-start",
priority=2,
serializer="ujson",
)
else:
log.warn("no rule for hijack {}".format(hijack_event))
def init_mitigation(self):
log.info("Initiating mitigation...")
log.info("Starting building mitigation prefix tree...")
self.prefix_tree = {
"v4": pytricia.PyTricia(32),
"v6": pytricia.PyTricia(128),
}
raw_prefix_count = 0
for rule in self.rules:
try:
for prefix in rule["prefixes"]:
for translated_prefix in translate_rfc2622(prefix):
ip_version = get_ip_version(translated_prefix)
node = {
"prefix": translated_prefix,
"data": {
"mitigation": rule["mitigation"]
},
}
self.prefix_tree[ip_version].insert(
translated_prefix, node)
raw_prefix_count += 1
except Exception:
log.exception("Exception")
log.info(
"{} prefixes integrated in mitigation prefix tree in total".
format(raw_prefix_count))
log.info("Finished building mitigation prefix tree.")
log.info("Mitigation initiated, configured and running.")
def start_monitors(self):
log.info("Initiating monitor...")
for proc_id in self.process_ids:
try:
proc_id[1].terminate()
except ProcessLookupError:
log.exception("process terminate")
self.process_ids.clear()
self.prefixes.clear()
log.info("Starting building monitor prefix tree...")
self.prefix_tree = {
"v4": pytricia.PyTricia(32),
"v6": pytricia.PyTricia(128),
}
raw_prefix_count = 0
for rule in self.rules:
try:
for prefix in rule["prefixes"]:
for translated_prefix in translate_rfc2622(prefix):
ip_version = get_ip_version(translated_prefix)
self.prefix_tree[ip_version].insert(translated_prefix, "")
raw_prefix_count += 1
except Exception:
log.exception("Exception")
log.info(
"{} prefixes integrated in monitor prefix tree in total".format(
raw_prefix_count
)
)
log.info("Finished building monitor prefix tree.")
# only keep super prefixes for monitors
log.info("Calculating monitored prefixes for monitor to supervise...")
for ip_version in self.prefix_tree:
for prefix in self.prefix_tree[ip_version]:
worst_prefix = search_worst_prefix(
prefix, self.prefix_tree[ip_version]
)
if worst_prefix:
self.prefixes.add(worst_prefix)
dump_json(list(self.prefixes), self.prefix_file)
log.info("Calculated monitored prefixes for monitor to supervise.")
log.info("Initiating configured monitoring instances....")
self.init_ris_instance()
self.init_exabgp_instance()
self.init_bgpstreamhist_instance()
self.init_bgpstreamlive_instance()
self.init_bgpstreamkafka_instance()
log.info("All configured monitoring instances initiated.")
log.info("Monitor initiated, configured and running.")
def comm_annotate_hijack(self, monitor_event: Dict, hijack: Dict) -> NoReturn:
"""
Annotates a hijack based on community checks (modifies "community_annotation"
field in-place)
"""
try:
if hijack.get("community_annotation", "NA") in [None, "", "NA"]:
hijack["community_annotation"] = "NA"
bgp_update_communities = set()
for comm_as_value in monitor_event["communities"]:
community = "{}:{}".format(comm_as_value[0], comm_as_value[1])
bgp_update_communities.add(community)
ip_version = get_ip_version(monitor_event["prefix"])
if monitor_event["prefix"] in self.prefix_tree[ip_version]:
prefix_node = self.prefix_tree[ip_version][monitor_event["prefix"]]
for item in prefix_node["data"]["confs"]:
annotations = []
for annotation_element in item.get("community_annotations", []):
for annotation in annotation_element:
annotations.append(annotation)
for annotation_element in item.get("community_annotations", []):
for annotation in annotation_element:
for community_rule in annotation_element[annotation]:
in_communities = set(community_rule.get("in", []))
out_communities = set(community_rule.get("out", []))
if (
in_communities <= bgp_update_communities
and out_communities.isdisjoint(
bgp_update_communities
)
):
if hijack["community_annotation"] == "NA":
hijack["community_annotation"] = annotation
elif annotations.index(
annotation
) < annotations.index(
hijack["community_annotation"]
):
hijack["community_annotation"] = annotation
except Exception:
log.exception("exception")
def find_best_prefix_node(self, prefix):
ip_version = get_ip_version(prefix)
if prefix in self.prefix_tree[ip_version]:
return self.prefix_tree[ip_version][prefix]
return None
def commit_hijack(
self, monitor_event: Dict, hijacker: int, hij_dimensions: List[str]
) -> NoReturn:
"""
Commit new or update an existing hijack to the database.
It uses redis server to store ongoing hijacks information
to not stress the db.
"""
hij_type = "|".join(hij_dimensions)
redis_hijack_key = redis_key(monitor_event["prefix"], hijacker, hij_type)
if "hij_key" in monitor_event:
monitor_event["final_redis_hijack_key"] = redis_hijack_key
hijack_value = {
"prefix": monitor_event["prefix"],
"hijack_as": hijacker,
"type": hij_type,
"time_started": monitor_event["timestamp"],
"time_last": monitor_event["timestamp"],
"peers_seen": {monitor_event["peer_asn"]},
"monitor_keys": {monitor_event["key"]},
"configured_prefix": monitor_event["matched_prefix"],
"timestamp_of_config": self.timestamp,
"end_tag": None,
"outdated_parent": None,
"rpki_status": "NA",
}
if (
RPKI_VALIDATOR_ENABLED == "true"
and self.rtrmanager
and monitor_event["path"]
):
try:
asn = monitor_event["path"][-1]
if "/" in monitor_event["prefix"]:
network, netmask = monitor_event["prefix"].split("/")
# /32 or /128
else:
ip_version = get_ip_version(monitor_event["prefix"])
network = monitor_event["prefix"]
netmask = 32
if ip_version == "v6":
netmask = 128
redis_rpki_asn_prefix_key = "rpki_as{}_p{}".format(
asn, monitor_event["prefix"]
)
redis_rpki_status = self.redis.get(redis_rpki_asn_prefix_key)
if not redis_rpki_status:
rpki_status = get_rpki_val_result(
self.rtrmanager, asn, network, int(netmask)
)
else:
rpki_status = redis_rpki_status.decode()
hijack_value["rpki_status"] = rpki_status
# the default refresh interval for the RPKI RTR manager is 3600 seconds
self.redis.set(redis_rpki_asn_prefix_key, rpki_status, ex=3600)
except Exception:
log.exception("exception")
if (
"hij_key" in monitor_event
and monitor_event["initial_redis_hijack_key"]
!= monitor_event["final_redis_hijack_key"]
):
hijack_value["outdated_parent"] = monitor_event["hij_key"]
# identify the number of infected ases
hijack_value["asns_inf"] = set()
if hij_dimensions[1] in {"0", "1"}:
hijack_value["asns_inf"] = set(
monitor_event["path"][: -(int(hij_dimensions[1]) + 1)]
)
elif hij_dimensions[3] == "L":
hijack_value["asns_inf"] = set(monitor_event["path"][:-2])
# assume the worst-case scenario of a type-2 hijack
elif len(monitor_event["path"]) > 2:
hijack_value["asns_inf"] = set(monitor_event["path"][:-3])
# make the following operation atomic using blpop (blocking)
# first, make sure that the semaphore is initialized
if self.redis.getset("{}token_active".format(redis_hijack_key), 1) != b"1":
redis_pipeline = self.redis.pipeline()
redis_pipeline.lpush("{}token".format(redis_hijack_key), "token")
# lock, by extracting the token (other processes that access
# it at the same time will be blocked)
# attention: it is important that this command is batched in the
# pipeline since the db may async delete
# the token
redis_pipeline.blpop("{}token".format(redis_hijack_key))
redis_pipeline.execute()
else:
# lock, by extracting the token (other processes that access it
# at the same time will be blocked)
token = self.redis.blpop("{}token".format(redis_hijack_key), timeout=60)
# if timeout after 60 seconds, return without hijack alert
# since this means that sth has been purged in the meanwhile (e.g., due to outdated hijack
# in another instance; a detector cannot be stuck for a whole minute in a single hijack BGP update)
if not token:
log.info(
"Monitor event {} encountered redis token timeout and will be cleared as benign for hijack {}".format(
str(monitor_event), redis_hijack_key
)
)
return
# proceed now that we have clearance
redis_pipeline = self.redis.pipeline()
try:
result = self.redis.get(redis_hijack_key)
if result:
result = json.loads(result)
result["time_started"] = min(
result["time_started"], hijack_value["time_started"]
)
result["time_last"] = max(
result["time_last"], hijack_value["time_last"]
)
result["peers_seen"] = set(result["peers_seen"])
result["peers_seen"].update(hijack_value["peers_seen"])
result["asns_inf"] = set(result["asns_inf"])
result["asns_inf"].update(hijack_value["asns_inf"])
# no update since db already knows!
result["monitor_keys"] = hijack_value["monitor_keys"]
self.comm_annotate_hijack(monitor_event, result)
result["outdated_parent"] = hijack_value["outdated_parent"]
result["bgpupdate_keys"] = set(result["bgpupdate_keys"])
result["bgpupdate_keys"].add(monitor_event["key"])
result["rpki_status"] = hijack_value["rpki_status"]
else:
hijack_value["time_detected"] = time.time()
hijack_value["key"] = get_hash(
[
monitor_event["prefix"],
hijacker,
hij_type,
"{0:.6f}".format(hijack_value["time_detected"]),
]
)
hijack_value["bgpupdate_keys"] = {monitor_event["key"]}
redis_pipeline.sadd("persistent-keys", hijack_value["key"])
result = hijack_value
self.comm_annotate_hijack(monitor_event, result)
mail_log.info(
"{}".format(
json.dumps(hijack_log_field_formatter(result), indent=4)
),
extra={
"community_annotation": result.get(
"community_annotation", "NA"
)
},
)
redis_pipeline.set(redis_hijack_key, json.dumps(result))
# store the origin, neighbor combination for this hijack BGP update
origin = None
neighbor = None
if monitor_event["path"]:
origin = monitor_event["path"][-1]
if len(monitor_event["path"]) > 1:
neighbor = monitor_event["path"][-2]
redis_pipeline.sadd(
"hij_orig_neighb_{}".format(redis_hijack_key),
"{}_{}".format(origin, neighbor),
)
# store the prefix and peer ASN for this hijack BGP update
redis_pipeline.sadd(
"prefix_{}_peer_{}_hijacks".format(
monitor_event["prefix"], monitor_event["peer_asn"]
),
redis_hijack_key,
)
redis_pipeline.sadd(
"hijack_{}_prefixes_peers".format(redis_hijack_key),
"{}_{}".format(monitor_event["prefix"], monitor_event["peer_asn"]),
)
except Exception:
log.exception("exception")
finally:
# execute whatever has been accumulated in redis till now
redis_pipeline.execute()
# publish hijack
self.publish_hijack_fun(result, redis_hijack_key)
hij_log.info(
"{}".format(json.dumps(hijack_log_field_formatter(result))),
extra={
"community_annotation": result.get("community_annotation", "NA")
},
)
# unlock, by pushing back the token (at most one other process
# waiting will be unlocked)
redis_pipeline = self.redis.pipeline()
redis_pipeline.set("{}token_active".format(redis_hijack_key), 1)
redis_pipeline.lpush("{}token".format(redis_hijack_key), "token")
redis_pipeline.execute()
def handle_bgp_update(self, message: Dict) -> NoReturn:
"""
Callback function that runs the main logic of
detecting hijacks for every bgp update.
"""
# log.debug('{}'.format(message))
if isinstance(message, dict):
monitor_event = message
else:
message.ack()
monitor_event = message.payload
monitor_event["path"] = monitor_event["as_path"]
monitor_event["timestamp"] = datetime(
*map(int, re.findall(r"\d+", monitor_event["timestamp"]))
).timestamp()
raw = monitor_event.copy()
# mark the initial redis hijack key since it may change upon
# outdated checks
if "hij_key" in monitor_event:
monitor_event["initial_redis_hijack_key"] = redis_key(
monitor_event["prefix"],
monitor_event["hijack_as"],
monitor_event["hij_type"],
)
is_hijack = False
if monitor_event["type"] == "A":
monitor_event["path"] = Detection.Worker.__clean_as_path(
monitor_event["path"]
)
ip_version = get_ip_version(monitor_event["prefix"])
if monitor_event["prefix"] in self.prefix_tree[ip_version]:
prefix_node = self.prefix_tree[ip_version][monitor_event["prefix"]]
monitor_event["matched_prefix"] = prefix_node["prefix"]
try:
path_hijacker = -1
pol_hijacker = -1
hij_dimensions = [
"-",
"-",
"-",
"-",
] # prefix, path, dplane, policy
hij_dimension_index = 0
for func_dim in self.__hijack_dimension_checker_gen():
if hij_dimension_index == 0:
# prefix dimension
for func_pref in func_dim():
hij_dimensions[hij_dimension_index] = func_pref(
monitor_event, prefix_node
)
if hij_dimensions[hij_dimension_index] != "-":
break
elif hij_dimension_index == 1:
# path type dimension
for func_path in func_dim(len(monitor_event["path"])):
(
path_hijacker,
hij_dimensions[hij_dimension_index],
) = func_path(monitor_event, prefix_node)
if hij_dimensions[hij_dimension_index] != "-":
break
elif hij_dimension_index == 2:
# data plane dimension
for func_dplane in func_dim():
hij_dimensions[hij_dimension_index] = func_dplane(
monitor_event, prefix_node
)
if hij_dimensions[hij_dimension_index] != "-":
break
elif hij_dimension_index == 3:
# policy dimension
for func_pol in func_dim(len(monitor_event["path"])):
(
pol_hijacker,
hij_dimensions[hij_dimension_index],
) = func_pol(monitor_event, prefix_node)
if hij_dimensions[hij_dimension_index] != "-":
break
hij_dimension_index += 1
# check if dimension combination in hijack combinations
# and commit hijack
if hij_dimensions in HIJACK_DIM_COMBINATIONS:
is_hijack = True
# show pol hijacker only if the path hijacker is uncertain
hijacker = path_hijacker
if path_hijacker == -1 and pol_hijacker != -1:
hijacker = pol_hijacker
self.commit_hijack(monitor_event, hijacker, hij_dimensions)
except Exception:
log.exception("exception")
outdated_hijack = None
if not is_hijack and "hij_key" in monitor_event:
try:
# outdated hijack, benign from now on
redis_hijack_key = redis_key(
monitor_event["prefix"],
monitor_event["hijack_as"],
monitor_event["hij_type"],
)
outdated_hijack = self.redis.get(redis_hijack_key)
purge_redis_eph_pers_keys(
self.redis, redis_hijack_key, monitor_event["hij_key"]
)
# mark in DB only if it is the first time this hijack was purged (pre-existent in redis)
if outdated_hijack:
self.mark_outdated(
monitor_event["hij_key"], redis_hijack_key
)
except Exception:
log.exception("exception")
elif (
is_hijack
and "hij_key" in monitor_event
and monitor_event["initial_redis_hijack_key"]
!= monitor_event["final_redis_hijack_key"]
):
try:
outdated_hijack = self.redis.get(
monitor_event["initial_redis_hijack_key"]
)
# outdated hijack, but still a hijack; need key change
purge_redis_eph_pers_keys(
self.redis,
monitor_event["initial_redis_hijack_key"],
monitor_event["hij_key"],
)
# mark in DB only if it is the first time this hijack was purged (pre-existsent in redis)
if outdated_hijack:
self.mark_outdated(
monitor_event["hij_key"],
monitor_event["initial_redis_hijack_key"],
)
except Exception:
log.exception("exception")
elif not is_hijack:
self.gen_implicit_withdrawal(monitor_event)
self.mark_handled(raw)
if outdated_hijack:
try:
outdated_hijack = json.loads(outdated_hijack)
outdated_hijack["end_tag"] = "outdated"
mail_log.info(
"{}".format(
json.dumps(
hijack_log_field_formatter(outdated_hijack),
indent=4,
)
),
extra={
"community_annotation": outdated_hijack.get(
"community_annotation", "NA"
)
},
)
hij_log.info(
"{}".format(
json.dumps(hijack_log_field_formatter(outdated_hijack))
),
extra={
"community_annotation": outdated_hijack.get(
"community_annotation", "NA"
)
},
)
except Exception:
log.exception("exception")
elif monitor_event["type"] == "W":
self.producer.publish(
{
"prefix": monitor_event["prefix"],
"peer_asn": monitor_event["peer_asn"],
"timestamp": monitor_event["timestamp"],
"key": monitor_event["key"],
},
exchange=self.update_exchange,
routing_key="withdraw",
priority=0,
serializer="ujson",
)
def test_should_return_ipv4(self):
input = "127.0.0.1"
output = get_ip_version(input)
self.assertEqual("IPV4", output)
def test_should_return_none_for_invalid_ipv6(self):
input = "0:0:0:0:0:0:0:0:0"
output = get_ip_version(input)
self.assertEqual(None, output)
def test_should_return_ipv6(self):
input = "::1"
output = get_ip_version(input)
self.assertEqual("IPV6", output)
def normalize_ripe_ris(msg, prefix_tree):
msgs = []
if isinstance(msg, dict):
msg["key"] = None # initial placeholder before passing the validator
if "community" in msg:
msg["communities"] = [{
"asn": comm[0],
"value": comm[1]
} for comm in msg["community"]]
del msg["community"]
if "host" in msg:
msg["service"] = "ripe-ris|" + msg["host"]
del msg["host"]
if "peer_asn" in msg:
msg["peer_asn"] = int(msg["peer_asn"])
if "path" not in msg:
msg["path"] = []
if "timestamp" in msg:
msg["timestamp"] = float(msg["timestamp"])
if "type" in msg:
del msg["type"]
if "raw" in msg:
del msg["raw"]
if "origin" in msg:
del msg["origin"]
if "id" in msg:
del msg["id"]
if "announcements" in msg and "withdrawals" in msg:
# need 2 separate messages
# one for announcements
msg_ann = deepcopy(msg)
msg_ann["type"] = update_to_type["announcements"]
prefixes = []
for element in msg_ann["announcements"]:
if "prefixes" in element:
prefixes.extend(element["prefixes"])
for prefix in prefixes:
ip_version = get_ip_version(prefix)
try:
if prefix in prefix_tree[ip_version]:
new_msg = deepcopy(msg_ann)
new_msg["prefix"] = prefix
del new_msg["announcements"]
del new_msg["withdrawals"]
msgs.append(new_msg)
except Exception:
log.exception("exception")
# one for withdrawals
msg_wit = deepcopy(msg)
msg_wit["type"] = update_to_type["withdrawals"]
msg_wit["path"] = []
msg_wit["communities"] = []
prefixes = msg_wit["withdrawals"]
for prefix in prefixes:
ip_version = get_ip_version(prefix)
try:
if prefix in prefix_tree[ip_version]:
new_msg = deepcopy(msg_wit)
new_msg["prefix"] = prefix
del new_msg["announcements"]
del new_msg["withdrawals"]
msgs.append(new_msg)
except Exception:
log.exception("exception")
else:
for update_type in update_types:
if update_type in msg:
msg["type"] = update_to_type[update_type]
prefixes = []
for element in msg[update_type]:
if update_type == "announcements":
if "prefixes" in element:
prefixes.extend(element["prefixes"])
elif update_type == "withdrawals":
prefixes.append(element)
for prefix in prefixes:
ip_version = get_ip_version(prefix)
try:
if prefix in prefix_tree[ip_version]:
new_msg = deepcopy(msg)
new_msg["prefix"] = prefix
del new_msg[update_type]
msgs.append(new_msg)
except Exception:
log.exception("exception")
return msgs
def parse_ripe_ris(connection, prefixes_file, hosts):
exchange = Exchange("bgp-update",
channel=connection,
type="direct",
durable=False)
exchange.declare()
prefixes = load_json(prefixes_file)
assert prefixes is not None
prefix_tree = {"v4": pytricia.PyTricia(32), "v6": pytricia.PyTricia(128)}
for prefix in prefixes:
ip_version = get_ip_version(prefix)
prefix_tree[ip_version].insert(prefix, "")
ris_suffix = os.getenv("RIS_ID", "my_as")
validator = mformat_validator()
with Producer(connection) as producer:
while True:
try:
events = requests.get(
"https://ris-live.ripe.net/v1/stream/?format=json&client=artemis-{}"
.format(ris_suffix),
stream=True,
timeout=10,
)
# http://docs.python-requests.org/en/latest/user/advanced/#streaming-requests
iterator = events.iter_lines()
next(iterator)
for data in iterator:
try:
parsed = json.loads(data)
msg = parsed["data"]
if "type" in parsed and parsed["type"] == "ris_error":
log.error(msg)
# also check if ris host is in the configuration
elif ("type" in msg and msg["type"] == "UPDATE"
and (not hosts or msg["host"] in hosts)):
norm_ris_msgs = normalize_ripe_ris(
msg, prefix_tree)
for norm_ris_msg in norm_ris_msgs:
redis.set(
"ris_seen_bgp_update",
"1",
ex=int(
os.getenv(
"MON_TIMEOUT_LAST_BGP_UPDATE",
DEFAULT_MON_TIMEOUT_LAST_BGP_UPDATE,
)),
)
if validator.validate(norm_ris_msg):
norm_path_msgs = normalize_msg_path(
norm_ris_msg)
for norm_path_msg in norm_path_msgs:
key_generator(norm_path_msg)
log.debug(norm_path_msg)
producer.publish(
norm_path_msg,
exchange=exchange,
routing_key="update",
serializer="ujson",
)
else:
log.warning(
"Invalid format message: {}".format(
msg))
except Exception:
log.exception("exception message {}".format(data))
log.warning(
"Iterator ran out of data; the connection will be retried")
except Exception:
log.exception("server closed connection")
time.sleep(60)