def test_produce_consume_noack(self):
channel = self.c.channel()
producer = Producer(channel, self.e)
consumer = Consumer(channel, self.q, no_ack=True)
for i in range(10):
producer.publish({'foo': i}, routing_key='test_transport_memory')
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
self.assertEqual(len(_received), 10)
def send(self, topic, message):
"""Publishes a pulse message to the proper exchange."""
if not message:
Log.error("Expecting a message")
message._prepare()
if not self.connection:
self.connect()
producer = Producer(channel=self.connection,
exchange=Exchange(self.settings.exchange,
type='topic'),
routing_key=topic)
# The message is actually a simple envelope format with a payload and
# some metadata.
final_data = Data(
payload=message.data,
_meta=set_default(
{
'exchange':
self.settings.exchange,
'routing_key':
message.routing_key,
'serializer':
self.settings.serializer,
'sent':
time_to_string(
datetime.datetime.now(
timezone(self.settings.broker_timezone))),
'count':
self.count
}, message.metadata))
producer.publish(jsons.scrub(final_data),
serializer=self.settings.serializer)
self.count += 1
def grab_loop(config, stream, channel, exchange):
logger.info('init queue producer')
producer = Producer(channel)
width, height = config['stream']['size']
logger.info('stream size: {size}'.format(size=(width, height)))
blob_size = tuple(config['face_detector']['blob_size'])
logger.info('min grab size: {blob_size}'.format(blob_size=blob_size))
logger.info('check init frame from stream')
image = stream.read()
motion_detector_config = config['motion_detector']
logger.info('init motion detector: {motion_detector_config}'
.format(motion_detector_config=motion_detector_config))
motion = Motion(image_size=(width, height), **motion_detector_config)
logger.info('reset motion detector')
motion.reset(image, image)
while True:
image = stream.read()
if image is None:
logger.info('stream end')
break
frame = Frame(image, 0, 0, width, height)
rects = motion.detect(frame.image)
if len(rects):
# TODO: use biggest rect, not a first
# TODO: detector sizes is redundant feature after face detector became single
# TODO: fit_cut should be greater than blob size but less than entire image size
motion_frame = frame.cut(*rects[0])
logger.info('publish frame {bounds}'.format(bounds=motion_frame.bounds))
producer.publish(motion_frame.to_json(),
exchange=exchange,
serializer='json', compression='zlib')
def request_service(conn, message_request, exchange_request,
request_routing_key):
id_response = uuid()
queue_response = Queue(name=id_response,
exchange=exchange_request,
routing_key=id_response,
exclusive=True,
auto_delete=True)
message_request['header']['reply_to'] = id_response
conn.ensure_connection()
with Producer(conn) as producer:
producer.publish(json.dumps(message_request),
exchange=exchange_request.name,
routing_key=request_routing_key,
declare=[queue_response],
retry=True)
message_response = None
def on_response(body, message):
nonlocal message_response
message_response = json.loads(body)
try:
with Consumer(conn,
queues=queue_response,
callbacks=[on_response],
no_ack=True):
try:
while message_response is None:
conn.drain_events(timeout=10)
except socket.timeout:
return {'error': 'Can not connect to service'}
except Exception:
print("cannot create Consumer: " + request_routing_key)
return {'error': 'Cannot create Consumer'}
return message_response
def send(self):
try:
# Connection
conn = Connection(self.broker)
# Channel
channel = conn.channel()
# Exchange
task_exchange = Exchange(self._exchange_name,
type=self._queue_type)
# Queues
if self._queue_name:
queue = Queue(name=self._queue_name, channel=channel,
exchange=task_exchange,
routing_key=self._routing_key)
queue.declare()
# Producer
producer = Producer(exchange=task_exchange, channel=channel,
routing_key=self._routing_key)
# Send message
for message in self._msgs:
serialized_message = json.dumps(message, ensure_ascii=False)
producer.publish(serialized_message)
conn.close()
except Exception, e:
self.log.error(
u'QueueManagerError - Error on sending objects from queue.')
self.log.debug(e)
raise Exception(
'QueueManagerError - Error on sending objects to queue.')
def send(self, hijack_keys, action):
log.debug("Send 'multiple_action - {0}' hijack message with keys: {1}".
format(action, hijack_keys))
self.response = None
self.correlation_id = uuid()
callback_queue = Queue(
uuid(),
durable=False,
exclusive=True,
auto_delete=True,
max_priority=2,
consumer_arguments={"x-priority": 2},
)
with Connection(RABBITMQ_URI) as connection:
with Producer(connection) as producer:
producer.publish(
{
"keys": hijack_keys,
"action": action
},
exchange="",
routing_key="database.rpc.hijack-multiple-action",
retry=True,
declare=[callback_queue],
reply_to=callback_queue.name,
correlation_id=self.correlation_id,
priority=4,
serializer="ujson",
)
with Consumer(
connection,
on_message=self.on_response,
queues=[callback_queue],
accept=["ujson"],
):
while self.response is None:
connection.drain_events()
return self.response["status"] == "accepted"
def get_list_platforms(self):
print("Get list platforms from Registry")
message = {
"header": {
'reply_to': 'registry.response.collector.api_get_list_platforms',
'PlatformStatus': "active"
}
}
#message['message_monitor'] = self.message_monitor.monitor({}, 'collector', 'get_list_platforms')
queue = Queue(name='registry.request.api_get_list_platforms', exchange=self.exchange,
routing_key='registry.request.api_get_list_platforms', message_ttl=20)
routing_key = 'registry.request.api_get_list_platforms'
self.producer_connection.ensure_connection()
with Producer(self.producer_connection) as producer:
producer.publish(
json.dumps(message),
exchange=self.exchange.name,
routing_key=routing_key,
declare=[queue],
retry=True
)
def __init__(self, name=None):
if app.config['DEBUG']:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = BrokerConnection(AMQP_URL)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', AMQP_URL,
e)
raise RuntimeError
self.channel = self.connection.channel()
self.exchange_name = AMQP_TOPIC
self.exchange = Exchange(name=self.exchange_name,
type='fanout',
channel=self.channel)
self.producer = Producer(exchange=self.exchange, channel=self.channel)
super(FanoutPublisher, self).__init__(name)
LOG.info('Configured fanout publisher on topic "%s"', AMQP_TOPIC)
def test_produce_consume_noack(self):
producer = Producer(self._add_channel(self.p.channel()), self.e)
consumer = Consumer(self._add_channel(self.c.channel()), self.q,
no_ack=True)
for i in range(10):
producer.publish({'foo': i},
routing_key='test_transport_filesystem')
_received = []
def callback(message_data, message):
_received.append(message)
consumer.register_callback(callback)
consumer.consume()
while 1:
if len(_received) == 10:
break
self.c.drain_events()
assert len(_received) == 10
def data_publish(connection, channel):
exchange = Exchange('data_exchange', type="direct")
producer = Producer(exchange=exchange,
channel=channel,
routing_key='data_info')
queue = Queue(name="data_q",
exchange=exchange,
routing_key="data_info",
queue_arguments={'x-ha-policy': 'all'})
queue.maybe_bind(connection)
queue.declare()
message = [
'{"device_id": "30c15b59-650a-42a7", "check": false}',
'{ "device_id": "30c15b59-650a-42a7", "check": true}',
'{ "device_id": "30c15b59-650a-42a7", "check": true}',
'{"device_id": "30c15b59-650a-42a7", "check": false}',
'{"device_id": "30c15b59-650a-42a7", "check": true}'
]
for msg in message:
producer.publish(msg)
print "Message published", msg
time.sleep(2)
def send(self, message: BrightsideMessage):
# we want to expose our logger to the functions defined in inner scope, so put it in their outer scope
logger = self._logger
def _build_message_header(msg: BrightsideMessage) -> Dict:
return KombuMessageFactory(msg).create_message_header()
def _publish(sender: Producer) -> None:
logger.debug(
"Send message {body} to broker {amqpuri} with routing key {routing_key}"
.format(body=message,
amqpuri=self._amqp_uri,
routing_key=message.header.topic))
sender.publish(
message.body.value,
headers=_build_message_header(message),
exchange=self._exchange,
serializer=
'json', # todo: fix this for the mime type of the message
routing_key=message.header.topic,
declare=[self._exchange])
def _error_callback(e, interval) -> None:
logger.debug(
'Publishing error: {e}. Will retry in {interval} seconds', e,
interval)
self._logger.debug(
"Connect to broker {amqpuri}".format(amqpuri=self._amqp_uri))
with connections[self._cnx].acquire(block=True) as conn:
with Producer(conn) as producer:
ensure_kwargs = self.RETRY_OPTIONS.copy()
ensure_kwargs['errback'] = _error_callback
safe_publish = conn.ensure(producer, _publish, **ensure_kwargs)
safe_publish(producer)
def handle_collect_by_platform_id(self, body, message):
print('Recived state from platform_id: ', json.loads(body)['header']['PlatformId'])
# print(msg.payload.decode('utf-8'))
# print(ast.literal_eval(msg.payload.decode('utf-8')))
states = json.loads(body)['body']['states']
# print(list_things)
#list_things['message_monitor'] = self.message_monitor.monitor(list_things, 'collector', 'handle_collect_by_platform_id')
points = []
for state in states:
points.append({
"MetricId": state['MetricId'],
"DataType": state['DataPoint']['DataType'],
"Value": state['DataPoint']['Value'],
"TimeCollect": datetime.now().strftime('%Y-%m-%d %H:%M:%S')
})
message = {
'header': {},
'body': {
'data_points': points
}
}
request_queue = Queue(name='dbwriter.request.api_write_db', exchange=self.exchange,
routing_key='dbwriter.request.api_write_db', message_ttl=20)
request_routing_key = 'dbwriter.request.api_write_db'
self.producer_connection.ensure_connection()
with Producer(self.producer_connection) as producer:
producer.publish(
json.dumps(message),
exchange=self.exchange.name,
routing_key=request_routing_key,
declare=[request_queue],
retry=True
)
print('Send new state to Dbwriter: {}'.format(message))
def send(self):
self.response = None
self.correlation_id = uuid()
callback_queue = Queue(
uuid(),
durable=False,
exclusive=True,
auto_delete=True,
max_priority=4,
consumer_arguments={"x-priority": 4},
)
with Connection(RABBITMQ_URI) as connection:
with Producer(connection) as producer:
producer.publish(
{},
exchange="",
routing_key="configuration.rpc.load-as-sets",
retry=True,
declare=[callback_queue],
reply_to=callback_queue.name,
correlation_id=self.correlation_id,
priority=4,
serializer="ujson",
)
with Consumer(
connection,
on_message=self.on_response,
queues=[callback_queue],
accept=["ujson"],
):
while self.response is None:
connection.drain_events()
if self.response["success"]:
return self.response["payload"]["message"], True
return self.response["error"], False
def send():
send_cnt = 0
msg_ = {
"orig_path": [],
"communities": [],
"service": "a",
"type": "A",
"path": [8, 4, 3, 2, 1],
"peer_asn": 8,
}
print("[+] Sending {}".format(LIMIT_UPDATES))
with Connection(RABBITMQ_URI) as connection:
exchange = Exchange("bgp-update",
channel=connection,
type="direct",
durable=False)
exchange.declare()
with Producer(connection) as producer:
for x in range(0, 256):
if send_cnt // LIMIT_UPDATES > 0:
break
for y in range(0, 256):
if send_cnt // LIMIT_UPDATES > 0:
break
msg_["timestamp"] = x * 1000 + y
msg_["key"] = "{}-{}".format(x, y)
msg_["prefix"] = "10.{}.{}.0/24".format(x, y)
producer.publish(
msg_,
exchange=exchange,
routing_key="update",
serializer="ujson",
)
send_cnt += 1
print("[+] Exit send")
def create_legacy_queue_connection():
logger.debug('Start create_legacy_queue_connection')
OUTGOING_QUEUE = QUEUE_DICT['OUTGOING_QUEUE'] # type: ignore
OUTGOING_QUEUE_HOSTNAME = QUEUE_DICT['OUTGOING_QUEUE_HOSTNAME'] # type: ignore
outgoing_exchange = Exchange("legacy_transmission", type='direct')
logger.info('Creating queue using queue: {}, hostname: {}, exchange: {}'.
format(OUTGOING_QUEUE, OUTGOING_QUEUE_HOSTNAME, outgoing_exchange))
queue = Queue(OUTGOING_QUEUE, outgoing_exchange, routing_key="legacy_transmission")
connection = BrokerConnection(hostname=OUTGOING_QUEUE_HOSTNAME,
userid=QUEUE_DICT['OUTGOING_QUEUE_USERID'], # type: ignore
password=QUEUE_DICT['OUTGOING_QUEUE_PASSWORD'], # type: ignore
virtual_host="/")
# Queue must be declared, otherwise messages are silently sent to a 'black hole'!
logger.info('Declaring queue')
bound_queue = queue(connection)
bound_queue.declare()
logger.info('Queue declared')
producer = Producer(connection, exchange=outgoing_exchange, routing_key="legacy_transmission")
logger.debug('End create_legacy_queue_connection. Returning producer.')
return producer
def test4():
#exchange = Exchange('test', type='fanout')
#queue = kombu.Queue('test', exchange, routing_key='test')
exchange = Exchange('download_task', 'direct')
queue = kombu.Queue('download_task', exchange, routing_key='download_task')
high_exchange = Exchange('download_task', 'direct')
high_queue = kombu.Queue('download_task_high',
high_exchange,
routing_key='download_task_high')
with Connection('amqp://*****:*****@192.168.3.82:5672//') as connection:
producer = Producer(connection)
while True:
s = create_tasks('low', 0)
js = json.dumps(s)
producer.publish(js,
exchange=exchange,
routing_key='download_task',
declare=[queue],
serializer='json',
compression='zlib')
print "=====1===="
time.sleep(5)
def set_state(thing_global_id, item_global_id, new_state):
thing = get_thing_info_by_global_id(thing_global_id)['things'][0]
for item in thing['items']:
if item['item_global_id'] == item_global_id:
message_request = {
'thing_local_id': thing['thing_local_id'],
'thing_name': thing['thing_name'],
'thing_type': thing['thing_type'],
'location': thing['location'],
'item_local_id': item['item_local_id'],
'item_name': item['item_name'],
'item_type': item['item_type'],
'new_state': new_state,
'platform_id': thing['platform_id']
}
break
request_routing_key = 'driver.request.api_set_state'
rabbitmq_connection.ensure_connection()
with Producer(rabbitmq_connection) as producer:
producer.publish(json.dumps(message_request),
exchange=exchange.name,
routing_key=request_routing_key,
retry=True)
def exabgp_msg(bgp_message):
msg = {
"type": bgp_message["type"],
"communities": bgp_message.get("communities", []),
"timestamp": float(bgp_message["timestamp"]),
"path": bgp_message.get("path", []),
"service": "exabgp|{}".format(self.host),
"prefix": bgp_message["prefix"],
"peer_asn": int(bgp_message["peer_asn"]),
}
if validator.validate(msg):
with Producer(connection) as producer:
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=self.exchange,
routing_key="update",
serializer="json",
)
else:
log.warning("Invalid format message: {}".format(msg))
def exabgp_msg(bgp_message):
msg = {
'type': bgp_message['type'],
'communities': bgp_message.get('communities', []),
'timestamp': float(bgp_message['timestamp']),
'path': bgp_message.get('path', []),
'service': 'exabgp|{}'.format(self.host),
'prefix': bgp_message['prefix'],
'peer_asn': int(bgp_message['peer_asn'])
}
if mformat_validator(msg):
with Producer(connection) as producer:
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=self.exchange,
routing_key='update',
serializer='json'
)
else:
log.warning('Invalid format message: {}'.format(msg))
def response_greenthread():
with get_connection() as conn:
with conn.channel() as chan:
queue = nova.get_topic_queue('test_rpc', 'test', channel=chan)
queue.declare()
queue_declared.send(True)
msg = ifirst(queue_iterator(queue, no_ack=True, timeout=2))
msgid, _, _, args = nova.parse_message(msg.payload)
exchange = nova.get_reply_exchange(msgid)
producer = Producer(chan, exchange=exchange, routing_key=msgid)
for _ in range(3):
msg = dict(result='should ignore this message',
failure=None,
ending=False)
producer.publish(msg)
eventlet.sleep(0.1)
msg = dict(result=args, failure=None, ending=False)
producer.publish(msg)
msg = dict(result=None, failure=None, ending=True)
producer.publish(msg)
def collect_by_platform_id(self, platform_id):
print('Collect data from platform_id: ', str(platform_id))
message_request = {
'header':{
'reply_to': 'driver.response.collector.api_get_states',
'PlatformId': platform_id,
'mode': 'PULL'
}
}
#message_request['message_monitor'] = self.message_monitor.monitor({}, 'collector', 'collect_by_platform_id')
request_queue = Queue(name='driver.request.api_get_states', exchange=self.exchange,
routing_key='driver.request.api_get_states', message_ttl=20)
request_routing_key = 'driver.request.api_get_states'
self.producer_connection.ensure_connection()
with Producer(self.producer_connection) as producer:
producer.publish(
json.dumps(message_request),
exchange=self.exchange.name,
routing_key=request_routing_key,
declare=[request_queue],
retry=True
)
def start():
with open(sys.argv[1], 'r') as f:
objs = json.load(f)
with Connection(os.getenv('RABBITMQ_HOST', 'localhost')) as connection:
exchange = Exchange(
'bgp-update',
channel=connection,
type='direct',
durable=False,
delivery_mode=1)
exchange.declare()
with Producer(connection) as producer:
for obj in objs:
time.sleep(1)
if 'key' not in obj:
obj['key'] = '{}'.format(time.time())
if 'timestamp' not in obj:
obj['timestamp'] = time.time()
producer.publish(
obj,
exchange=exchange,
routing_key='update',
serializer='json')
def process(self, data, steps):
self.response = None
self.correlation_id = uuid()
logger.info('publish queue={} next={} reply_to={}'.format(steps[0], steps[1:] or None, self.callback_queue.name))
with Producer(self.conn) as producer:
producer.publish(
data,
exchange='',
routing_key=steps[0],
# headers={'next': json.loads(json.dumps(steps[1:]))},
# headers={'next': ('aa', 'bbb')},
# headers={'next': json.dumps(['aa', 'bbb'])},
headers={'next': '|'.join(steps[1:])},
declare=[self.callback_queue],
reply_to=self.callback_queue.name,
correlation_id=self.correlation_id,
)
with Consumer(self.conn,
on_message=self.on_response,
queues=[self.callback_queue], no_ack=True):
while self.response is None:
self.conn.drain_events()
logger.info("Response {}".format(self.response))
return self.response
def send(self, message: BrightsideMessage):
# we want to expose our logger to the functions defined in inner scope, so put it in their outer scope
logger = self._logger
def _build_message_header(msg: BrightsideMessage) -> Dict:
return KombuMessageFactory(msg).create_message_header()
def _publish(sender: Producer) -> None:
logger.debug(
"Send message {body} to broker {amqpuri} with routing key {routing_key}"
.format(body=message,
amqpuri=self._amqp_uri,
routing_key=message.header.topic))
sender.publish(message.body.bytes,
headers=_build_message_header(message),
exchange=self._exchange,
content_type="text/plain",
routing_key=message.header.topic,
declare=[self._exchange])
def _error_callback(e, interval) -> None:
logger.debug(
'Publishing error: {e}. Will retry in {interval} seconds', e,
interval)
self._logger.debug(
"Connect to broker {amqpuri}".format(amqpuri=self._amqp_uri))
# Producer uses a pool, because you may have many instances in your code, but no heartbeat as a result
with connections[self._cnx].acquire(block=True) as conn:
with Producer(conn) as producer:
ensure_kwargs = self.RETRY_OPTIONS.copy()
ensure_kwargs['errback'] = _error_callback
safe_publish = conn.ensure(producer, _publish, **ensure_kwargs)
safe_publish(producer)
def send(document, routing_key=None):
"""
Attempt to send a message to the AMQP broker.
If we cannot obtain a new connection then the message will be dropped. Note
that we do not block when waiting for a connection.
:param document: the taskstatus Document we want to send
:type document: mongoengine.Document
:param routing_key: The routing key for the message
:type routing_key: str
"""
# if the user has not enabled notifications, just bail
event_notifications_enabled = config.getboolean(
'messaging', 'event_notifications_enabled')
if not event_notifications_enabled:
return
try:
payload = document.to_json()
except TypeError:
_logger.warn(
"unable to convert document to JSON; event message not sent")
return
broker_url = config.get('messaging', 'event_notification_url')
notification_topic = Exchange(name=DEFAULT_EXCHANGE_NAME, type='topic')
with Connection(broker_url) as connection:
producer = Producer(connection)
producer.maybe_declare(notification_topic)
producer.publish(payload,
exchange=notification_topic,
routing_key=routing_key)
def test_publish__consume(self):
connection = Connection(transport=Transport)
channel = connection.channel()
producer = Producer(channel, self.exchange, routing_key='test_Redis')
consumer = Consumer(channel, queues=[self.queue])
producer.publish({'hello2': 'world2'})
_received = []
def callback(message_data, message):
_received.append(message_data)
message.ack()
consumer.register_callback(callback)
consumer.consume()
self.assertIn(channel, channel.connection.cycle._channels)
try:
connection.drain_events(timeout=1)
self.assertTrue(_received)
with self.assertRaises(socket.timeout):
connection.drain_events(timeout=0.01)
finally:
channel.close()
def get_thing_info_by_global_id(thing_global_id):
print("API get things by thing_global_id")
message_request = {
'reply_to': 'registry.response.api.api_get_thing_by_global_id',
'thing_global_id': thing_global_id
}
# request to api_get_things of Registry
queue_response = Queue(
name='registry.response.api.api_get_thing_by_global_id',
exchange=exchange,
routing_key='registry.response.api.api_get_thing_by_global_id')
request_routing_key = 'registry.request.api_get_thing_by_global_id'
rabbitmq_connection.ensure_connection()
with Producer(rabbitmq_connection) as producer:
producer.publish(json.dumps(message_request),
exchange=exchange.name,
routing_key=request_routing_key,
declare=[queue_response],
retry=True)
message_response = None
def on_response(body, message):
nonlocal message_response
message_response = json.loads(body)
with Consumer(rabbitmq_connection,
queues=queue_response,
callbacks=[on_response],
no_ack=True):
while message_response is None:
rabbitmq_connection.drain_events()
return message_response
def _manage_lock(self):
p = Producer(
Connection(),
self.exchange,
auto_declare=True,
)
while self.redis_connection.get('is_consuming_' + self.name):
if not self.redis_connection.get('current_lock_owner_' +
self.name):
# Get next candidate owner from queue
message = self.acquire_requests_q.get()
if not message:
continue
print(message.payload)
self.redis_connection.set('current_lock_owner_' + self.name,
message.payload.get('id'))
# Inform the candidate owner that lock has been granted
# message not deleted until ack'ed
message.ack()
p.publish(
dict(request='GRANTED', id=message.payload.get('id')),
routing_key=message.payload.get('id'),
exchange=self.exchange,
)
def publish_cmd():
parser = get_parser("publish")
(options, args) = parser.parse_args(sys.argv[2:])
if len(args) < 1:
parser.error("You must provide a queue hostname to publish to!")
exchange = Exchange(options.exchange,
type="fanout",
delivery_mode="transient")
queue = Queue(options.queue, exchange)
connection = BrokerConnection(hostname=args[0],
userid=options.userid,
password=options.password,
virtual_host=options.vhost)
channel = connection.channel()
producer = Producer(channel, exchange)
pods = pod.load_pod_dir(
dirname=options.directory,
timeout=options.timeout,
# The first arg is the queue hostname, the rest
# will be pod names
filter_fn=pod.get_pod_filter(options.pod))
pods.update(pod.get_barker_metadata())
producer.publish(pods, serializer="json")
return 0
def run(self):
# update redis
ping_redis(redis)
redis.set("bgpstreamkafka_seen_bgp_update",
"1",
ex=MON_TIMEOUT_LAST_BGP_UPDATE)
# create a new bgpstream instance and a reusable bgprecord instance
stream = _pybgpstream.BGPStream()
# set kafka data interface
stream.set_data_interface("kafka")
# set host connection details
stream.set_data_interface_option("kafka", "brokers",
"{}:{}".format(self.host, self.port))
# set topic
stream.set_data_interface_option("kafka", "topic", self.topic)
# filter prefixes
for prefix in self.prefixes:
stream.add_filter("prefix", prefix)
# build monitored prefix tree
prefix_tree = {
"v4": pytricia.PyTricia(32),
"v6": pytricia.PyTricia(128)
}
for prefix in self.prefixes:
ip_version = get_ip_version(prefix)
prefix_tree[ip_version].insert(prefix, "")
# filter record type
stream.add_filter("record-type", "updates")
# filter based on timing (if end=0 --> live mode)
# Bypass for https://github.com/FORTH-ICS-INSPIRE/artemis/issues/411#issuecomment-661325802
start_time = int(time.time()) - START_TIME_OFFSET
if "BGPSTREAM_TIMESTAMP_BYPASS" in os.environ:
log.warn(
"Using BGPSTREAM_TIMESTAMP_BYPASS, meaning BMP timestamps are thrown away from BGPStream"
)
start_time = 0
stream.add_interval_filter(start_time, 0)
# set live mode
stream.set_live_mode()
# start the stream
stream.start()
# start producing
validator = MformatValidator()
with Producer(self.connection) as producer:
while True:
if not self.shared_memory_manager_dict[
"data_worker_should_run"]:
break
# get next record
try:
rec = stream.get_next_record()
except BaseException:
continue
if (rec.status != "valid") or (rec.type != "update"):
continue
# get next element
try:
elem = rec.get_next_elem()
except BaseException:
continue
while elem:
if not self.shared_memory_manager_dict[
"data_worker_should_run"]:
break
if elem.type in {"A", "W"}:
redis.set(
"bgpstreamkafka_seen_bgp_update",
"1",
ex=MON_TIMEOUT_LAST_BGP_UPDATE,
)
this_prefix = str(elem.fields["prefix"])
service = "bgpstreamkafka|{}".format(str(
rec.collector))
type_ = elem.type
if type_ == "A":
as_path = elem.fields["as-path"].split(" ")
communities = [{
"asn": int(comm.split(":")[0]),
"value": int(comm.split(":")[1]),
} for comm in elem.fields["communities"]]
else:
as_path = []
communities = []
timestamp = float(rec.time)
if timestamp == 0:
timestamp = time.time()
log.debug("fixed timestamp: {}".format(timestamp))
peer_asn = elem.peer_asn
ip_version = get_ip_version(this_prefix)
if this_prefix in prefix_tree[ip_version]:
msg = {
"type": type_,
"timestamp": timestamp,
"path": as_path,
"service": service,
"communities": communities,
"prefix": this_prefix,
"peer_asn": peer_asn,
}
try:
if validator.validate(msg):
msgs = normalize_msg_path(msg)
for msg in msgs:
key_generator(msg)
log.debug(msg)
producer.publish(
msg,
exchange=self.update_exchange,
routing_key="update",
serializer="ujson",
)
else:
log.debug(
"Invalid format message: {}".format(
msg))
except BaseException:
log.exception(
"Error when normalizing BGP message: {}".
format(msg))
try:
elem = rec.get_next_elem()
except BaseException:
continue
