def scan_s3_bucket(bucket=None,
aws_access_key_id=None,
aws_secret_access_key=None,
rabbitmq_setting=None,
timeout=180,
dt=0.1):
# post jobs
resource = libs3.resource(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
files = libs3.grep(libs3.ls(bucket, resource=resource),
r'\.doc|\.pdf|\.xlsm|\.xls')
tmp_queue = 'avanan_' + str(random.random())
out_rmq = rabbitmq.RMQBlockingConnection()
out_rmq.rmq_connect(out_rmq.rmq_get_parameters(rabbitmq_setting),
prefetch_count=1,
is_consumer=False)
messages = dict()
for file in files:
message = {'tag': tmp_queue, 'bucket': bucket, 'key': file['key']}
messages[message['key']] = message
# print(('enqueue', 'bucket', message['bucket'], 'file', message['key']))
out_rmq.rmq_send_message(json.dumps(message),
queue_name=rabbitmq_setting['RABBITMQ_JOBS'])
out_rmq.rmq_disconnect()
# collect responses
in_rmq = rabbitmq.RMQBlockingConnection()
in_rmq.rmq_connect(in_rmq.rmq_get_parameters(rabbitmq_setting),
prefetch_count=1,
is_consumer=False)
in_rmq.queue_declare(queue_name=tmp_queue)
n_waiting = len(messages)
_timeout = 0
while n_waiting > 0 and _timeout < timeout:
message_method, message_header_frame, message_body = in_rmq.rmq_next_message(
queue_name=tmp_queue, auto_ack=True)
# print(('response', 'bucket', message_body ))
if message_body is not None:
message = json.loads(message_body)
messages[message['key']] = message
n_waiting = len(
[v for v in messages.values() if 'summary' not in v])
# print(('response', 'bucket', message['bucket'], 'file', message['key']))
else:
time.sleep(dt)
_timeout += dt
in_rmq.queue_delete(tmp_queue)
in_rmq.rmq_disconnect()
return messages
def test_blocking_connection_get_message(self):
# connect
rmq = rabbitmq.RMQBlockingConnection()
rmq.rmq_connect(rmq.rmq_get_parameters(self.settings),
prefetch_count=1,
is_consumer=False)
# count messages
n0 = rmq.rmq_get_message_count(queue_name=self.rmq_queue)
# send message
body = json.dumps({'a': 1, 'b': 2, "c": 3})
rmq.rmq_send_message(body, queue_name=self.rmq_queue)
# count messages again
n1 = rmq.rmq_get_message_count(queue_name=self.rmq_queue)
self.assertTrue(n1 - n0 == 1)
message_method, message_header_frame, message_body = rmq.rmq_next_message(
queue_name=self.rmq_queue, auto_ack=False)
self.assertTrue(message_body.decode('ascii') == str(body))
rmq.rmq_ack_message(message_method.delivery_tag)
# count messages again
n2 = rmq.rmq_get_message_count(queue_name=self.rmq_queue)
self.assertTrue(n1 - n2 == 1)
self.assertTrue(n2 == n0)
rmq.rmq_disconnect()
def test_blocking_connection(self):
rmq = rabbitmq.RMQBlockingConnection()
rmq.rmq_connect(rmq.rmq_get_parameters(self.settings),
prefetch_count=1,
is_consumer=False)
rmq.rmq_disconnect()
self.assertTrue((rmq is not None))
def test_blocking_connection_publish(self):
rmq = rabbitmq.RMQBlockingConnection()
rmq.rmq_connect(rmq.rmq_get_parameters(self.settings),
prefetch_count=1,
is_consumer=False)
n0 = rmq.rmq_get_message_count(queue_name=self.rmq_queue)
rmq.rmq_send_message(json.dumps({
'a': 1,
'b': 2,
"c": 3
}),
queue_name=self.rmq_queue)
time.sleep(2)
n1 = rmq.rmq_get_message_count(queue_name=self.rmq_queue)
self.assertTrue(n1 - n0 == 1)
rmq.rmq_disconnect()
RABBITMQ_HOST=os.getenv("RABBITMQ_HOST"),
RABBITMQ_PORT=os.getenv("RABBITMQ_PORT"),
RABBITMQ_USER=os.getenv("RABBITMQ_USER"),
RABBITMQ_PASS=os.getenv("RABBITMQ_PASS"),
RABBITMQ_VIRTUAL_HOST=os.getenv("RABBITMQ_VIRTUAL_HOST"),
)
TMP_DIR = os.getenv("TMP_DIR")
AWS_ACCESS_KEY_ID = os.getenv("AWS_ACCESS_KEY_ID")
AWS_SECRET_ACCESS_KEY = os.getenv("AWS_SECRET_ACCESS_KEY")
random.seed()
while True:
# get job
in_rmq = rabbitmq.RMQBlockingConnection()
in_rmq.rmq_connect(in_rmq.rmq_get_parameters(RABBITMQ_SETTINGS),
prefetch_count=1,
is_consumer=False)
message_method, message_header_frame, message_body = in_rmq.rmq_next_message(
queue_name=RABBITMQ_JOBS, auto_ack=True)
in_rmq.rmq_disconnect()
if message_body is None:
time.sleep(0.5)
continue
'''
process message
it is assumed that
message={
'tag':'outgoing queue name',
def test_blocking_connection_constructor(self):
rmq = rabbitmq.RMQBlockingConnection()
self.assertTrue((rmq is not None))