def do_work(connection, channel, delivery_tag, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
conn = Connection()
# file = conn.get_file(oid)
file = conn.get_doc_mongo(file_oid=oid)
result = ast.literal_eval(file.decode('utf-8'))
#print(result, flush=True)
count = 0
dict_result = {}
previous_duration = 0
for key, value in result.items():
result = main(value['bytes'])
dict_result[count] = result
count += 1
#time.sleep(1)
payload = bytes(str(dict_result), encoding='utf-8')
conn = Connection()
# inserts the result of processing in database
file_oid = conn.insert_doc_mongo(payload)
conn.insert_jobs(type='asr',
status='done',
file=file_oid,
project_id=project_id)
message = {
'type': 'aggregator',
'status': 'new',
'oid': file_oid,
'project_id': project_id
}
# post a message on topic_segmentation queue
connection_out = pika.BlockingConnection(
pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel2 = connection_out.channel()
channel2.queue_declare(queue='aggregator', durable=True)
channel2.basic_publish(exchange='',
routing_key='aggregator',
body=json.dumps(message))
except Exception as e:
# print(e, flush=True)
print('Connection Error %s' % e, flush=True)
print(" [x] Done", flush=True)
cb = functools.partial(ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
def do_work(connection, channel, delivery_tag, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
print(str(oid) + '!!!???', flush=True)
print(str(project_id) + '!!!???', flush=True)
flows = aggregate_flow(project_id)
if flows:
data = {}
for flow in flows:
conn = Connection()
file = conn.get_doc_mongo(file_oid=flow[0])
data[flow[1]] = file
print(data.keys(), flush=True)
# calls the audio extract algorithm
# print(data, flush=True)
conn = Connection()
try:
file_oid = conn.insert_doc_mongo(bytes(str(data), encoding='utf-8'))
conn.insert_jobs('aggregator', 'done', file_oid, project_id)
message = {'type': 'segmentation', 'status': 'new', 'oid': file_oid, 'project_id': project_id}
connection_out = pika.BlockingConnection(pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel2 = connection_out.channel()
channel2.queue_declare(queue='segmentation', durable=True)
channel2.basic_publish(exchange='', routing_key='segmentation', body=json.dumps(message))
except Exception as e:
print(e, flush=True)
except Exception as e:
print(e, flush=True)
print(" [x] Done", flush=True)
cb = functools.partial(ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
def callback(ch, method, properties, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
print(str(oid) + '!!!???', flush=True)
print(str(project_id) + '!!!???', flush=True)
conn = Connection()
file = conn.get_doc_mongo(file_oid=oid)
data = extract(file) # calls the audio extract algorithm
# print(data, flush=True)
conn = Connection()
try:
file_oid = conn.insert_doc_mongo(data)
conn.insert_jobs('audio_extractor', 'done', file_oid, project_id)
message = {
'type': 'vad',
'status': 'new',
'oid': file_oid,
'project_id': project_id
}
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel = connection.channel()
channel.queue_declare(queue='vad', durable=True)
channel.basic_publish(exchange='',
routing_key='vad',
body=json.dumps(message))
except Exception as e:
print(e, flush=True)
except Exception as e:
print(e, flush=True)
print(" [x] Done", flush=True)
ch.basic_ack(delivery_tag=method.delivery_tag)
def do_work(connection, channel, delivery_tag, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
conn = Connection()
# file = conn.get_file(oid)
file = conn.get_doc_mongo(file_oid=oid)
result = ast.literal_eval(file.decode('utf-8'))
#print(result.keys(), flush=True)
chunks = []
low_features_dict = ast.literal_eval(
result['low_level_features'].decode('utf-8'))
asr_dict = ast.literal_eval(result['asr'].decode('utf-8'))
print(low_features_dict, flush=True)
print(asr_dict, flush=True)
for k, v in low_features_dict.items():
s = Shot(k,
low_features_dict[k]['pitch'],
low_features_dict[k]['volume'],
low_features_dict[k]['pause'], [],
init_time=low_features_dict[k]['init_time'],
end_time=0)
s.extractTranscriptAndConcepts(asr_dict[k], False, docSim=docSim)
chunks.append(s)
# print(result['low_level_features'], flush=True)
# print(result['asr'], flush=True)
chunks = [s for s in chunks if s.valid_vector]
if len(chunks) < 2:
boundaries = [0]
else:
'''calls the genetic algorithm'''
ga = GA.GeneticAlgorithm(population_size=100,
constructiveHeuristic_percent=0.3,
mutation_rate=0.05,
cross_over_rate=0.4,
docSim=docSim,
shots=chunks,
n_chunks=len(chunks),
generations=500,
local_search_percent=0.3,
video_length=100,
stopwords=stopwords,
ocr_on=False)
boundaries = ga.run()
#print(chunks, flush=True)
print(boundaries, flush=True)
topics = {}
topics["topics"] = boundaries
payload = bytes(str(topics), encoding='utf-8')
file_oid = conn.insert_doc_mongo(payload)
conn = Connection()
conn.insert_jobs(type='segmentation',
status='done',
file=file_oid,
project_id=project_id)
#
# #print(result, flush=True)
# count = 0
# dict_result = {}
# previous_duration = 0
# for key, value in result.items():
# result = main(value['bytes'])
# dict_result[count] = result
# count += 1
# #time.sleep(1)
#
# payload = bytes(str(dict_result), encoding='utf-8')
# conn = Connection()
#
# # inserts the result of processing in database
# file_oid = conn.insert_doc_mongo(payload)
# conn.insert_jobs(type='asr', status='done', file=file_oid, project_id=project_id)
#
# message = {'type': 'aggregator', 'status': 'new', 'oid': file_oid, 'project_id': project_id}
#
# # post a message on topic_segmentation queue
# connection_out = pika.BlockingConnection(pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
# channel2 = connection_out.channel()
#
# channel2.queue_declare(queue='aggregator', durable=True)
# channel2.basic_publish(exchange='', routing_key='aggregator', body=json.dumps(message))
except Exception as e:
# print(e, flush=True)
print('Connection Error %s' % e, flush=True)
print(" [x] Done", flush=True)
cb = functools.partial(ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
def callback(ch, method, properties, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
conn = Connection()
file = conn.get_doc_mongo(file_oid=oid)
try:
data = main(file) # calls the VAD algorithm
# print(data(, flush=True)
except Exception as e:
print('aaaaaaaaaaa', flush=True)
logging.debug('Connection Error %s' % e)
conn = Connection()
try:
file_oid = conn.insert_doc_mongo(data)
conn.insert_jobs('vad', 'done', file_oid, project_id)
## Posts low level features jobs
message = {
'type': 'low_level_features',
'status': 'new',
'oid': file_oid,
'project_id': project_id
}
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel = connection.channel()
channel.queue_declare(queue='low_level_features', durable=True)
channel.basic_publish(exchange='',
routing_key='low_level_features',
body=json.dumps(message))
## Posts asr jobs
channel_asr = connection.channel()
message_asr = {
'type': 'asr',
'status': 'new',
'oid': file_oid,
'project_id': project_id
}
channel_asr.queue_declare(queue='asr', durable=True)
channel_asr.basic_publish(exchange='',
routing_key='asr',
body=json.dumps(message_asr))
except Exception as e:
print(e, flush=True)
LOGGER.info('Error Inserting % ' % e)
except Exception as e:
print(e, flush=True)
logging.info('error')
print(" [x] Done", flush=True)
ch.basic_ack(delivery_tag=method.delivery_tag)