def extract_audio():
if request.method == 'POST':
file = request.files['file']
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and (file.filename.endswith('.mp4') or file.filename.endswith('.avi')):
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='rabbitmq'))
channel = connection.channel()
channel.queue_declare(queue='audio_extractor', durable=True)
print("XAXA: ", os.getenv('FILES_URL', 'XORUMELOS'))
file_url = upload(file.read(), buffer=True, mime=file.mimetype)['name']
# DB
db_conn = Connection()
file_oid = db_conn.insert_doc_mongo(file.read())
oid, project_id = db_conn.insert_jobs(
type='audio_extractor', status='new', file=file_oid, file_name=file.filename)
message = {'type': 'audio_extractor', 'status': 'new',
'oid': file_oid, 'project_id': project_id, 'file': file_url}
channel.basic_publish(
exchange='', routing_key='audio_extractor', body=json.dumps(message))
connection.close()
return {'project_id': project_id}
else:
flash('Invalid file')
return redirect(request.url)
def asr():
if request.method == 'POST':
file = request.files['file']
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and (file.filename.endswith('.mp3') or file.filename.endswith('.wav')):
connection = pika.BlockingConnection(
pika.ConnectionParameters(host='rabbitmq'))
channel = connection.channel()
channel.queue_declare(queue='asr')
file_url = upload(file.read(), buffer=True, mime=file.mimetype)['name']
# DB
db_conn = Connection()
oid = db_conn.insert_jobs(
type='asr', status='new', file=file.read())
message = {'type': 'asr', 'status': 'new',
'oid': oid, 'file': file_url}
channel.basic_publish(
exchange='', routing_key='asr', body=str(message))
connection.close()
return 'Done'
else:
flash('Invalid file')
return redirect(request.url)
def do_work(connection, channel, delivery_tag, body):
try:
print(" [x] Received %r" % body, flush=True)
args = json.loads(body)
oid = args['oid']
project_id = args['project_id']
# conn = Connection()
# file = conn.get_file(oid)
# file = conn.get_doc_mongo(file_oid=oid)
file = download(args['file'], buffer=True)
result = ast.literal_eval(file.decode('utf-8'))
count = 0
dict_result = {}
previous_duration = 0
for key, value in result.items():
result = main(value['bytes'])
dict_result[count] = result
count += 1
#time.sleep(1)
payload = bytes(str(dict_result), encoding='utf-8')
uploaded = upload(payload, buffer=True, mime='text/plain')
conn = Connection()
# inserts the result of processing in database
file_oid = conn.insert_doc_mongo(payload)
conn.insert_jobs(type='asr',
status='done',
file=file_oid,
project_id=project_id)
message = {
'type': 'aggregator',
'status': 'new',
'oid': file_oid,
'project_id': project_id,
'file': uploaded['name'],
'queue': 'asr'
}
# post a message on topic_segmentation queue
connection_out = pika.BlockingConnection(
pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel2 = connection_out.channel()
channel2.queue_declare(queue='aggregator', durable=True)
channel2.basic_publish(exchange='',
routing_key='aggregator',
body=json.dumps(message))
except Exception as e:
print('Connection Error %s' % e, flush=True)
print(" [x] Done", flush=True)
cb = functools.partial(ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
def do_work(connection, channel, delivery_tag, body):
try:
print(" [x] Received %r" % body, flush=True)
args = json.loads(body)
oid = args['oid']
project_id = args['project_id']
print('data: ', args)
print(str(oid) + '!!!???', flush=True)
print(str(project_id) + '!!!???', flush=True)
flows = new_aggregate_flow(args, args['queue'])
if flows:
data = {}
for ch in flows.values():
data[ch['queue']] = download(ch['file'], buffer=True)
print(data.keys(), flush=True)
# calls the audio extract algorithm
# print(data, flush=True)
conn = Connection()
try:
payload = bytes(str(data), encoding='utf-8')
file_oid = conn.insert_doc_mongo(payload)
conn.insert_jobs('aggregator', 'done', file_oid, project_id)
uploaded = upload(payload, buffer=True, mime='text/plain')
message = {'type': 'segmentation', 'status': 'new',
'oid': file_oid, 'project_id': project_id, 'file': uploaded['name']}
connection_out = pika.BlockingConnection(
pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel2 = connection_out.channel()
channel2.queue_declare(queue='segmentation', durable=True)
channel2.basic_publish(
exchange='', routing_key='segmentation', body=json.dumps(message))
except Exception as e:
print(e, flush=True)
except Exception as e:
print(e, flush=True)
print(" [x] Done", flush=True)
cb = functools.partial(ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
def aggregate_flow(project_id):
conn = Connection()
if conn.check_job_done(project_id): # already aggregate this flow
return []
conn = Connection()
res = conn.async_jobs(project_id)
while not res:
print('Not yet', flush=True)
time.sleep(30)
conn = Connection()
res = conn.async_jobs(project_id)
print(res, flush=True)
return res
def vad():
if request.method == 'POST':
file = request.files['file']
if file.filename == '':
flash('No selected file')
return redirect(request.url)
if file and (file.filename.endswith('.mp3') or file.filename.endswith('.wav')):
connection = pika.BlockingConnection(pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel = connection.channel()
channel.queue_declare(queue='vad', durable=True)
db_conn = Connection()
oid = db_conn.insert_jobs(type='vad', status='new', file=file.read())
message = {'type': 'vad', 'status': 'new', 'oid': oid}
channel.basic_publish(exchange='', routing_key='vad', body=json.dumps(message))
connection.close()
return 'Done'
else:
return 'Invalid file'
def callback(ch, method, properties, body):
try:
print(" [x] Received %r" % body, flush=True)
args = json.loads(body)
oid = args['oid']
project_id = args['project_id']
print(str(oid) + '!!!???', flush=True)
print(str(project_id) + '!!!???', flush=True)
# conn = Connection()
# file = conn.get_doc_mongo(file_oid=oid)
file = download(args['file'], buffer=True)
data = extract(file) # calls the audio extract algorithm
# print(data, flush=True)
conn = Connection()
try:
uploaded = upload(data, buffer=True, mime='audio/wav')
file_oid = conn.insert_doc_mongo(data)
conn.insert_jobs('audio_extractor', 'done', file_oid, project_id)
message = {'type': 'vad', 'status': 'new',
'oid': file_oid, 'project_id': project_id, 'file': uploaded['name']}
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel = connection.channel()
channel.queue_declare(queue='vad', durable=True)
channel.basic_publish(
exchange='', routing_key='vad', body=json.dumps(message))
except Exception as e:
print(e, flush=True)
except Exception as e:
print(e, flush=True)
print(" [x] Done", flush=True)
ch.basic_ack(delivery_tag=method.delivery_tag)
def do_work(connection, channel, delivery_tag, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
print(str(oid) + '!!!???', flush=True)
print(str(project_id) + '!!!???', flush=True)
flows = aggregate_flow(project_id)
if flows:
data = {}
for flow in flows:
conn = Connection()
file = conn.get_doc_mongo(file_oid=flow[0])
data[flow[1]] = file
print(data.keys(), flush=True)
# calls the audio extract algorithm
# print(data, flush=True)
conn = Connection()
try:
file_oid = conn.insert_doc_mongo(bytes(str(data), encoding='utf-8'))
conn.insert_jobs('aggregator', 'done', file_oid, project_id)
message = {'type': 'segmentation', 'status': 'new', 'oid': file_oid, 'project_id': project_id}
connection_out = pika.BlockingConnection(pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel2 = connection_out.channel()
channel2.queue_declare(queue='segmentation', durable=True)
channel2.basic_publish(exchange='', routing_key='segmentation', body=json.dumps(message))
except Exception as e:
print(e, flush=True)
except Exception as e:
print(e, flush=True)
print(" [x] Done", flush=True)
cb = functools.partial(ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
def do_work(connection, channel, delivery_tag, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
conn = Connection()
# file = conn.get_file(oid)
file = conn.get_doc_mongo(file_oid=oid)
result = ast.literal_eval(file.decode('utf-8'))
#print(result.keys(), flush=True)
chunks = []
low_features_dict = ast.literal_eval(
result['low_level_features'].decode('utf-8'))
asr_dict = ast.literal_eval(result['asr'].decode('utf-8'))
print(low_features_dict, flush=True)
print(asr_dict, flush=True)
for k, v in low_features_dict.items():
s = Shot(k,
low_features_dict[k]['pitch'],
low_features_dict[k]['volume'],
low_features_dict[k]['pause'], [],
init_time=low_features_dict[k]['init_time'],
end_time=0)
s.extractTranscriptAndConcepts(asr_dict[k], False, docSim=docSim)
chunks.append(s)
# print(result['low_level_features'], flush=True)
# print(result['asr'], flush=True)
chunks = [s for s in chunks if s.valid_vector]
if len(chunks) < 2:
boundaries = [0]
else:
'''calls the genetic algorithm'''
ga = GA.GeneticAlgorithm(population_size=100,
constructiveHeuristic_percent=0.3,
mutation_rate=0.05,
cross_over_rate=0.4,
docSim=docSim,
shots=chunks,
n_chunks=len(chunks),
generations=500,
local_search_percent=0.3,
video_length=100,
stopwords=stopwords,
ocr_on=False)
boundaries = ga.run()
#print(chunks, flush=True)
print(boundaries, flush=True)
topics = {}
topics["topics"] = boundaries
payload = bytes(str(topics), encoding='utf-8')
file_oid = conn.insert_doc_mongo(payload)
conn = Connection()
conn.insert_jobs(type='segmentation',
status='done',
file=file_oid,
project_id=project_id)
#
# #print(result, flush=True)
# count = 0
# dict_result = {}
# previous_duration = 0
# for key, value in result.items():
# result = main(value['bytes'])
# dict_result[count] = result
# count += 1
# #time.sleep(1)
#
# payload = bytes(str(dict_result), encoding='utf-8')
# conn = Connection()
#
# # inserts the result of processing in database
# file_oid = conn.insert_doc_mongo(payload)
# conn.insert_jobs(type='asr', status='done', file=file_oid, project_id=project_id)
#
# message = {'type': 'aggregator', 'status': 'new', 'oid': file_oid, 'project_id': project_id}
#
# # post a message on topic_segmentation queue
# connection_out = pika.BlockingConnection(pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
# channel2 = connection_out.channel()
#
# channel2.queue_declare(queue='aggregator', durable=True)
# channel2.basic_publish(exchange='', routing_key='aggregator', body=json.dumps(message))
except Exception as e:
# print(e, flush=True)
print('Connection Error %s' % e, flush=True)
print(" [x] Done", flush=True)
cb = functools.partial(ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)
def callback(ch, method, properties, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
conn = Connection()
file = conn.get_doc_mongo(file_oid=oid)
try:
data = main(file) # calls the VAD algorithm
# print(data(, flush=True)
except Exception as e:
print('aaaaaaaaaaa', flush=True)
logging.debug('Connection Error %s' % e)
conn = Connection()
try:
file_oid = conn.insert_doc_mongo(data)
conn.insert_jobs('vad', 'done', file_oid, project_id)
## Posts low level features jobs
message = {
'type': 'low_level_features',
'status': 'new',
'oid': file_oid,
'project_id': project_id
}
connection = pika.BlockingConnection(
pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel = connection.channel()
channel.queue_declare(queue='low_level_features', durable=True)
channel.basic_publish(exchange='',
routing_key='low_level_features',
body=json.dumps(message))
## Posts asr jobs
channel_asr = connection.channel()
message_asr = {
'type': 'asr',
'status': 'new',
'oid': file_oid,
'project_id': project_id
}
channel_asr.queue_declare(queue='asr', durable=True)
channel_asr.basic_publish(exchange='',
routing_key='asr',
body=json.dumps(message_asr))
except Exception as e:
print(e, flush=True)
LOGGER.info('Error Inserting % ' % e)
except Exception as e:
print(e, flush=True)
logging.info('error')
print(" [x] Done", flush=True)
ch.basic_ack(delivery_tag=method.delivery_tag)
itemSql = ItemSql()
for index, row in df.iterrows():
item = Item(row['ID_SERIE'], row['TIPO_PROVA'], row['DISCIPLINA'], row['ID_SERIE_ITEM'], row['ID_BLOCO'], row['ID_POSICAO'], row['ID_ITEM'], row['DESCRITOR_HABILIDADE'], row['GABARITO'])
itemSql.insert_item(connection.conexao(), item)
print(index, "de", len(df))
def aluno(df):
newDf = []
for index, row in df.iterrows():
aluno = (int(row["ID_PROVA_BRASIL"]), int(row["ID_REGIAO"]), int(row["ID_UF"]), long(row["ID_MUNICIPIO"]), int(row["ID_AREA"]), long(row["ID_ESCOLA"]), int(row["ID_DEPENDENCIA_ADM"]), int(row["ID_LOCALIZACAO"]), int(row["ID_TURMA"]), int(row["ID_TURNO"]), int(row["ID_SERIE"]), long(row["ID_ALUNO"]), int(row["IN_SITUACAO_CENSO"]), int(row["IN_PREENCHIMENTO_PROVA"]), int(row["IN_PRESENCA_PROVA"]), int(row["ID_CADERNO"]), int(row["ID_BLOCO_1"]), int(row["ID_BLOCO_2"]), row["TX_RESP_BLOCO_1_LP"], row["TX_RESP_BLOCO_2_LP"], row["TX_RESP_BLOCO_1_MT"], row["TX_RESP_BLOCO_2_MT"], int(row["IN_PROFICIENCIA"]), int(row["IN_PROVA_BRASIL"]), str(row["ESTRATO_ANEB"]), float(row["PESO_ALUNO_LP"]), float(row["PESO_ALUNO_MT"]), float(row["PROFICIENCIA_LP"]), float(row["ERRO_PADRAO_LP"]), float(row["PROFICIENCIA_LP_SAEB"]), float(row["ERRO_PADRAO_LP_SAEB"]), float(row["PROFICIENCIA_MT"]), float(row["ERRO_PADRAO_MT"]), float(row["PROFICIENCIA_MT_SAEB"]), float(row["ERRO_PADRAO_MT_SAEB"]), bool(row["IN_PREENCHIMENTO_QUESTIONARIO"]))
newDf.append(aluno)
for i in newDf:
print(i)
alunoSql = AlunoSql()
alunoSql.insertAluno(connection.conexao(), newDf)
if __name__ == '__main__':
connection = Connection()
df = leitura_aluno()
newDf = df.replace(to_replace="", value="0")
aluno(newDf)
connection.con.close()
def do_work(connection, channel, delivery_tag, body):
try:
print(" [x] Received %r" % body, flush=True)
oid = json.loads(body)['oid']
project_id = json.loads(body)['project_id']
conn = Connection()
# file = conn.get_file(oid)
file = conn.get_doc_mongo(file_oid=oid)
result = ast.literal_eval(file.decode('utf-8'))
timestamps = [0]
duration = []
pause_duration = []
count = 0
dict_result = {}
previous_duration = 0
for key, value in result.items():
dict_result[count] = {}
if count == 0:
dict_result[count]['pause'] = float(value['timestamp'])
else:
dict_result[count]['pause'] = float(
value['timestamp']) - previous_duration
dict_result[count]['init_time'] = float(value['timestamp'])
previous_duration = float(value['timestamp']) + float(
value['duration'])
dict_result[count]['pitch'], dict_result[count][
'volume'] = extract(value['bytes'])
count += 1
payload = bytes(str(dict_result), encoding='utf-8')
conn = Connection()
# inserts the result of processing in database
file_oid = conn.insert_doc_mongo(payload)
conn.insert_jobs(type='low_level_features',
status='done',
file=file_oid,
project_id=project_id)
message = {
'type': 'aggregator',
'status': 'new',
'oid': file_oid,
'project_id': project_id
}
# post a message on topic_segmentation queue
connection_out = pika.BlockingConnection(
pika.ConnectionParameters(host=os.environ['QUEUE_SERVER']))
channel2 = connection_out.channel()
channel2.queue_declare(queue='aggregator', durable=True)
channel2.basic_publish(exchange='',
routing_key='aggregator',
body=json.dumps(message))
except Exception as e:
# print(e, flush=True)
print('Connection Error %s' % e, flush=True)
print(" [x] Done", flush=True)
cb = functools.partial(ack_message, channel, delivery_tag)
connection.add_callback_threadsafe(cb)