def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
job_data = {
'cruiseID': self.cruise_id,
'loweringID': self.lowering_id,
'loweringStartDate': self.lowering_start_date
}
if current_job.task == 'setupNewLowering':
gm_client = python3_gearman.GearmanClient(
[self.ovdm.get_gearman_server()])
for task in self.ovdm.get_tasks_for_hook('setupNewLowering'):
logging.info("Adding post task: %s", task)
gm_client.submit_job(task,
json.dumps(job_data),
background=True)
elif current_job.task == 'finalizeCurrentLowering':
gm_client = python3_gearman.GearmanClient(
[self.ovdm.get_gearman_server()])
for task in self.ovdm.get_tasks_for_hook(
'finalizeCurrentLowering'):
logging.info("Adding post task: %s", task)
gm_client.submit_job(task,
json.dumps(job_data),
background=True)
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
if int(self.task['taskID']) > 0:
self.ovdm.set_error_task(
self.task['taskID'],
results_obj['parts'][-1]['reason'])
else:
self.ovdm.send_msg(self.task['longName'] + ' failed',
results_obj['parts'][-1]['reason'])
else:
self.ovdm.set_idle_task(self.task['taskID'])
else:
self.ovdm.set_idle_task(self.task['taskID'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s (%s) completed at: %s", self.task['longName'],
current_job.handle, time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def task_setup_new_lowering(gearman_worker, gearman_job):
"""
Setup a new lowering
"""
job_results = {'parts':[]}
payload_obj = json.loads(gearman_job.data)
logging.debug("Payload: %s", json.dumps(payload_obj, indent=2))
lowering_config_filepath = os.path.join(gearman_worker.lowering_dir, DEFAULT_LOWERING_CONFIG_FN)
gearman_worker.send_job_status(gearman_job, 1, 10)
gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()])
logging.info("Creating lowering data directory")
completed_job_request = gm_client.submit_job("createLoweringDirectory", gearman_job.data)
result_obj = json.loads(completed_job_request.result)
if result_obj['parts'][-1]['result'] == "Pass": # Final Verdict
job_results['parts'].append({"partName": "Create lowering data directory structure", "result": "Pass"})
else:
logging.error("Failed to create lowering data directory")
job_results['parts'].append({"partName": "Create lowering data directory structure", "result": "Fail", "reason": result_obj['parts'][-1]['reason']})
return json.dumps(job_results)
gearman_worker.send_job_status(gearman_job, 5, 10)
#build lowering Config file
logging.info("Exporting Lowering Configuration")
output_results = export_lowering_config(gearman_worker, lowering_config_filepath)
if output_results['verdict']:
job_results['parts'].append({"partName": "Export lowering config data to file", "result": "Pass"})
else:
job_results['parts'].append({"partName": "Export lowering config data to file", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], lowering_config_filepath)
if not output_results['verdict']:
job_results['parts'].append({"partName": "Set lowering config file ownership/permissions", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
gearman_worker.send_job_status(gearman_job, 9, 10)
logging.info("Updating Lowering Size")
lowering_size_proc = subprocess.run(['du','-sb', gearman_worker.lowering_dir], capture_output=True, text=True, check=False)
if lowering_size_proc.returncode == 0:
logging.info("Cruise Size: %s", lowering_size_proc.stdout.split()[0])
gearman_worker.ovdm.set_lowering_size(lowering_size_proc.stdout.split()[0])
else:
gearman_worker.ovdm.set_lowering_size("0")
gearman_worker.send_job_status(gearman_job, 10, 10)
return json.dumps(job_results)
def task_run_cruise_data_transfer(gearman_worker, current_job):
"""
Run the cruise data transfer
"""
time.sleep(randint(0, 2))
job_results = {
'parts': [{
"partName": "Transfer In-Progress",
"result": "Pass"
}, {
"partName": "Transfer Enabled",
"result": "Pass"
}],
'files': {}
}
logging.debug("Setting transfer status to 'Running'")
gearman_worker.ovdm.set_running_cruise_data_transfer(
gearman_worker.cruise_data_transfer['cruiseDataTransferID'],
os.getpid(), current_job.handle)
logging.info("Testing configuration")
gearman_worker.send_job_status(current_job, 1, 10)
gm_client = python3_gearman.GearmanClient(
[gearman_worker.ovdm.get_gearman_server()])
gm_data = {
'cruiseDataTransfer': gearman_worker.cruise_data_transfer,
'cruiseID': gearman_worker.cruise_id
}
completed_job_request = gm_client.submit_job("testCruiseDataTransfer",
json.dumps(gm_data))
results_obj = json.loads(completed_job_request.result)
logging.debug('Connection Test Results: %s',
json.dumps(results_obj, indent=2))
if results_obj['parts'][-1]['result'] == "Pass": # Final Verdict
logging.debug("Connection test passed")
job_results['parts'].append({
"partName": "Connection Test",
"result": "Pass"
})
else:
logging.warning("Connection test failed, quitting job")
job_results['parts'].append({
"partName":
"Connection Test",
"result":
"Fail",
"reason":
results_obj['parts'][-1]['reason']
})
return json.dumps(job_results)
gearman_worker.send_job_status(current_job, 2, 10)
logging.info("Transferring files")
output_results = None
if gearman_worker.cruise_data_transfer[
'transferType'] == "1": # Local Directory
output_results = transfer_local_dest_dir(gearman_worker, current_job)
elif gearman_worker.cruise_data_transfer[
'transferType'] == "2": # Rsync Server
output_results = transfer_rsync_dest_dir(gearman_worker, current_job)
elif gearman_worker.cruise_data_transfer[
'transferType'] == "3": # SMB Server
output_results = transfer_smb_dest_dir(gearman_worker, current_job)
elif gearman_worker.cruise_data_transfer[
'transferType'] == "4": # SSH Server
output_results = transfer_ssh_dest_dir(gearman_worker, current_job)
else:
logging.error("Unknown Transfer Type")
job_results['parts'].append({
"partName": "Transfer Files",
"result": "Fail",
"reason": "Unknown transfer type"
})
return json.dumps(job_results)
if not output_results['verdict']:
logging.error("Transfer of remote files failed: %s",
output_results['reason'])
job_results['parts'].append({
"partName": "Transfer Files",
"result": "Fail",
"reason": output_results['reason']
})
return json.dumps(job_results)
logging.debug("Transfer completed successfully")
job_results['files'] = output_results['files']
job_results['parts'].append({
"partName": "Transfer Files",
"result": "Pass"
})
if len(job_results['files']['new']) > 0:
logging.debug("%s file(s) added", len(job_results['files']['new']))
if len(job_results['files']['updated']) > 0:
logging.debug("%s file(s) updated",
len(job_results['files']['updated']))
if len(job_results['files']['exclude']) > 0:
logging.debug("%s file(s) intentionally skipped",
len(job_results['files']['exclude']))
gearman_worker.send_job_status(current_job, 9, 10)
time.sleep(2)
return json.dumps(job_results)
def post(self, request):
"""
Verifica se o usuário pagou.
Recebe uma imagem de um dígito numérico.
Salva essa imagem no disco.
Lança um pedido pelo gearman para um worker rodando em outro processo resgatar essa imagem no disco
e rodar uma CNN feita com Keras em cima dela."""
retorno = Retorno()
"""
Verificar se o usurário pagou para usar
"""
chave_endereco = "endereco"
"""
Vemos se não está faltando o campo ou se seu valor é inválido
"""
if chave_endereco not in request.data:
retorno.falhar("Informe o endereço do usuário sob a chave '%s'" %
chave_endereco)
return Response(retorno.get(), status=status.HTTP_400_BAD_REQUEST)
usuario = Blkn.validar_usuario(request.data[chave_endereco])
if not usuario:
retorno.falhar("Endereço informado não é válido")
return Response(retorno.get(), status=status.HTTP_400_BAD_REQUEST)
if Blkn.verificar_saldo(usuario) < self.preco:
retorno.falhar("Saldo insuficiente.")
return Response(retorno.get(),
status=status.HTTP_402_PAYMENT_REQUIRED)
Blkn.debitar_creditos(usuario, self.preco)
retorno.adicionar("Pagamento realizado")
"""
Salvar a imagem em disco
"""
nome_chave_imagem = "data"
if nome_chave_imagem not in request.FILES:
retorno.falhar(
"Faça upload do arquivo de imagem sob a chave '%s'" %
nome_chave_imagem)
return Response(retorno.get(), status=status.HTTP_400_BAD_REQUEST)
imagem = request.FILES[nome_chave_imagem]
caminho_imagem = path.join(settings.MEDIA_ROOT, imagem.name)
with open(caminho_imagem, "wb+") as myf:
# Iterar nos chunks do arquivo para não sobrecarregar memória
for chunk in imagem.chunks():
myf.write(chunk)
"""
Acionar o worker, passando o caminho para a imagem salva em disco
"""
gm_client = gearman.GearmanClient(['localhost:4730'])
# TODO: retornar falha caso não haja um worker disponível e estornar pagamento
try:
requisicao_execucao = gm_client.submit_job(self.atividade,
caminho_imagem)
except Exception as e:
retorno.falhar("Deu erro na requsição: %s" % e.args[0])
# TODO: estornar pagamento
else:
analise_execucao = check_request_status(requisicao_execucao)
# Se tiver warnings, entendemos que houve falha na execução
if not analise_execucao['warnings']:
retorno.suceder("Tarefa completada.")
else:
retorno.falhar(analise_execucao["warnings"])
# TODO: estornar pagamento
retorno.resultado = analise_execucao["resultado"]
return Response(retorno.get(), status=status.HTTP_200_OK)
def on_job_complete(self, current_job, job_result):
"""
Function run whenever the current job completes
"""
results_obj = json.loads(job_result)
logging.debug("Preparing subsequent Gearman jobs")
job_data = {
'cruiseID': self.cruise_id,
'loweringID': self.lowering_id,
'files': results_obj['files']
}
if current_job.task == 'updateDataDashboard':
gm_client = python3_gearman.GearmanClient(
[self.ovdm.get_gearman_server()])
payload_obj = json.loads(current_job.data)
job_data['collectionSystemTransferID'] = payload_obj[
'collectionSystemTransferID']
for task in self.ovdm.get_tasks_for_hook(current_job.task):
logging.info("Adding post task: %s", task)
gm_client.submit_job(task,
json.dumps(job_data),
background=True)
elif current_job.task == 'rebuildDataDashboard':
gm_client = python3_gearman.GearmanClient(
[self.ovdm.get_gearman_server()])
for task in self.ovdm.get_tasks_for_hook(current_job.task):
logging.info("Adding post task: %s", task)
gm_client.submit_job(task,
json.dumps(job_data),
background=True)
if len(results_obj['parts']) > 0:
if results_obj['parts'][-1]['result'] == "Fail": # Final Verdict
if int(self.task['taskID']) > 0:
self.ovdm.set_error_task(
self.task['taskID'],
results_obj['parts'][-1]['reason'])
else:
self.ovdm.send_msg(self.task['longName'] + ' failed',
results_obj['parts'][-1]['reason'])
else:
if int(self.task['taskID']) > 0:
self.ovdm.set_idle_task(self.task['taskID'])
else:
if int(self.task['taskID']) > 0:
self.ovdm.set_idle_task(self.task['taskID'])
logging.debug("Job Results: %s", json.dumps(results_obj, indent=2))
logging.info("Job: %s (%s) completed at: %s", self.task['longName'],
current_job.handle, time.strftime("%D %T", time.gmtime()))
return super().send_job_complete(current_job, job_result)
def task_rsync_publicdata_to_cruise_data(gearman_worker, gearman_job):
"""
Sync the contents of the PublicData share to the from_PublicData Extra Directory
"""
job_results = {'parts':[]}
publicdata_dir = gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehousePublicDataDir']
from_publicdata_dir = os.path.join(gearman_worker.cruise_dir, gearman_worker.ovdm.get_required_extra_directory_by_name('From_PublicData')['destDir'])
gearman_worker.send_job_status(gearman_job, 1, 10)
if os.path.exists(from_publicdata_dir):
job_results['parts'].append({"partName": "Verify From_PublicData directory exists", "result": "Pass"})
else:
job_results['parts'].append({"partName": "Verify From_PublicData directory exists", "result": "Fail", "reason": "Unable to locate the From_PublicData directory: " + from_publicdata_dir})
return json.dumps(job_results)
if os.path.exists(publicdata_dir):
job_results['parts'].append({"partName": "Verify PublicData directory exists", "result": "Pass"})
else:
job_results['parts'].append({"partName": "Verify PublicData directory exists", "result": "Fail", "reason": "Unable to locate the PublicData directory: " + publicdata_dir})
return json.dumps(job_results)
gearman_worker.send_job_status(gearman_job, 5, 10)
logging.info("Transferring files from PublicData to the cruise data directory")
output_results = transfer_publicdata_dir(gearman_worker, gearman_job)
if not output_results['verdict']:
job_results['parts'].append({"partName": "Transfer files", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Transfer files", "result": "Pass"})
files = output_results['files']
logging.debug("Files Transferred: %s",json.dumps(files, indent=2))
gearman_worker.send_job_status(gearman_job, 8, 10)
if len(files['new']) > 0 or len(files['updated']) > 0:
logging.info("Setting file permissions")
output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], from_publicdata_dir)
if output_results['verdict']:
job_results['parts'].append({"partName": "Set file/directory ownership/permissions", "result": "Pass"})
else:
job_results['parts'].append({"partName": "Set file/directory ownership/permissions", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
gearman_worker.send_job_status(gearman_job, 9, 100)
logging.info("Initiating MD5 Summary Task")
gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()])
gm_data = {}
gm_data['cruiseID'] = gearman_worker.cruise_id
gm_data['files'] = files
gm_data['files']['new'] = [os.path.join(from_publicdata_dir,filename) for filename in gm_data['files']['new']]
gm_data['files']['updated'] = [os.path.join(from_publicdata_dir,filename) for filename in gm_data['files']['updated']]
gm_client.submit_job("updateMD5Summary", json.dumps(gm_data))
logging.info("MD5 Summary Task Complete")
# need to verify update MD5 completed successfully
gearman_worker.send_job_status(gearman_job, 10, 10)
return json.dumps(job_results)
def task_finalize_current_cruise(gearman_worker, gearman_job): # pylint: disable=too-many-return-statements,too-many-statements
"""
Finalize the current cruise
"""
job_results = {'parts':[]}
gearman_worker.send_job_status(gearman_job, 1, 10)
publicdata_dir = gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehousePublicDataDir']
from_publicdata_dir = os.path.join(gearman_worker.cruise_dir, gearman_worker.ovdm.get_required_extra_directory_by_name('From_PublicData')['destDir'])
ovdm_config_file_path = os.path.join(gearman_worker.cruise_dir, DEFAULT_CRUISE_CONFIG_FN)
if not os.path.exists(gearman_worker.cruise_dir):
job_results['parts'].append({"partName": "Verify cruise directory exists", "result": "Fail", "reason": "Cruise directory: " + gearman_worker.cruise_dir + " could not be found"})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Verify cruise directory exists", "result": "Pass"})
gearman_worker.send_job_status(gearman_job, 2, 10)
logging.info("Queuing Collection System Transfers")
gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()])
gm_data = {
'cruiseID': gearman_worker.cruise_id,
'cruiseStartDate': gearman_worker.cruise_start_date,
'systemStatus': "On",
'collectionSystemTransfer': {}
}
collection_system_transfer_jobs = []
collection_system_transfers = gearman_worker.ovdm.get_active_collection_system_transfers(lowering=False)
for collection_system_transfer in collection_system_transfers:
logging.debug("Queuing runCollectionSystemTransfer job for %s", collection_system_transfer['name'])
gm_data['collectionSystemTransfer']['collectionSystemTransferID'] = collection_system_transfer['collectionSystemTransferID']
collection_system_transfer_jobs.append( {"task": "runCollectionSystemTransfer", "data": json.dumps(gm_data)} )
gearman_worker.send_job_status(gearman_job, 3, 10)
if len(collection_system_transfer_jobs) > 0:
logging.info("Submitting runCollectionSystemTransfer jobs")
submitted_job_request = gm_client.submit_multiple_jobs(collection_system_transfer_jobs, background=False, wait_until_complete=False)
gearman_worker.send_job_status(gearman_job, 4, 10)
time.sleep(1)
gm_client.wait_until_jobs_completed(submitted_job_request)
logging.info("Completed runCollectionSystemTransfers jobs")
gearman_worker.send_job_status(gearman_job, 5, 10)
if gearman_worker.ovdm.get_transfer_public_data():
logging.info("Transferring files from PublicData to the cruise data directory")
logging.debug("Verify From_PublicData directory exists within the cruise data directory")
if not os.path.exists(from_publicdata_dir):
job_results['parts'].append({"partName": "Verify From_PublicData directory exists", "result": "Fail", "reason": "From_PublicData directory: " + from_publicdata_dir + " could not be found"})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Verify From_PublicData directory exists", "result": "Pass"})
logging.debug("Verify PublicData Directory exists")
if not os.path.exists(publicdata_dir):
job_results['parts'].append({"partName": "Verify PublicData directory exists", "result": "Fail", "reason": "PublicData directory: " + publicdata_dir + " could not be found"})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Verify PublicData directory exists", "result": "Pass"})
logging.debug("Transferring files")
output_results = transfer_publicdata_dir(gearman_worker, gearman_job)
logging.debug("Transfer Complete")
if not output_results['verdict']:
job_results['parts'].append({"partName": "Transfer PublicData files", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Transfer PublicData files", "result": "Pass"})
files = output_results['files']
logging.debug("PublicData Files Transferred: %s", json.dumps(files, indent=2))
logging.info("Clearing files from PublicData")
output_results = clear_directory(publicdata_dir)
logging.debug("Clearing Complete")
if not output_results['verdict']:
job_results['parts'].append({"partName": "Clear out PublicData files", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Clear out PublicData files", "result": "Pass"})
gearman_worker.send_job_status(gearman_job, 9, 10)
if len(files['new']) > 0 or len(files['updated']) > 0:
output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], from_publicdata_dir)
if not output_results['verdict']:
job_results['parts'].append({"partName": "Set file/directory ownership/permissions", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Set file/directory ownership/permissions", "result": "Pass"})
else:
files = { 'include':[], 'exclude':[], 'new':[], 'updated':[] }
gearman_worker.send_job_status(gearman_job, 95, 100)
#build OpenVDM Config file
logging.info("Exporting OpenVDM Configuration")
output_results = export_ovdm_config(gearman_worker, ovdm_config_file_path, finalize=True)
if not output_results['verdict']:
job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Pass"})
output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], ovdm_config_file_path)
if not output_results['verdict']:
job_results['parts'].append({"partName": "Set OpenVDM config file ownership/permissions", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Set OpenVDM config file ownership/permissions", "result": "Pass"})
logging.info("Initiating MD5 Summary Task")
gm_data = {
'cruiseID': gearman_worker.cruise_id,
'files': files
}
# gm_data['files']['new'] = [from_publicdata_dir.replace(gearman_worker.cruise_dir, '') + '/' + filename for filename in gm_data['files']['new']]
# gm_data['files']['updated'] = [from_publicdata_dir.replace(gearman_worker.cruise_dir, '') + '/' + filename for filename in gm_data['files']['updated']]
gm_data['files']['updated'].append(DEFAULT_CRUISE_CONFIG_FN)
gm_client.submit_job("updateMD5Summary", json.dumps(gm_data))
logging.debug("MD5 Summary Task Complete")
# need to add code for cruise data transfers
gearman_worker.send_job_status(gearman_job, 10, 10)
return json.dumps(job_results)
def task_setup_new_cruise(gearman_worker, gearman_job): # pylint: disable=too-many-return-statements,too-many-statements
"""
Setup a new cruise
"""
job_results = {'parts':[]}
payload_obj = json.loads(gearman_job.data)
logging.debug("Payload: %s", json.dumps(payload_obj, indent=2))
ovdm_config_file_path = os.path.join(gearman_worker.cruise_dir, DEFAULT_CRUISE_CONFIG_FN)
gearman_worker.send_job_status(gearman_job, 1, 10)
gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()])
logging.info("Set ownership/permissions for the CruiseData directory")
completed_job_request = gm_client.submit_job("setCruiseDataDirectoryPermissions", gearman_job.data)
result_obj = json.loads(completed_job_request.result)
if result_obj['parts'][-1]['result'] != "Pass": # Final Verdict
logging.error("Failed to lockdown the CruiseData directory")
job_results['parts'].append({"partName": "Set ownership/permissions for CruiseData directory", "result": "Fail", "reason": result_obj['parts'][-1]['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Set ownership/permissions for CruiseData directory", "result": "Pass"})
logging.info("Creating cruise data directory")
completed_job_request = gm_client.submit_job("createCruiseDirectory", gearman_job.data)
result_obj = json.loads(completed_job_request.result)
if result_obj['parts'][-1]['result'] != "Pass": # Final Verdict
logging.error("Failed to create cruise data directory")
job_results['parts'].append({"partName": "Create cruise data directory structure", "result": "Fail", "reason": result_obj['parts'][-1]['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Create cruise data directory structure", "result": "Pass"})
gearman_worker.send_job_status(gearman_job, 5, 10)
#build OpenVDM Config file
logging.info("Exporting Cruise Configuration")
output_results = export_ovdm_config(gearman_worker, ovdm_config_file_path)
if not output_results['verdict']:
job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Export OpenVDM config data to file", "result": "Pass"})
output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], ovdm_config_file_path)
if not output_results['verdict']:
job_results['parts'].append({"partName": "Set OpenVDM config file ownership/permissions", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
gearman_worker.send_job_status(gearman_job, 7, 10)
logging.info("Creating MD5 summary files")
completed_job_request = gm_client.submit_job("rebuildMD5Summary", gearman_job.data)
result_obj = json.loads(completed_job_request.result)
if result_obj['parts'][-1]['result'] != "Pass": # Final Verdict
logging.error("Failed to create MD5 summary files")
job_results['parts'].append({"partName": "Create MD5 summary files", "result": "Fail", "reason": result_obj['parts'][-1]['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Create MD5 summary files", "result": "Pass"})
gearman_worker.send_job_status(gearman_job, 8, 10)
logging.info("Creating data dashboard directory structure and manifest file")
completed_job_request = gm_client.submit_job("rebuildDataDashboard", gearman_job.data)
result_obj = json.loads(completed_job_request.result)
if result_obj['parts'][-1]['result'] != "Pass": # Final Verdict
logging.error("Failed to create data dashboard directory structure and/or manifest file")
job_results['parts'].append({"partName": "Create data dashboard directory structure and manifest file", "result": "Fail", "reason": result_obj['parts'][-1]['reason']})
return json.dumps(job_results)
job_results['parts'].append({"partName": "Create data dashboard directory structure and manifest file", "result": "Pass"})
gearman_worker.send_job_status(gearman_job, 9, 10)
logging.info("Updating Cruise Size")
cruise_size_proc = subprocess.run(['du','-sb', gearman_worker.cruise_dir], capture_output=True, text=True, check=False)
if cruise_size_proc.returncode == 0:
logging.info("Cruise Size: %s", cruise_size_proc.stdout.split()[0])
gearman_worker.ovdm.set_cruise_size(cruise_size_proc.stdout.split()[0])
else:
gearman_worker.ovdm.set_cruise_size("0")
gearman_worker.ovdm.set_lowering_size("0")
gearman_worker.send_job_status(gearman_job, 10, 10)
return json.dumps(job_results)
import python3_gearman
def check_request_status(job_request):
if job_request.complete:
print("Job %s finished! Result: %s - %s" %
(job_request.job.unique, job_request.state, job_request.result))
elif job_request.timed_out:
print("Job %s timed out!" % job_request.unique)
elif job_request.state == JOB_UNKNOWN:
print("Job %s connection failed!" % job_request.unique)
gm_client = python3_gearman.GearmanClient(['localhost:4730'])
completed_job_request = gm_client.submit_job("reverse", "Hello World!")
check_request_status(completed_job_request)
import python3_gearman as gearman
def check_request_status(job_request):
# if job_request.complete:
print("Job %s finished! Result: %s - %s" %
(job_request.job.unique, job_request.state, job_request.result))
# elif job_request.timed_out:
# print "Job %s timed out!" % job_request.unique
gm_client = gearman.GearmanClient(['localhost:4730'])
arquivo_imagem = "/home/diego/projetos/ferias2018.2/core/mnist/fotos_teste/0.png"
completed_job_request = gm_client.submit_job("process",
'{"data":"%s"}' % arquivo_imagem)
check_request_status(completed_job_request)
print("Exception:", completed_job_request.exception)
print("Warnings:", ', '.join(completed_job_request.warning_updates))
print("Result:", completed_job_request.result)
print('Work complete with state %s' % completed_job_request.state)
def task_run_ship_to_shore_transfer(gearman_worker, current_job): # pylint: disable=too-many-statements
"""
Perform the ship-to-shore transfer
"""
time.sleep(randint(0, 2))
job_results = {
'parts': [{
"partName": "Transfer In-Progress",
"result": "Pass"
}, {
"partName": "Transfer Enabled",
"result": "Pass"
}],
'files': {}
}
logging.debug("Setting transfer status to 'Running'")
gearman_worker.ovdm.set_running_cruise_data_transfer(
gearman_worker.cruise_data_transfer['cruiseDataTransferID'],
os.getpid(), current_job.handle)
logging.info("Testing configuration")
gearman_worker.send_job_status(current_job, 1, 10)
gm_client = python3_gearman.GearmanClient(
[gearman_worker.ovdm.get_gearman_server()])
gm_data = {
'cruiseDataTransfer': gearman_worker.cruise_data_transfer,
'cruiseID': gearman_worker.cruise_id
}
completed_job_request = gm_client.submit_job("testCruiseDataTransfer",
json.dumps(gm_data))
results_obj = json.loads(completed_job_request.result)
logging.debug('Connection Test Results: %s',
json.dumps(results_obj, indent=2))
if results_obj['parts'][-1]['result'] == "Pass": # Final Verdict
logging.debug("Connection test passed")
job_results['parts'].append({
"partName": "Connection Test",
"result": "Pass"
})
else:
logging.warning("Connection test failed, quitting job")
job_results['parts'].append({
"partName":
"Connection Test",
"result":
"Fail",
"reason":
results_obj['parts'][-1]['reason']
})
return json.dumps(job_results)
gearman_worker.send_job_status(current_job, 2, 10)
logging.info("Transferring files")
output_results = None
if gearman_worker.cruise_data_transfer[
'transferType'] == "4": # SSH Server
output_results = transfer_ssh_dest_dir(gearman_worker, current_job)
else:
logging.error("Unknown Transfer Type")
job_results['parts'].append({
"partName": "Transfer Files",
"result": "Fail",
"reason": "Unknown transfer type"
})
return json.dumps(job_results)
if not output_results['verdict']:
logging.error("Transfer of remote files failed: %s",
output_results['reason'])
job_results['parts'].append({
"partName": "Transfer Files",
"result": "Fail",
"reason": output_results['reason']
})
return job_results
logging.debug("Transfer completed successfully")
job_results['files'] = output_results['files']
job_results['parts'].append({
"partName": "Transfer Files",
"result": "Pass"
})
if len(job_results['files']['new']) > 0:
logging.debug("%s file(s) added", len(job_results['files']['new']))
if len(job_results['files']['updated']) > 0:
logging.debug("%s file(s) updated",
len(job_results['files']['updated']))
if len(job_results['files']['exclude']) > 0:
logging.debug("%s file(s) intentionally skipped",
len(job_results['files']['exclude']))
gearman_worker.send_job_status(current_job, 9, 10)
if job_results['files']['new'] or job_results['files']['updated']:
logging.debug("Building logfiles")
logfile_filename = gearman_worker.cruise_data_transfer[
'name'] + '_' + gearman_worker.transfer_start_date + '.log'
log_contents = {
'files': {
'new': job_results['files']['new'],
'updated': job_results['files']['updated']
}
}
output_results = output_json_data_to_file(
os.path.join(build_logfile_dirpath(gearman_worker),
logfile_filename), log_contents['files'])
if output_results['verdict']:
job_results['parts'].append({
"partName": "Write transfer logfile",
"result": "Pass"
})
else:
logging.error("Error writing transfer logfile: %s",
logfile_filename)
job_results['parts'].append({
"partName": "Write transfer logfile",
"result": "Fail",
"reason": output_results['reason']
})
return json.dumps(job_results)
output_results = set_owner_group_permissions(
gearman_worker.
shipboard_data_warehouse_config['shipboardDataWarehouseUsername'],
os.path.join(build_logfile_dirpath(gearman_worker),
logfile_filename))
if not output_results['verdict']:
job_results['parts'].append({
"partName": "Set OpenVDM config file ownership/permissions",
"result": "Fail",
"reason": output_results['reason']
})
return json.dumps(job_results)
gearman_worker.send_job_status(current_job, 10, 10)
time.sleep(2)
return json.dumps(job_results)
def task_finalize_current_lowering(gearman_worker, gearman_job):
"""
Finalize the current lowering
"""
job_results = {'parts':[]}
gearman_worker.send_job_status(gearman_job, 1, 10)
lowering_config_filepath = os.path.join(gearman_worker.lowering_dir, DEFAULT_LOWERING_CONFIG_FN)
if os.path.exists(gearman_worker.lowering_dir) and (gearman_worker.lowering_id != ''):
job_results['parts'].append({"partName": "Verify Lowering Directory exists", "result": "Pass"})
else:
job_results['parts'].append({"partName": "Verify Lowering Directory exists", "result": "Fail", "reason": "Lowering directory: " + gearman_worker.lowering_dir + " could not be found"})
return json.dumps(job_results)
gearman_worker.send_job_status(gearman_job, 3, 10)
logging.info("Queuing Collection System Transfers")
gm_client = python3_gearman.GearmanClient([gearman_worker.ovdm.get_gearman_server()])
gm_data = {
'loweringID': gearman_worker.lowering_id,
'loweringStartDate': gearman_worker.lowering_start_date,
'systemStatus': "On",
'collectionSystemTransfer': {}
}
collection_system_transfer_jobs = []
collection_system_transfers = gearman_worker.ovdm.get_active_collection_system_transfers(cruise=False)
for collection_system_transfer in collection_system_transfers:
logging.debug("Queuing runCollectionSystemTransfer job for %s", collection_system_transfer['name'])
gm_data['collectionSystemTransfer']['collectionSystemTransferID'] = collection_system_transfer['collectionSystemTransferID']
collection_system_transfer_jobs.append( {"task": "runCollectionSystemTransfer", "data": json.dumps(gm_data)} )
gearman_worker.send_job_status(gearman_job, 5, 10)
if len(collection_system_transfer_jobs) > 0:
logging.info("Submitting runCollectionSystemTransfer jobs")
submitted_job_request = gm_client.submit_multiple_jobs(collection_system_transfer_jobs, background=False, wait_until_complete=False)
gearman_worker.send_job_status(gearman_job, 7, 10)
time.sleep(1)
gm_client.wait_until_jobs_completed(submitted_job_request)
logging.info("Completed runCollectionSystemTransfers jobs")
gearman_worker.send_job_status(gearman_job, 9, 10)
#build Lowering Config file
logging.info("Exporting Lowering Configuration")
output_results = export_lowering_config(gearman_worker, lowering_config_filepath, finalize=True)
if output_results['verdict']:
job_results['parts'].append({"partName": "Export Lowering config data to file", "result": "Pass"})
else:
job_results['parts'].append({"partName": "Export Lowering config data to file", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
output_results = set_owner_group_permissions(gearman_worker.shipboard_data_warehouse_config['shipboardDataWarehouseUsername'], lowering_config_filepath)
if output_results['verdict']:
job_results['parts'].append({"partName": "Set Lowering config file ownership/permissions", "result": "Pass"})
else:
job_results['parts'].append({"partName": "Set Lowering config file ownership/permissions", "result": "Fail", "reason": output_results['reason']})
return json.dumps(job_results)
# need to add code for updating MD5
gearman_worker.send_job_status(gearman_job, 10, 10)
return json.dumps(job_results)
