def _operation_finished(operation, simulator_gid):
op_ident = dao.get_operation_process_for_operation(operation.id)
# TODO: Handle login
job = Job(
Transport(os.environ[HPCSchedulerClient.CSCS_LOGIN_TOKEN_ENV_KEY]),
op_ident.job_id)
operation = dao.get_operation_by_id(operation.id)
folder = HPCSchedulerClient.storage_interface.get_project_folder(
operation.project.name)
storage_interface = StorageInterface()
if storage_interface.encryption_enabled():
storage_interface.inc_project_usage_count(folder)
storage_interface.sync_folders(folder)
try:
sim_h5_filenames, metric_op, metric_h5_filename = \
HPCSchedulerClient.stage_out_to_operation_folder(job.working_dir, operation, simulator_gid)
operation.mark_complete(STATUS_FINISHED)
dao.store_entity(operation)
HPCSchedulerClient().update_db_with_results(
operation, sim_h5_filenames, metric_op, metric_h5_filename)
except OperationException as exception:
HPCOperationService.LOGGER.error(exception)
HPCOperationService._operation_error(operation)
finally:
if storage_interface.encryption_enabled():
storage_interface.sync_folders(folder)
storage_interface.set_project_inactive(operation.project)
def check_operations_job():
operations = dao.get_operations()
if operations is None or len(operations) == 0:
return
for operation in operations:
HPCOperationService.LOGGER.info("Start processing operation {}".format(operation.id))
try:
op_ident = dao.get_operation_process_for_operation(operation.id)
if op_ident is not None:
transport = Transport(os.environ[HPCSchedulerClient.CSCS_LOGIN_TOKEN_ENV_KEY])
job = Job(transport, op_ident.job_id)
job_status = job.properties['status']
if job.is_running():
if operation.status == STATUS_PENDING and job_status == HPCJobStatus.READY.value:
HPCOperationService._operation_started(operation)
HPCOperationService.LOGGER.info(
"CSCS job status: {} for operation {}.".format(job_status, operation.id))
return
HPCOperationService.LOGGER.info(
"Job for operation {} has status {}".format(operation.id, job_status))
if job_status == HPCJobStatus.SUCCESSFUL.value:
simulator_gid = operation.view_model_gid
HPCOperationService._operation_finished(operation, simulator_gid)
else:
HPCOperationService._operation_error(operation)
except Exception:
HPCOperationService.LOGGER.error(
"There was an error on background processing process for operation {}".format(operation.id),
exc_info=True)
def _operation_finished(operation, simulator_gid):
op_ident = dao.get_operation_process_for_operation(operation.id)
# TODO: Handle login
job = Job(
Transport(os.environ[HPCSchedulerClient.CSCS_LOGIN_TOKEN_ENV_KEY]),
op_ident.job_id)
sim_h5_filenames, metric_op, metric_h5_filename = \
HPCSchedulerClient.stage_out_to_operation_folder(job.working_dir, operation, simulator_gid)
operation.mark_complete(STATUS_FINISHED)
dao.store_entity(operation)
HPCSchedulerClient().update_db_with_results(operation,
sim_h5_filenames,
metric_op,
metric_h5_filename)