def process_new_processing(self, processing):
transform_id = processing['transform_id']
processing_metadata = processing['processing_metadata']
input_coll_id = processing_metadata['input_collection']
input_collection = core_catalog.get_collection(coll_id=input_coll_id)
output_coll_id = processing_metadata['output_collection']
output_collection = core_catalog.get_collection(coll_id=output_coll_id)
transform = core_transforms.get_transform(transform_id)
ret = self.submit_processing(processing, transform, input_collection, output_collection)
if ret:
return ret
else:
return {'processing_id': processing['processing_id'],
'locking': ProcessingLocking.Idle}
def is_input_collection_all_processed(self, coll_id_list):
is_all_processed = True
for coll_id in coll_id_list:
coll = core_catalog.get_collection(coll_id)
if not (coll['status'] == CollectionStatus.Closed
and coll['total_files'] == coll['processed_files']):
is_all_processed = False
return is_all_processed
return is_all_processed
def process_monitor_processing(self, processing):
transform_id = processing['transform_id']
processing_metadata = processing['processing_metadata']
input_coll_id = processing_metadata['input_collection']
input_collection = core_catalog.get_collection(coll_id=input_coll_id)
output_coll_id = processing_metadata['output_collection']
output_collection = core_catalog.get_collection(coll_id=output_coll_id)
output_contents = core_catalog.get_contents_by_coll_id_status(coll_id=output_coll_id)
transform = core_transforms.get_transform(transform_id)
ret_poll = self.poll_processing(processing, transform, input_collection, output_collection, output_contents)
if not ret_poll:
return {'processing_id': processing['processing_id'],
'locking': ProcessingLocking.Idle}
new_files = []
if 'new_files' in ret_poll:
new_files = ret_poll['new_files']
updated_files = ret_poll['updated_files']
file_msg = []
if updated_files:
file_msg = self.generate_file_message(transform, updated_files)
processing_parameters = {'status': ret_poll['processing_updates']['status'],
'locking': ProcessingLocking.Idle,
'processing_metadata': ret_poll['processing_updates']['processing_metadata']}
if 'output_metadata' in ret_poll['processing_updates']:
processing_parameters['output_metadata'] = ret_poll['processing_updates']['output_metadata']
updated_processing = {'processing_id': processing['processing_id'],
'parameters': processing_parameters}
ret = {'transform': transform,
'processing_updates': updated_processing,
'updated_files': updated_files,
'new_files': new_files,
'file_message': file_msg}
return ret