def get_transform(transform_id):
"""
Get transform.
:param transform_id: Transform id.
:returns: Transform.
"""
return transforms.get_transform(transform_id)
def process_new_processing(self, processing):
transform_id = processing['transform_id']
processing_metadata = processing['processing_metadata']
input_coll_id = processing_metadata['input_collection']
input_collection = core_catalog.get_collection(coll_id=input_coll_id)
output_coll_id = processing_metadata['output_collection']
output_collection = core_catalog.get_collection(coll_id=output_coll_id)
transform = core_transforms.get_transform(transform_id)
ret = self.submit_processing(processing, transform, input_collection, output_collection)
if ret:
return ret
else:
return {'processing_id': processing['processing_id'],
'locking': ProcessingLocking.Idle}
def process_running_workprogress(self, workprogress):
"""
process running workprogresses
"""
self.logger.info("process_running_workprogress: workprogress_id: %s" %
workprogress['workprogress_id'])
workprogress_metadata = workprogress['workprogress_metadata']
wf = workprogress_metadata['workflow']
works = wf.get_current_works()
for work in works:
tf = core_transforms.get_transform(transform_id=work.get_work_id())
work_status = WorkStatus(tf['status'].value)
work.set_status(work_status)
work.set_terminated_msg(msg=None) # TODO
if wf.is_terminated():
if wf.is_finished():
wp_status = WorkprogressStatus.Finished
elif wf.is_subfinished():
wp_status = WorkprogressStatus.SubFinished
elif wf.is_failed():
wp_status = WorkprogressStatus.Failed
else:
wp_status = WorkprogressStatus.Failed
wp_msg = wf.get_terminated_msg()
else:
wp_status = WorkprogressStatus.Transforming
wp_msg = None
parameters = {
'status': wp_status,
'locking': WorkprogressLocking.Idle,
'errors': {
'msg': wp_msg
}
}
ret = {
'workprogress_id': workprogress['workprogress_id'],
'parameters': parameters
}
return ret
def process_monitor_processing(self, processing):
transform_id = processing['transform_id']
processing_metadata = processing['processing_metadata']
input_coll_id = processing_metadata['input_collection']
input_collection = core_catalog.get_collection(coll_id=input_coll_id)
output_coll_id = processing_metadata['output_collection']
output_collection = core_catalog.get_collection(coll_id=output_coll_id)
output_contents = core_catalog.get_contents_by_coll_id_status(coll_id=output_coll_id)
transform = core_transforms.get_transform(transform_id)
ret_poll = self.poll_processing(processing, transform, input_collection, output_collection, output_contents)
if not ret_poll:
return {'processing_id': processing['processing_id'],
'locking': ProcessingLocking.Idle}
new_files = []
if 'new_files' in ret_poll:
new_files = ret_poll['new_files']
updated_files = ret_poll['updated_files']
file_msg = []
if updated_files:
file_msg = self.generate_file_message(transform, updated_files)
processing_parameters = {'status': ret_poll['processing_updates']['status'],
'locking': ProcessingLocking.Idle,
'processing_metadata': ret_poll['processing_updates']['processing_metadata']}
if 'output_metadata' in ret_poll['processing_updates']:
processing_parameters['output_metadata'] = ret_poll['processing_updates']['output_metadata']
updated_processing = {'processing_id': processing['processing_id'],
'parameters': processing_parameters}
ret = {'transform': transform,
'processing_updates': updated_processing,
'updated_files': updated_files,
'new_files': new_files,
'file_message': file_msg}
return ret
def process_output_collection(self, coll):
"""
Process output collection
"""
if coll['coll_metadata'] and 'to_register' in coll[
'coll_metadata'] and coll['coll_metadata'][
'to_register'] is True:
contents = core_catalog.get_contents(coll_id=coll['coll_id'])
registered_contents = self.get_contents(coll['scope'],
coll['name'])
registered_content_names = [
con['name'] for con in registered_contents
]
new_contents = []
for content in contents:
if content['name'] not in registered_content_names:
new_contents.append(content)
if new_contents:
self.register_contents(coll['scope'], coll['name'],
new_contents)
is_input_collection_all_processed = False
if coll['coll_metadata'] and 'input_collections' in coll[
'coll_metadata'] and coll['coll_metadata']['input_collections']:
input_coll_list = coll['coll_metadata']['input_collections']
is_input_collection_all_processed = self.is_input_collection_all_processed(
input_coll_list)
contents_statistics = core_catalog.get_content_status_statistics(
coll_id=coll['coll_id'])
contents_statistics_with_name = {}
for key in contents_statistics:
contents_statistics_with_name[key.name] = contents_statistics[key]
content_status_keys = list(contents_statistics.keys())
total_files = sum(contents_statistics.values())
new_files = 0
processed_files = 0
output_metadata = None
if ContentStatus.Available in contents_statistics:
processed_files += contents_statistics[ContentStatus.Available]
if ContentStatus.Available.value in contents_statistics:
processed_files += contents_statistics[
ContentStatus.Available.value]
if ContentStatus.New in contents_statistics:
new_files += contents_statistics[ContentStatus.New]
if ContentStatus.New.value in contents_statistics:
new_files += contents_statistics[ContentStatus.New.value]
coll_msg = None
if not is_input_collection_all_processed:
coll_status = CollectionStatus.Processing
elif content_status_keys == [
ContentStatus.Available
] or content_status_keys == [ContentStatus.Available.value]:
coll_status = CollectionStatus.Closed
transform = core_transforms.get_transform(
coll['coll_metadata']['transform_id'])
if 'processing_id' in transform['transform_metadata']:
processing = core_processings.get_processing(
transform['transform_metadata']['processing_id'])
output_metadata = processing['output_metadata']
if transform['transform_type'] in [
TransformType.StageIn, TransformType.StageIn.value
]:
msg_type = 'collection_stagein'
msg_type_c = MessageType.StageInCollection
elif transform['transform_type'] in [
TransformType.ActiveLearning,
TransformType.ActiveLearning.value
]:
msg_type = 'collection_activelearning'
msg_type_c = MessageType.ActiveLearningCollection
elif transform['transform_type'] in [
TransformType.HyperParameterOpt,
TransformType.HyperParameterOpt.value
]:
msg_type = 'collection_hyperparameteropt'
msg_type_c = MessageType.HyperParameterOptCollection
else:
msg_type = 'collection_unknown'
msg_type_c = MessageType.UnknownCollection
msg_content = {
'msg_type':
msg_type,
'workload_id':
coll['coll_metadata']['workload_id']
if 'workload_id' in coll['coll_metadata'] else None,
'collections': [{
'scope': coll['scope'],
'name': coll['name'],
'status': 'Available'
}],
'output':
output_metadata
}
coll_msg = {
'msg_type':
msg_type_c,
'status':
MessageStatus.New,
'source':
MessageSource.Transporter,
'transform_id':
coll['coll_metadata']['transform_id']
if 'transform_id' in coll['coll_metadata'] else None,
'num_contents':
1,
'msg_content':
msg_content
}
elif content_status_keys == [
ContentStatus.FinalFailed
] or content_status_keys == [ContentStatus.FinalFailed.value]:
coll_status = CollectionStatus.Failed
elif (len(content_status_keys) == 2 # noqa: W503
and (ContentStatus.FinalFailed in content_status_keys
or ContentStatus.FinalFailed.value in content_status_keys
) # noqa: W503
and (ContentStatus.Available in content_status_keys
or ContentStatus.Available.value
in content_status_keys)): # noqa: W503
coll_status = CollectionStatus.SubClosed
elif (ContentStatus.New in content_status_keys
or ContentStatus.New.value in content_status_keys # noqa: W503
or ContentStatus.Failed in content_status_keys or
ContentStatus.Failed.value in content_status_keys): # noqa: W503
coll_status = CollectionStatus.Processing
coll_metadata = coll['coll_metadata']
if not coll_metadata:
coll_metadata = {}
coll_metadata['status_statistics'] = contents_statistics_with_name
coll_metadata['output_metadata'] = output_metadata
ret_coll = {
'coll_id': coll['coll_id'],
'total_files': total_files,
'status': coll_status,
'new_files': new_files,
'processing_files': 0,
'processed_files': processed_files,
'coll_metadata': coll_metadata,
'coll_msg': coll_msg
}
return ret_coll
def process_running_workprogress(self, workprogress):
"""
process running workprogresses
"""
self.logger.info("process_running_workprogress: workprogress_id: %s" %
workprogress['workprogress_id'])
workprogress_metadata = workprogress['workprogress_metadata']
wf = workprogress_metadata['workflow']
new_transforms = []
if workprogress['status'] in [WorkprogressStatus.Transforming]:
# new works
works = wf.get_new_works()
for work in works:
new_work = work.copy()
new_work.add_proxy(wf.get_proxy())
new_transform = {
'workprogress_id': workprogress['workprogress_id'],
'request_id': workprogress['request_id'],
'workload_id': workprogress['workload_id'],
'transform_type': work.get_work_type(),
'transform_tag': work.get_work_tag(),
'priority': workprogress['priority'],
'status': TransformStatus.New,
'retries': 0,
'expired_at': workprogress['expired_at'],
'transform_metadata': {
'orginal_work': work,
'work': new_work
}
# 'collections': related_collections
}
new_transforms.append(new_transform)
self.logger.info(
"Processing workprogress(%s): new transforms: %s" %
(workprogress['workprogress_id'], new_transforms))
update_transforms = {}
if workprogress['status'] in [WorkprogressStatus.ToCancel]:
# current works
works = wf.get_current_works()
# print(works)
for work in works:
if work.get_status() not in [
WorkStatus.Finished, WorkStatus.SubFinished,
WorkStatus.Failed, WorkStatus.Cancelling,
WorkStatus.Cancelled
]:
update_transforms[work.get_work_id()] = {
'status': TransformStatus.ToCancel
}
# current works
works = wf.get_current_works()
# print(works)
for work in works:
# print(work.get_work_id())
tf = core_transforms.get_transform(transform_id=work.get_work_id())
work_status = WorkStatus(tf['status'].value)
work.set_status(work_status)
work.set_terminated_msg(msg=None) # TODO
if wf.is_terminated():
if wf.is_finished():
wp_status = WorkprogressStatus.Finished
elif wf.is_subfinished():
wp_status = WorkprogressStatus.SubFinished
elif wf.is_failed():
wp_status = WorkprogressStatus.Failed
elif wf.is_cancelled():
wp_status = WorkprogressStatus.Cancelled
else:
wp_status = WorkprogressStatus.Failed
wp_msg = wf.get_terminated_msg()
else:
wp_status = WorkprogressStatus.Transforming
wp_msg = None
parameters = {
'status': wp_status,
'locking': WorkprogressLocking.Idle,
'workprogress_metadata': workprogress_metadata,
'errors': {
'msg': wp_msg
}
}
ret = {
'workprogress_id': workprogress['workprogress_id'],
'parameters': parameters,
'new_transforms': new_transforms,
'update_transforms': update_transforms
}
return ret
def process_running_request_real(self, req):
"""
process running request
"""
self.logger.info("process_running_request: request_id: %s" %
req['request_id'])
wf = req['request_metadata']['workflow']
new_transforms = []
if req['status'] in [RequestStatus.Transforming]:
# new works
works = wf.get_new_works()
for work in works:
# new_work = work.copy()
new_work = work
new_work.add_proxy(wf.get_proxy())
new_transform = self.generate_transform(req, new_work)
new_transforms.append(new_transform)
self.logger.debug("Processing request(%s): new transforms: %s" %
(req['request_id'], str(new_transforms)))
# current works
works = wf.get_current_works()
# print(works)
for work in works:
# print(work.get_work_id())
tf = core_transforms.get_transform(transform_id=work.get_work_id())
transform_work = tf['transform_metadata']['work']
# work_status = WorkStatus(tf['status'].value)
# work.set_status(work_status)
work.sync_work_data(status=tf['status'],
substatus=tf['substatus'],
work=transform_work)
wf.refresh_works()
is_operation = False
if wf.is_terminated():
if wf.is_finished():
req_status = RequestStatus.Finished
elif wf.is_subfinished():
req_status = RequestStatus.SubFinished
elif wf.is_expired():
req_status = RequestStatus.Expired
elif wf.is_failed():
req_status = RequestStatus.Failed
elif wf.is_cancelled():
req_status = RequestStatus.Cancelled
elif wf.is_suspended():
req_status = RequestStatus.Suspended
else:
req_status = RequestStatus.Failed
req_msg = wf.get_terminated_msg()
else:
req_msg = None
if req['status'] in [
RequestStatus.ToSuspend, RequestStatus.Suspending
]:
req_status = RequestStatus.Suspending
is_operation = True
elif req['status'] in [
RequestStatus.ToCancel, RequestStatus.Cancelling
]:
req_status = RequestStatus.Cancelling
is_operation = True
elif wf.is_to_expire(req['expired_at'],
self.pending_time,
request_id=req['request_id']):
wf.expired = True
req_status = RequestStatus.ToExpire
is_operation = True
req_msg = "Workflow expired"
else:
req_status = RequestStatus.Transforming
# processing_metadata['workflow_data'] = wf.get_running_data()
if not is_operation:
next_poll_at = datetime.datetime.utcnow() + datetime.timedelta(
seconds=self.poll_time_period)
else:
next_poll_at = datetime.datetime.utcnow() + datetime.timedelta(
seconds=self.poll_operation_time_period)
parameters = {
'status': req_status,
'locking': RequestLocking.Idle,
'next_poll_at': next_poll_at,
'request_metadata': req['request_metadata'],
'errors': {
'msg': req_msg
}
}
new_messages = []
if req_status == RequestStatus.ToExpire:
# parameters['substatus'] = req_status
new_message = self.get_message_for_update_request(req, req_status)
new_messages.append(new_message)
ret = {
'request_id': req['request_id'],
'parameters': parameters,
'new_messages': new_messages,
'new_transforms': new_transforms
} # 'update_transforms': update_transforms}
return ret