def poll_processing(self, processing, transform, input_collection,
output_collection, output_contents):
if transform['transform_type'] == TransformType.StageIn:
if 'stagein_poller' not in self.plugins:
raise AgentPluginError('Plugin stagein_poller is required')
return self.plugins['stagein_poller'](processing, transform,
input_collection,
output_collection,
output_contents)
if transform['transform_type'] == TransformType.ActiveLearning:
if 'activelearning_poller' not in self.plugins:
raise AgentPluginError(
'Plugin activelearning_poller is required')
return self.plugins['activelearning_poller'](processing, transform,
input_collection,
output_collection,
output_contents)
if transform['transform_type'] == TransformType.HyperParameterOpt:
if 'hyperparameteropt_poller' not in self.plugins:
raise AgentPluginError(
'Plugin hyperparameteropt_poller is required')
return self.plugins['hyperparameteropt_poller'](processing,
transform,
input_collection,
output_collection,
output_contents)
return None
def submit_processing(self, processing, transform, input_collection, output_collection):
if transform['transform_type'] == TransformType.StageIn:
if 'stagein_submitter' not in self.plugins:
raise AgentPluginError('Plugin stagein_submitter is required')
return self.plugins['stagein_submitter'](processing, transform, input_collection, output_collection)
if transform['transform_type'] == TransformType.ActiveLearning:
if 'activelearning_submitter' not in self.plugins:
raise AgentPluginError('Plugin activelearning_submitter is required')
return self.plugins['activelearning_submitter'](processing, transform, input_collection, output_collection)
return None
def generate_transform_output_contents(self, transform, input_collection, output_collection, contents):
self.logger.debug("generate_transform_output_contents: transform: %s, number of input_contents: %s" % (transform, len(contents)))
if transform['transform_type'] == TransformType.StageIn:
if 'stagein_transformer' not in self.plugins:
raise AgentPluginError('Plugin stagein_transformer is required')
return self.plugins['stagein_transformer'](transform, input_collection, output_collection, contents)
if transform['transform_type'] == TransformType.ActiveLearning:
if 'activelearning_transformer' not in self.plugins:
raise AgentPluginError('Plugin activelearning_transformer is required')
return self.plugins['activelearning_transformer'](transform, input_collection, output_collection, contents)
if transform['transform_type'] == TransformType.HyperParameterOpt:
if 'hyperparameteropt_transformer' not in self.plugins:
raise AgentPluginError('Plugin hyperparameteropt_transformer is required')
return self.plugins['hyperparameteropt_transformer'](transform, input_collection, output_collection, contents)
return []
def start_notifier(self):
if 'notifier' not in self.plugins:
raise AgentPluginError('Plugin notifier is required')
self.notifier = self.plugins['notifier']
self.logger.info("Starting notifier: %s" % self.notifier)
self.notifier.set_request_queue(self.message_queue)
self.notifier.start()
def get_collections(self, scope, name, req=None):
if (req and req['request_metadata']
and 'is_pseudo_input' in req['request_metadata'] # noqa: W503
and req['request_metadata']['is_pseudo_input']
== True): # noqa: W503
collection = {
'scope': scope,
'name': name,
'total_files': 1,
'bytes': 0,
'coll_type': CollectionType.PseudoDataset
}
return [collection]
else:
if 'collection_lister' not in self.plugins:
raise AgentPluginError('Plugin collection_lister is required')
return self.plugins['collection_lister'](scope, name)
def get_contents(self, scope, name):
if 'contents_lister' not in self.plugins:
raise AgentPluginError('Plugin contents_lister is required')
return self.plugins['contents_lister'](scope, name)
def get_collection_metadata(self, scope, name):
if 'collection_metadata_reader' not in self.plugins:
raise AgentPluginError(
'Plugin collection_metadata_reader is required')
return self.plugins['collection_metadata_reader'](scope, name)