def poll_processing(self, processing, transform, input_collection, output_collection, output_contents): if transform['transform_type'] == TransformType.StageIn: if 'stagein_poller' not in self.plugins: raise AgentPluginError('Plugin stagein_poller is required') return self.plugins['stagein_poller'](processing, transform, input_collection, output_collection, output_contents) if transform['transform_type'] == TransformType.ActiveLearning: if 'activelearning_poller' not in self.plugins: raise AgentPluginError( 'Plugin activelearning_poller is required') return self.plugins['activelearning_poller'](processing, transform, input_collection, output_collection, output_contents) if transform['transform_type'] == TransformType.HyperParameterOpt: if 'hyperparameteropt_poller' not in self.plugins: raise AgentPluginError( 'Plugin hyperparameteropt_poller is required') return self.plugins['hyperparameteropt_poller'](processing, transform, input_collection, output_collection, output_contents) return None
def submit_processing(self, processing, transform, input_collection, output_collection): if transform['transform_type'] == TransformType.StageIn: if 'stagein_submitter' not in self.plugins: raise AgentPluginError('Plugin stagein_submitter is required') return self.plugins['stagein_submitter'](processing, transform, input_collection, output_collection) if transform['transform_type'] == TransformType.ActiveLearning: if 'activelearning_submitter' not in self.plugins: raise AgentPluginError('Plugin activelearning_submitter is required') return self.plugins['activelearning_submitter'](processing, transform, input_collection, output_collection) return None
def generate_transform_output_contents(self, transform, input_collection, output_collection, contents): self.logger.debug("generate_transform_output_contents: transform: %s, number of input_contents: %s" % (transform, len(contents))) if transform['transform_type'] == TransformType.StageIn: if 'stagein_transformer' not in self.plugins: raise AgentPluginError('Plugin stagein_transformer is required') return self.plugins['stagein_transformer'](transform, input_collection, output_collection, contents) if transform['transform_type'] == TransformType.ActiveLearning: if 'activelearning_transformer' not in self.plugins: raise AgentPluginError('Plugin activelearning_transformer is required') return self.plugins['activelearning_transformer'](transform, input_collection, output_collection, contents) if transform['transform_type'] == TransformType.HyperParameterOpt: if 'hyperparameteropt_transformer' not in self.plugins: raise AgentPluginError('Plugin hyperparameteropt_transformer is required') return self.plugins['hyperparameteropt_transformer'](transform, input_collection, output_collection, contents) return []
def start_notifier(self): if 'notifier' not in self.plugins: raise AgentPluginError('Plugin notifier is required') self.notifier = self.plugins['notifier'] self.logger.info("Starting notifier: %s" % self.notifier) self.notifier.set_request_queue(self.message_queue) self.notifier.start()
def get_collections(self, scope, name, req=None): if (req and req['request_metadata'] and 'is_pseudo_input' in req['request_metadata'] # noqa: W503 and req['request_metadata']['is_pseudo_input'] == True): # noqa: W503 collection = { 'scope': scope, 'name': name, 'total_files': 1, 'bytes': 0, 'coll_type': CollectionType.PseudoDataset } return [collection] else: if 'collection_lister' not in self.plugins: raise AgentPluginError('Plugin collection_lister is required') return self.plugins['collection_lister'](scope, name)
def get_contents(self, scope, name): if 'contents_lister' not in self.plugins: raise AgentPluginError('Plugin contents_lister is required') return self.plugins['contents_lister'](scope, name)
def get_collection_metadata(self, scope, name): if 'collection_metadata_reader' not in self.plugins: raise AgentPluginError( 'Plugin collection_metadata_reader is required') return self.plugins['collection_metadata_reader'](scope, name)