Example #1
0
    def poll_processing(self, processing, transform, input_collection,
                        output_collection, output_contents):
        if transform['transform_type'] == TransformType.StageIn:
            if 'stagein_poller' not in self.plugins:
                raise AgentPluginError('Plugin stagein_poller is required')
            return self.plugins['stagein_poller'](processing, transform,
                                                  input_collection,
                                                  output_collection,
                                                  output_contents)
        if transform['transform_type'] == TransformType.ActiveLearning:
            if 'activelearning_poller' not in self.plugins:
                raise AgentPluginError(
                    'Plugin activelearning_poller is required')
            return self.plugins['activelearning_poller'](processing, transform,
                                                         input_collection,
                                                         output_collection,
                                                         output_contents)
        if transform['transform_type'] == TransformType.HyperParameterOpt:
            if 'hyperparameteropt_poller' not in self.plugins:
                raise AgentPluginError(
                    'Plugin hyperparameteropt_poller is required')
            return self.plugins['hyperparameteropt_poller'](processing,
                                                            transform,
                                                            input_collection,
                                                            output_collection,
                                                            output_contents)

        return None
Example #2
0
    def submit_processing(self, processing, transform, input_collection, output_collection):
        if transform['transform_type'] == TransformType.StageIn:
            if 'stagein_submitter' not in self.plugins:
                raise AgentPluginError('Plugin stagein_submitter is required')
            return self.plugins['stagein_submitter'](processing, transform, input_collection, output_collection)
        if transform['transform_type'] == TransformType.ActiveLearning:
            if 'activelearning_submitter' not in self.plugins:
                raise AgentPluginError('Plugin activelearning_submitter is required')
            return self.plugins['activelearning_submitter'](processing, transform, input_collection, output_collection)

        return None
Example #3
0
    def generate_transform_output_contents(self, transform, input_collection, output_collection, contents):
        self.logger.debug("generate_transform_output_contents: transform: %s, number of input_contents: %s" % (transform, len(contents)))
        if transform['transform_type'] == TransformType.StageIn:
            if 'stagein_transformer' not in self.plugins:
                raise AgentPluginError('Plugin stagein_transformer is required')
            return self.plugins['stagein_transformer'](transform, input_collection, output_collection, contents)
        if transform['transform_type'] == TransformType.ActiveLearning:
            if 'activelearning_transformer' not in self.plugins:
                raise AgentPluginError('Plugin activelearning_transformer is required')
            return self.plugins['activelearning_transformer'](transform, input_collection, output_collection, contents)
        if transform['transform_type'] == TransformType.HyperParameterOpt:
            if 'hyperparameteropt_transformer' not in self.plugins:
                raise AgentPluginError('Plugin hyperparameteropt_transformer is required')
            return self.plugins['hyperparameteropt_transformer'](transform, input_collection, output_collection, contents)

        return []
Example #4
0
    def start_notifier(self):
        if 'notifier' not in self.plugins:
            raise AgentPluginError('Plugin notifier is required')
        self.notifier = self.plugins['notifier']

        self.logger.info("Starting notifier: %s" % self.notifier)
        self.notifier.set_request_queue(self.message_queue)
        self.notifier.start()
Example #5
0
 def get_collections(self, scope, name, req=None):
     if (req and req['request_metadata']
             and 'is_pseudo_input' in req['request_metadata']  # noqa: W503
             and req['request_metadata']['is_pseudo_input']
             == True):  # noqa: W503
         collection = {
             'scope': scope,
             'name': name,
             'total_files': 1,
             'bytes': 0,
             'coll_type': CollectionType.PseudoDataset
         }
         return [collection]
     else:
         if 'collection_lister' not in self.plugins:
             raise AgentPluginError('Plugin collection_lister is required')
         return self.plugins['collection_lister'](scope, name)
Example #6
0
 def get_contents(self, scope, name):
     if 'contents_lister' not in self.plugins:
         raise AgentPluginError('Plugin contents_lister is required')
     return self.plugins['contents_lister'](scope, name)
Example #7
0
 def get_collection_metadata(self, scope, name):
     if 'collection_metadata_reader' not in self.plugins:
         raise AgentPluginError(
             'Plugin collection_metadata_reader is required')
     return self.plugins['collection_metadata_reader'](scope, name)