def __init__(self, name, active=True, batch_args=None, submission_args=None): ''' Parameters ---------- name: str name of the step active: bool, optional whether the step should be processed batch_args: tmlib.workflow.args.BatchArguments, optional batch arguments submission_args: tmlib.workflow.args.SubmissionArguments, optional submission arguments Raises ------ WorkflowDescriptionError when a provided argument is not a valid argument for the given step ''' self.name = str(name) self.fullname, self.help = get_step_information(name) self.active = active BatchArgs, SubmissionArgs = get_step_args(name) if batch_args is None: self.batch_args = BatchArgs() else: self.batch_args = batch_args if submission_args is None: self.submission_args = SubmissionArgs() else: self.submission_args = submission_args
def __init__(self, name, active=True, batch_args=None, submission_args=None): ''' Parameters ---------- name: str name of the step active: bool, optional whether the step should be processed batch_args: tmlib.workflow.args.BatchArguments, optional batch arguments submission_args: tmlib.workflow.args.SubmissionArguments, optional submission arguments Raises ------ WorkflowDescriptionError when a provided argument is not a valid argument for the given step ''' self.name = str(name) self.fullname, self.help = get_step_information(name) self.active = active BatchArgs, SubmissionArgs = get_step_args(name) if batch_args is None: self.batch_args = BatchArgs() else: self.batch_args = batch_args if submission_args is None: self.submission_args = SubmissionArgs() else: self.submission_args = submission_args
def __init__(self, type, name, mode='sequential', active=True, steps=None): ''' Parameters ---------- type: str name of the workflow type name: str name of the stage mode: str, optional mode of workflow stage submission, i.e. whether steps are submitted simultaneously or one after another (options: ``{"sequential", "parallel"}``) active: bool, optional whether the stage should be processed steps: List[dict] description of steps in form of key-value pairs Raises ------ TypeError when `name` or `steps` have the wrong type ''' self.type = type self.dependencies = get_workflow_dependencies(self.type) self.name = str(name) self.mode = mode self.active = active if self.mode not in {'parallel', 'sequential'}: raise ValueError( 'Attribute "mode" must be either "parallel" or "sequential"' ) self.steps = list() if steps is not None: for step in steps: BatchArgs, SubmissionArgs = get_step_args(step['name']) batch_arg_values = { a['name']: a['value'] for a in step['batch_args'] } batch_args = BatchArgs(**batch_arg_values) submission_arg_values = { a['name']: a['value'] for a in step['submission_args'] } submission_args = SubmissionArgs(**submission_arg_values) # NOTE: not every step has extra arguments self.add_step( WorkflowStepDescription( step['name'], step.get('active', True), batch_args, submission_args ) ) else: for name in self.dependencies.STEPS_PER_STAGE[self.name]: self.add_step( WorkflowStepDescription(name, True) )
def __init__(self, type, name, mode='sequential', active=True, steps=None): ''' Parameters ---------- type: str name of the workflow type name: str name of the stage mode: str, optional mode of workflow stage submission, i.e. whether steps are submitted simultaneously or one after another (options: ``{"sequential", "parallel"}``) active: bool, optional whether the stage should be processed steps: List[dict] description of steps in form of key-value pairs Raises ------ TypeError when `name` or `steps` have the wrong type ''' self.type = type self.dependencies = get_workflow_dependencies(self.type) self.name = str(name) self.mode = mode self.active = active if self.mode not in {'parallel', 'sequential'}: raise ValueError( 'Attribute "mode" must be either "parallel" or "sequential"' ) self.steps = [] if steps is not None: for step in steps: BatchArgs, SubmissionArgs = get_step_args(step['name']) batch_arg_values = { a['name']: a['value'] for a in step['batch_args'] } batch_args = BatchArgs(**batch_arg_values) submission_arg_values = { a['name']: a['value'] for a in step['submission_args'] } submission_args = SubmissionArgs(**submission_arg_values) # NOTE: not every step has extra arguments self.add_step( WorkflowStepDescription( step['name'], step.get('active', True), batch_args, submission_args ) ) else: for name in self.dependencies.STEPS_PER_STAGE[self.name]: self.add_step( WorkflowStepDescription(name, True) )
def run_jobs(experiment_id): '''Runs one or more jobs of the current project with pipeline and module descriptions provided by the UI. This requires the pipeline and module descriptions to be saved to *pipe* and *handles* files, respectively. ''' logger.info('submit jobs for jterator project of experiment %d', experiment_id) data = json.loads(request.data) job_ids = map(int, data['job_ids']) project = yaml.load(data['project']) pipeline_description = PipelineDescription( **project['pipe']['description']) handles_descriptions = { h['name']: HandleDescriptions(**h['description']) for h in project['handles'] } jt = ImageAnalysisPipelineEngine( experiment_id, pipeline_description=pipeline_description, handles_descriptions=handles_descriptions, ) # 1. Delete figures and logs from previous submission # since they are not tracked per submission. jt.remove_previous_pipeline_output() # TODO: remove figure files of previous runs!! # 2. Build job descriptions channel_names = [ ch.name for ch in jt.project.pipe.description.input.channels ] job_descriptions = list() with tm.utils.ExperimentSession(experiment_id) as session: sites = session.query(tm.Site.id).\ order_by(tm.Site.id).\ all() for j in job_ids: site_id = sites[j].id image_file_count = session.query(tm.ChannelImageFile.id).\ join(tm.Channel).\ filter(tm.Channel.name.in_(channel_names)).\ filter(tm.ChannelImageFile.site_id == site_id).\ count() if image_file_count == 0: raise MalformedRequestError( 'No images found for job ID {j}.'.format(j=j)) job_descriptions.append({'site_id': site_id, 'plot': True}) with tm.utils.MainSession() as session: submission = tm.Submission(experiment_id=experiment_id, program='jtui', user_id=current_identity.id) session.add(submission) session.flush() SubmitArgs = get_step_args('jterator')[1] submit_args = SubmitArgs() job_collection = jt.create_debug_run_phase(submission.id) jobs = jt.create_debug_run_jobs(user_name=current_identity.name, batches=job_descriptions, job_collection=job_collection, verbosity=2, duration=submit_args.duration, memory=submit_args.memory, cores=submit_args.cores) # 3. Store jobs in session gc3pie.store_task(jobs) # session.remove(data['previousSubmissionId']) gc3pie.submit_task(jobs) return jsonify(submission_id=jobs.submission_id)
def __init__(cls, clsname, bases, attrs): super(_CliMeta, cls).__init__(clsname, bases, attrs) if '__abstract__' in vars(cls).keys(): return pkg_name = '.'.join(cls.__module__.split('.')[:-1]) pkg = importlib.import_module(pkg_name) cls.__doc__ = pkg.__description__ cls.__logo__ = pkg.__logo__ parser = argparse.ArgumentParser() parser.description = pkg.__description__ parser.version = __version__ # The parser for each step receives at least two arguments, which are # passed to the corresponding API class. parser.add_argument( 'experiment_id', type=int, help='ID of the experiment that should be processed') parser.add_argument('--verbosity', '-v', action='count', default=0, help='increase logging verbosity') # Extra arguments are added to the main parser as well because they # also need to be parssed to the constructor of the API class. step_name = cls.__name__.lower() BatchArgs, SubmissionArgs = get_step_args(step_name) subparsers = parser.add_subparsers(dest='method', help='methods') subparsers.required = True # flags = collections.defaultdict(list) for attr_name in dir(cls): if attr_name.startswith('__'): continue attr_value = getattr(cls, attr_name) # The climethod decorator provides argument descriptions via # the "args" attribute of the decoreated method. # These arguments are added to the method-specific subparser. if isinstance(attr_value, types.MethodType): if getattr(attr_value, 'is_climethod', False): method_parser = subparsers.add_parser(attr_name, help=attr_value.help) method_parser.description = attr_value.help for arg in attr_value.args.iterargs(): arg.add_to_argparser(method_parser) # if arg.flag is not None: # flags[attr_name].append(arg.flag) # if arg.short_flag is not None: # flags[attr_name].append(arg.short_flag) # The "init" and "submit" methods require additional arguments # that also need to be accessible outside the scope of the # command line interface. Therefore, they are handled separately. # Each workflow step must implement BatchArguments and # SubmissionArguments and register them using the batch_args and # submission_args decorator, respectively. # These arguments are added to the corresponding method-specific # subparser as a separate group to highlight that they represent a # different type of argument. def add_step_specific_method_args(step_name, method_name, args_class): method_parser = subparsers.choices[method_name] parser_group = method_parser.add_argument_group( 'step-specific arguments') for arg in args_class.iterargs(): arg.add_to_argparser(parser_group) # if arg.flag is not None: # flags[attr_name].append(arg.flag) # if arg.short_flag is not None: # flags[attr_name].append(arg.short_flag) add_step_specific_method_args(step_name, 'init', BatchArgs) setattr(cls, '_batch_args_class', BatchArgs) add_step_specific_method_args(step_name, 'submit', SubmissionArgs) setattr(cls, '_submission_args_class', SubmissionArgs) api = get_step_api(step_name) setattr(cls, '_api_class', api) setattr(cls, '_parser', parser)
def run_jobs(experiment_id): '''Runs one or more jobs of the current project with pipeline and module descriptions provided by the UI. This requires the pipeline and module descriptions to be saved to *pipe* and *handles* files, respectively. ''' logger.info( 'submit jobs for jterator project of experiment %d', experiment_id ) data = json.loads(request.data) job_ids = map(int, data['job_ids']) project = yaml.load(data['project']) pipeline_description = PipelineDescription(**project['pipe']['description']) handles_descriptions = { h['name']: HandleDescriptions(**h['description']) for h in project['handles'] } jt = ImageAnalysisPipelineEngine( experiment_id, pipeline_description=pipeline_description, handles_descriptions=handles_descriptions, ) # 1. Delete figures and logs from previous submission # since they are not tracked per submission. jt.remove_previous_pipeline_output() # TODO: remove figure files of previous runs!! # 2. Build job descriptions channel_names = [ ch.name for ch in jt.project.pipe.description.input.channels ] object_names = [ ob.name for ob in jt.project.pipe.description.input.objects ] job_descriptions = list() with tm.utils.ExperimentSession(experiment_id) as session: sites = session.query(tm.Site.id).\ order_by(tm.Site.id).\ all() for j in job_ids: site_id = sites[j-1].id # user-input is expected between [1..] image_file_count = 0 image_file_count += session.query(tm.ChannelImageFile.id).\ join(tm.Channel).\ filter(tm.Channel.name.in_(channel_names)).\ filter(tm.ChannelImageFile.site_id == site_id).\ count() image_file_count += session.query(tm.ChannelImageFile.id).\ join(tm.Site).\ join(tm.Well).\ join(tm.Plate).\ join(tm.Experiment).\ join(tm.MapobjectType).\ filter(tm.MapobjectType.name.in_(object_names)).\ filter(tm.ChannelImageFile.site_id == site_id).\ count() if image_file_count == 0: raise MalformedRequestError( 'No images found for job ID {j}.' .format(j=j)) job_descriptions.append({'site_id': site_id, 'plot': True}) with tm.utils.MainSession() as session: submission = tm.Submission( experiment_id=experiment_id, program='jtui', user_id=current_identity.id ) session.add(submission) session.flush() SubmitArgs = get_step_args('jterator')[1] submit_args = SubmitArgs() job_collection = jt.create_debug_run_phase(submission.id) jobs = jt.create_debug_run_jobs( user_name=current_identity.name, batches=job_descriptions, job_collection=job_collection, verbosity=2, duration=submit_args.duration, memory=submit_args.memory, cores=submit_args.cores ) # 3. Store jobs in session gc3pie.store_task(jobs) # session.remove(data['previousSubmissionId']) gc3pie.submit_task(jobs) return jsonify(submission_id=jobs.submission_id)