def submit_jobs(self, job_files): task = self.task pool = get_param(task.htcondor_pool) scheduler = get_param(task.htcondor_scheduler) # prepare objects for dumping intermediate submission data dump_freq = task.htcondor_dump_intermediate_submission_data() if dump_freq and not is_number(dump_freq): dump_freq = 50 # progress callback to inform the scheduler def progress_callback(i, job_ids): job_num = i + 1 job_id = job_ids[0] # set the job id early self.submission_data.jobs[job_num]["job_id"] = job_id # log a message every 25 jobs if job_num in (1, len(job_files)) or job_num % 25 == 0: task.publish_message("submitted {}/{} job(s)".format( job_num, len(job_files))) # dump intermediate submission data with a certain frequency if dump_freq and job_num % dump_freq == 0: self.dump_submission_data() return self.job_manager.submit_batch(job_files, pool=pool, scheduler=scheduler, retries=3, threads=task.threads, callback=progress_callback)
def log(fn, opts, task, *args, **kwargs): """ log() Wraps a bound method of a task and redirects output of both stdin and stdout to the file defined by the tasks's *log* parameter. It its value is ``"-"`` or *None*, the output is not redirected. """ _task = get_task(task) log = get_param(_task.log_file, _task.default_log_file) if log == "-" or not log: return fn(task, *args, **kwargs) else: # use the local target functionality to create the parent directory LocalFileTarget(log).parent.touch() with open(log, "a", 1) as f: tee = TeeStream(f, sys.__stdout__) sys.stdout = tee sys.stderr = tee try: ret = fn(task, *args, **kwargs) except Exception as e: traceback.print_exc(file=tee) raise e finally: sys.stdout = sys.__stdout__ sys.stderr = sys.__stderr__ tee.flush() return ret
def _get_job_kwargs(self, name): attr = "{}_job_kwargs_{}".format(self.workflow_type, name) kwargs = getattr(self.task, attr, None) if kwargs is None: attr = "{}_job_kwargs".format(self.workflow_type) kwargs = getattr(self.task, attr) # when kwargs is not a dict, it is assumed to be a list whose # elements represent task attributes that are stored without the workflow type prefix if not isinstance(kwargs, dict): _kwargs = {} for param_name in kwargs: kwarg_name = param_name if param_name.startswith(self.workflow_type + "_"): kwarg_name = param_name[len(self.workflow_type) + 1:] _kwargs[kwarg_name] = get_param(getattr(self.task, param_name)) kwargs = _kwargs return kwargs