def run(self, trans, tmp_dir, kwd): tmp_dir = os.path.abspath(tmp_dir) lane = 1 fastq_dir, fastq_files, qual_format = self._prepare_fastq( trans, lane, tmp_dir, kwd) config_file, config = self._prepare_config(trans, lane, fastq_files, qual_format, tmp_dir, kwd) dirs = { "work": tmp_dir, "config": os.path.dirname(self._process_config_file) } send_data = { "fc_dir": fastq_dir, "run_yaml": config_file, "directory": "%s_%s" % (config["fc_date"], config["fc_name"]) } with open(self._process_config_file) as in_handle: config = yaml.load(in_handle) with utils.chdir(tmp_dir): runner = messaging.runner("bcbio.distributed.tasks", dirs, config, self._process_config_file, wait=False) runner("analyze_and_upload", [[send_data]])
def finished_message(fn_name, run_module, directory, files_to_copy, config, config_file, pushed=False): """Wait for messages with the give tag, passing on to the supplied handler. """ logger2.debug("Calling remote function: %s" % fn_name) user = getpass.getuser() hostname = socket.gethostbyaddr(socket.gethostname())[0] data = dict( machine_type='illumina', hostname=hostname, user=user, directory=directory, to_copy=files_to_copy ) dirs = {"work": os.getcwd(), "config": os.path.dirname(config_file)} runner = messaging.runner(run_module, dirs, config, config_file, wait=False) if pushed: config["directory"] = directory runner(fn_name, [[config]]) else: runner(fn_name, [[data]])
def queue_report(fc_date, fc_name, run_info_yaml, dirs, config, config_file): if "gdocs_upload" not in config: return False runner = messaging.runner("bcbio.distributed.google_tasks", {"work": os.getcwd(),"config": os.path.dirname(config_file)}, config, config_file, wait=False) runner("create_report_on_gdocs",[[fc_date,fc_name,run_info_yaml,dirs,config]]) return True
def _run_parallel(fn_name, items, dirs, config): """Process a supplied function: single, multi-processor or distributed. """ parallel = config["algorithm"]["num_cores"] if str(parallel).lower() == "messaging": runner = messaging.runner(dirs, config) return runner(fn_name, items) else: out = [] fn = globals()[fn_name] with utils.cpmap(int(parallel)) as cpmap: for data in cpmap(fn, items): if data: out.extend(data) return out
def queue_report(fc_date, fc_name, run_info_yaml, dirs, config, config_file): if "gdocs_upload" not in config: return False runner = messaging.runner("bcbio.distributed.google_tasks", \ {"work": os.getcwd(), \ "config": os.path.dirname(config_file)}, \ config, \ config_file, \ wait=False) runner("create_report_on_gdocs", [[fc_date, fc_name, run_info_yaml, dirs, config]]) return True