def _run_toplevel(config, config_file, work_dir, parallel, fc_dir=None, run_info_yaml=None): """ Run toplevel analysis, processing a set of input files. config_file -- Main YAML configuration file with system parameters fc_dir -- Directory of fastq files to process run_info_yaml -- YAML configuration file specifying inputs to process """ parallel = log.create_base_logger(config, parallel) log.setup_local_logging(config, parallel) fastq_dir, galaxy_dir, config_dir = _get_full_paths(flowcell.get_fastq_dir(fc_dir) if fc_dir else None, config, config_file) config_file = os.path.join(config_dir, os.path.basename(config_file)) dirs = {"fastq": fastq_dir, "galaxy": galaxy_dir, "work": work_dir, "flowcell": fc_dir, "config": config_dir} samples = run_info.organize(dirs, config, run_info_yaml) pipelines = _pair_lanes_with_pipelines(samples) final = [] with utils.curdir_tmpdir() as tmpdir: tempfile.tempdir = tmpdir for pipeline, pipeline_items in pipelines.items(): pipeline_items = _add_provenance(pipeline_items, dirs, parallel, config) versioncheck.testall(pipeline_items) for xs in pipeline.run(config, config_file, parallel, dirs, pipeline_items): if len(xs) == 1: upload.from_sample(xs[0]) final.append(xs[0])
def setup_directories(work_dir, fc_dir, config, config_file): fastq_dir, galaxy_dir, config_dir = _get_full_paths(flowcell.get_fastq_dir(fc_dir) if fc_dir else None, config, config_file) # check default install for tool data if not found locally if not os.path.exists(os.path.join(galaxy_dir, "tool-data")): _, config_file = config_utils.load_system_config(work_dir=work_dir) if os.path.exists(os.path.join(os.path.dirname(config_file), "tool-data")): galaxy_dir = os.path.dirname(config_file) return {"fastq": fastq_dir, "galaxy": galaxy_dir, "work": work_dir, "flowcell": fc_dir, "config": config_dir}
def setup_directories(work_dir, fc_dir, config, config_file): fastq_dir, galaxy_dir, config_dir = _get_full_paths(flowcell.get_fastq_dir(fc_dir) if fc_dir else None, config, config_file) # check default install for tool data if not found locally if not os.path.exists(os.path.join(galaxy_dir, "tool-data")): _, config_file = config_utils.load_system_config(work_dir=work_dir, allow_missing=True) if config_file and os.path.exists(os.path.join(os.path.dirname(config_file), "tool-data")): galaxy_dir = os.path.dirname(config_file) return {"fastq": fastq_dir, "galaxy": galaxy_dir, "work": work_dir, "flowcell": fc_dir, "config": config_dir}
def setup_directories(work_dir, fc_dir, config, config_file): fastq_dir, galaxy_dir, config_dir = _get_full_paths( flowcell.get_fastq_dir(fc_dir) if fc_dir else None, config, config_file) return { "fastq": fastq_dir, "galaxy": galaxy_dir, "work": work_dir, "flowcell": fc_dir, "config": config_dir }
def _normalize_files(item, fc_dir): """Ensure the files argument is a list of absolute file names. Handles BAM, single and paired end fastq. """ files = item.get("files") if files: if isinstance(files, basestring): files = [files] fastq_dir = flowcell.get_fastq_dir(fc_dir) if fc_dir else os.getcwd() files = [_file_to_abs(x, [os.getcwd(), fc_dir, fastq_dir]) for x in files] _sanity_check_files(item, files) item["files"] = files return item
def _normalize_files(item, fc_dir=None): """Ensure the files argument is a list of absolute file names. Handles BAM, single and paired end fastq. """ files = item.get("files") if files: if isinstance(files, basestring): files = [files] fastq_dir = flowcell.get_fastq_dir(fc_dir) if fc_dir else os.getcwd() files = [_file_to_abs(x, [os.getcwd(), fc_dir, fastq_dir]) for x in files] files = [x for x in files if x] _sanity_check_files(item, files) item["files"] = files return item
def _normalize_files(item, fc_dir): """Ensure the files argument is a list of absolute file names. Handles BAM, single and paired end fastq. """ files = item.get("files") if files: if isinstance(files, basestring): files = [files] if fc_dir: fastq_dir = flowcell.get_fastq_dir(fc_dir) else: fastq_dir = os.getcwd() files = [x if os.path.isabs(x) else os.path.normpath(os.path.join(fastq_dir, x)) for x in files] _sanity_check_files(item, files) item["files"] = files return item
def main(config_file, fc_dir): work_dir = os.getcwd() config = load_config(config_file) galaxy_api = GalaxyApiAccess(config["galaxy_url"], config["galaxy_api_key"]) fc_name, fc_date = flowcell.parse_dirname(fc_dir) run_info = galaxy_api.run_details(fc_name) fastq_dir = flowcell.get_fastq_dir(fc_dir) if config["algorithm"]["num_cores"] > 1: pool = Pool(config["algorithm"]["num_cores"]) try: pool.map( _process_wrapper, ((i, fastq_dir, fc_name, fc_date, config, config_file) for i in run_info["details"]) ) except: pool.terminate() raise else: map(_process_wrapper, ((i, fastq_dir, fc_name, fc_date, config, config_file) for i in run_info["details"]))
def main(config_file, fc_dir): work_dir = os.getcwd() config = load_config(config_file) galaxy_api = GalaxyApiAccess(config['galaxy_url'], config['galaxy_api_key']) fc_name, fc_date = flowcell.parse_dirname(fc_dir) run_info = galaxy_api.run_details(fc_name) fastq_dir = flowcell.get_fastq_dir(fc_dir) if config["algorithm"]["num_cores"] > 1: pool = Pool(config["algorithm"]["num_cores"]) try: pool.map(_process_wrapper, ((i, fastq_dir, fc_name, fc_date, config, config_file) for i in run_info["details"])) except: pool.terminate() raise else: map(_process_wrapper, ((i, fastq_dir, fc_name, fc_date, config, config_file) for i in run_info["details"]))
def setup_directories(work_dir, fc_dir, config, config_file): fastq_dir, galaxy_dir, config_dir = _get_full_paths(flowcell.get_fastq_dir(fc_dir) if fc_dir else None, config, config_file) return {"fastq": fastq_dir, "galaxy": galaxy_dir, "work": work_dir, "flowcell": fc_dir, "config": config_dir}