def _run_toplevel(config, config_file, work_dir, parallel, fc_dir=None, run_info_yaml=None): """ Run toplevel analysis, processing a set of input files. config_file -- Main YAML configuration file with system parameters fc_dir -- Directory of fastq files to process run_info_yaml -- YAML configuration file specifying inputs to process """ parallel = log.create_base_logger(config, parallel) log.setup_local_logging(config, parallel) dirs = setup_directories(work_dir, fc_dir, config, config_file) config_file = os.path.join(dirs["config"], os.path.basename(config_file)) samples = run_info.organize(dirs, config, run_info_yaml) pipelines = _pair_samples_with_pipelines(samples) final = [] with tx_tmpdir(config) as tmpdir: tempfile.tempdir = tmpdir for pipeline, pipeline_items in pipelines.items(): pipeline_items = _add_provenance(pipeline_items, dirs, parallel, config) versioncheck.testall(pipeline_items) for xs in pipeline.run(config, config_file, parallel, dirs, pipeline_items): if len(xs) == 1: upload.from_sample(xs[0]) final.append(xs[0])
def _run_toplevel(config, config_file, work_dir, parallel, fc_dir=None, run_info_yaml=None): """ Run toplevel analysis, processing a set of input files. config_file -- Main YAML configuration file with system parameters fc_dir -- Directory of fastq files to process run_info_yaml -- YAML configuration file specifying inputs to process """ parallel = log.create_base_logger(config, parallel) log.setup_local_logging(config, parallel) fastq_dir, galaxy_dir, config_dir = _get_full_paths(get_fastq_dir(fc_dir) if fc_dir else None, config, config_file) config_file = os.path.join(config_dir, os.path.basename(config_file)) dirs = {"fastq": fastq_dir, "galaxy": galaxy_dir, "work": work_dir, "flowcell": fc_dir, "config": config_dir} samples = run_info.organize(dirs, config, run_info_yaml) pipelines = _pair_lanes_with_pipelines(samples) final = [] with utils.curdir_tmpdir() as tmpdir: tempfile.tempdir = tmpdir for pipeline, pipeline_items in pipelines.items(): pipeline_items = _add_provenance(pipeline_items, dirs, parallel, config) versioncheck.testall(pipeline_items) for xs in pipeline.run(config, config_file, parallel, dirs, pipeline_items): if len(xs) == 1: upload.from_sample(xs[0]) final.append(xs[0])
def _run_toplevel(config, config_file, work_dir, parallel, fc_dir=None, run_info_yaml=None): """ Run toplevel analysis, processing a set of input files. config_file -- Main YAML configuration file with system parameters fc_dir -- Directory of fastq files to process run_info_yaml -- YAML configuration file specifying inputs to process """ parallel = log.create_base_logger(config, parallel) log.setup_local_logging(config, parallel) fastq_dir, galaxy_dir, config_dir = _get_full_paths(get_fastq_dir(fc_dir) if fc_dir else None, config, config_file) config_file = os.path.join(config_dir, os.path.basename(config_file)) dirs = {"fastq": fastq_dir, "galaxy": galaxy_dir, "work": work_dir, "flowcell": fc_dir, "config": config_dir} run_items = run_info.organize(dirs, config, run_info_yaml) run_parallel = parallel_runner(parallel, dirs, config, config_file) # process each flowcell lane lane_items = lane.process_all_lanes(run_items, run_parallel) pipelines = _pair_lanes_with_pipelines(lane_items) final = [] with utils.curdir_tmpdir() as tmpdir: tempfile.tempdir = tmpdir for pipeline, pipeline_items in pipelines.items(): pipeline_items = _add_provenance(pipeline_items, dirs, run_parallel, parallel, config) versioncheck.testall(pipeline_items) for xs in pipeline.run(config, config_file, run_parallel, parallel, dirs, pipeline_items): if len(xs) == 1: upload.from_sample(xs[0]) final.append(xs[0])
def run(args): """Run a CWL preparation pipeline. """ dirs, config, run_info_yaml = run_info.prep_system(args.sample_config, args.systemconfig) world = run_info.organize(dirs, config, run_info_yaml) create.from_world(world, run_info_yaml)
def run(args): """Run a CWL preparation pipeline. """ dirs, config, run_info_yaml = run_info.prep_system(args.sample_config, args.systemconfig) integrations = args.integrations if hasattr(args, "integrations") else {} world = run_info.organize(dirs, config, run_info_yaml, add_provenance=False, integrations=integrations) create.from_world(world, run_info_yaml)
def run(args): """Run a CWL preparation pipeline. """ dirs, config, run_info_yaml = run_info.prep_system(args.sample_config, args.systemconfig) integrations = args.integrations if hasattr(args, "integrations") else {} world = run_info.organize(dirs, config, run_info_yaml, is_cwl=True, integrations=integrations) create.from_world(world, run_info_yaml, integrations=integrations, add_container_tag=args.add_container_tag)
def _run_toplevel(config, config_file, work_dir, parallel, fc_dir=None, run_info_yaml=None, samples=None): """ Run toplevel analysis, processing a set of input files. config_file -- Main YAML configuration file with system parameters fc_dir -- Directory of fastq files to process run_info_yaml -- YAML configuration file specifying inputs to process samples -- Pre-processed samples, useful if run inside of docker containers. """ parallel = log.create_base_logger(config, parallel) log.setup_local_logging(config, parallel) dirs = run_info.setup_directories(work_dir, fc_dir, config, config_file) config_file = os.path.join(dirs["config"], os.path.basename(config_file)) if samples: dockerized = True else: dockerized = False samples = run_info.organize(dirs, config, run_info_yaml) pipelines = _pair_samples_with_pipelines(samples) final = [] with tx_tmpdir(config) as tmpdir: tempfile.tempdir = tmpdir for pipeline, pipeline_items in pipelines.items(): pipeline_items = _add_provenance(pipeline_items, dirs, parallel, config) if not dockerized: versioncheck.testall(pipeline_items) for xs in pipeline.run(config, config_file, parallel, dirs, pipeline_items): if len(xs) == 1: upload.from_sample(xs[0]) final.append(xs[0])
def run(args): """Run a CWL preparation pipeline. """ dirs, config, run_info_yaml = run_info.prep_system(args.sample_config, args.systemconfig) integrations = args.integrations if hasattr(args, "integrations") else {} world = run_info.organize(dirs, config, run_info_yaml, add_provenance=False, integrations=integrations) create.from_world(world, run_info_yaml, integrations=integrations)
def organize_samples(*args): return run_info.organize(*args)
def run(args): """Run a CWL preparation pipeline. """ dirs, config, run_info_yaml = run_info.prep_system(args.sample_config, args.systemconfig) world = run_info.organize(dirs, config, run_info_yaml, add_provenance=False) create.from_world(world, run_info_yaml)