def _run_toplevel(config, config_file, work_dir, parallel,
fc_dir=None, run_info_yaml=None):
"""
Run toplevel analysis, processing a set of input files.
config_file -- Main YAML configuration file with system parameters
fc_dir -- Directory of fastq files to process
run_info_yaml -- YAML configuration file specifying inputs to process
"""
parallel = log.create_base_logger(config, parallel)
log.setup_local_logging(config, parallel)
fastq_dir, galaxy_dir, config_dir = _get_full_paths(flowcell.get_fastq_dir(fc_dir)
if fc_dir else None,
config, config_file)
config_file = os.path.join(config_dir, os.path.basename(config_file))
dirs = {"fastq": fastq_dir, "galaxy": galaxy_dir,
"work": work_dir, "flowcell": fc_dir, "config": config_dir}
samples = run_info.organize(dirs, config, run_info_yaml)
pipelines = _pair_lanes_with_pipelines(samples)
final = []
with utils.curdir_tmpdir() as tmpdir:
tempfile.tempdir = tmpdir
for pipeline, pipeline_items in pipelines.items():
pipeline_items = _add_provenance(pipeline_items, dirs, parallel, config)
versioncheck.testall(pipeline_items)
for xs in pipeline.run(config, config_file, parallel, dirs, pipeline_items):
if len(xs) == 1:
upload.from_sample(xs[0])
final.append(xs[0])
def setup_directories(work_dir, fc_dir, config, config_file):
fastq_dir, galaxy_dir, config_dir = _get_full_paths(flowcell.get_fastq_dir(fc_dir)
if fc_dir else None,
config, config_file)
# check default install for tool data if not found locally
if not os.path.exists(os.path.join(galaxy_dir, "tool-data")):
_, config_file = config_utils.load_system_config(work_dir=work_dir)
if os.path.exists(os.path.join(os.path.dirname(config_file), "tool-data")):
galaxy_dir = os.path.dirname(config_file)
return {"fastq": fastq_dir, "galaxy": galaxy_dir,
"work": work_dir, "flowcell": fc_dir, "config": config_dir}
def setup_directories(work_dir, fc_dir, config, config_file):
fastq_dir, galaxy_dir, config_dir = _get_full_paths(flowcell.get_fastq_dir(fc_dir)
if fc_dir else None,
config, config_file)
# check default install for tool data if not found locally
if not os.path.exists(os.path.join(galaxy_dir, "tool-data")):
_, config_file = config_utils.load_system_config(work_dir=work_dir, allow_missing=True)
if config_file and os.path.exists(os.path.join(os.path.dirname(config_file), "tool-data")):
galaxy_dir = os.path.dirname(config_file)
return {"fastq": fastq_dir, "galaxy": galaxy_dir,
"work": work_dir, "flowcell": fc_dir, "config": config_dir}
def setup_directories(work_dir, fc_dir, config, config_file):
fastq_dir, galaxy_dir, config_dir = _get_full_paths(
flowcell.get_fastq_dir(fc_dir) if fc_dir else None, config,
config_file)
return {
"fastq": fastq_dir,
"galaxy": galaxy_dir,
"work": work_dir,
"flowcell": fc_dir,
"config": config_dir
}
def _normalize_files(item, fc_dir):
"""Ensure the files argument is a list of absolute file names.
Handles BAM, single and paired end fastq.
"""
files = item.get("files")
if files:
if isinstance(files, basestring):
files = [files]
fastq_dir = flowcell.get_fastq_dir(fc_dir) if fc_dir else os.getcwd()
files = [_file_to_abs(x, [os.getcwd(), fc_dir, fastq_dir]) for x in files]
_sanity_check_files(item, files)
item["files"] = files
return item
def _normalize_files(item, fc_dir=None):
"""Ensure the files argument is a list of absolute file names.
Handles BAM, single and paired end fastq.
"""
files = item.get("files")
if files:
if isinstance(files, basestring):
files = [files]
fastq_dir = flowcell.get_fastq_dir(fc_dir) if fc_dir else os.getcwd()
files = [_file_to_abs(x, [os.getcwd(), fc_dir, fastq_dir]) for x in files]
files = [x for x in files if x]
_sanity_check_files(item, files)
item["files"] = files
return item
def _normalize_files(item, fc_dir):
"""Ensure the files argument is a list of absolute file names.
Handles BAM, single and paired end fastq.
"""
files = item.get("files")
if files:
if isinstance(files, basestring):
files = [files]
if fc_dir:
fastq_dir = flowcell.get_fastq_dir(fc_dir)
else:
fastq_dir = os.getcwd()
files = [x if os.path.isabs(x) else os.path.normpath(os.path.join(fastq_dir, x))
for x in files]
_sanity_check_files(item, files)
item["files"] = files
return item
def main(config_file, fc_dir):
work_dir = os.getcwd()
config = load_config(config_file)
galaxy_api = GalaxyApiAccess(config["galaxy_url"], config["galaxy_api_key"])
fc_name, fc_date = flowcell.parse_dirname(fc_dir)
run_info = galaxy_api.run_details(fc_name)
fastq_dir = flowcell.get_fastq_dir(fc_dir)
if config["algorithm"]["num_cores"] > 1:
pool = Pool(config["algorithm"]["num_cores"])
try:
pool.map(
_process_wrapper, ((i, fastq_dir, fc_name, fc_date, config, config_file) for i in run_info["details"])
)
except:
pool.terminate()
raise
else:
map(_process_wrapper, ((i, fastq_dir, fc_name, fc_date, config, config_file) for i in run_info["details"]))
def main(config_file, fc_dir):
work_dir = os.getcwd()
config = load_config(config_file)
galaxy_api = GalaxyApiAccess(config['galaxy_url'],
config['galaxy_api_key'])
fc_name, fc_date = flowcell.parse_dirname(fc_dir)
run_info = galaxy_api.run_details(fc_name)
fastq_dir = flowcell.get_fastq_dir(fc_dir)
if config["algorithm"]["num_cores"] > 1:
pool = Pool(config["algorithm"]["num_cores"])
try:
pool.map(_process_wrapper,
((i, fastq_dir, fc_name, fc_date, config, config_file)
for i in run_info["details"]))
except:
pool.terminate()
raise
else:
map(_process_wrapper,
((i, fastq_dir, fc_name, fc_date, config, config_file)
for i in run_info["details"]))
def setup_directories(work_dir, fc_dir, config, config_file):
fastq_dir, galaxy_dir, config_dir = _get_full_paths(flowcell.get_fastq_dir(fc_dir)
if fc_dir else None,
config, config_file)
return {"fastq": fastq_dir, "galaxy": galaxy_dir,
"work": work_dir, "flowcell": fc_dir, "config": config_dir}
