def name_to_config(template):
"""Read template file into a dictionary to use as base for all samples.
Handles well-known template names, pulled from GitHub repository and local
files.
"""
if template.startswith(utils.SUPPORTED_REMOTES):
with utils.s3_handle(template) as in_handle:
config = yaml.load(in_handle)
txt_config = None
elif os.path.isfile(template):
with open(template) as in_handle:
txt_config = in_handle.read()
with open(template) as in_handle:
config = yaml.load(in_handle)
else:
base_url = "https://raw.github.com/chapmanb/bcbio-nextgen/master/config/templates/%s.yaml"
try:
with contextlib.closing(urllib2.urlopen(base_url %
template)) as in_handle:
txt_config = in_handle.read()
with contextlib.closing(urllib2.urlopen(base_url %
template)) as in_handle:
config = yaml.load(in_handle)
except (urllib2.HTTPError, urllib2.URLError):
raise ValueError(
"Could not find template '%s' locally or in standard templates on GitHub"
% template)
return config, txt_config
def name_to_config(template):
"""Read template file into a dictionary to use as base for all samples.
Handles well-known template names, pulled from GitHub repository and local
files.
"""
if template.startswith(utils.SUPPORTED_REMOTES):
with utils.s3_handle(template) as in_handle:
config = yaml.load(in_handle)
txt_config = None
elif os.path.isfile(template):
with open(template) as in_handle:
txt_config = in_handle.read()
with open(template) as in_handle:
config = yaml.load(in_handle)
else:
base_url = "https://raw.github.com/chapmanb/bcbio-nextgen/master/config/templates/%s.yaml"
try:
with contextlib.closing(urllib2.urlopen(base_url % template)) as in_handle:
txt_config = in_handle.read()
with contextlib.closing(urllib2.urlopen(base_url % template)) as in_handle:
config = yaml.load(in_handle)
except (urllib2.HTTPError, urllib2.URLError):
raise ValueError("Could not find template '%s' locally or in standard templates on GitHub"
% template)
return config, txt_config
def open_fastq(in_file):
""" open a fastq file, using gzip if it is gzipped
"""
if in_file.startswith("s3:"):
return utils.s3_handle(in_file)
_, ext = os.path.splitext(in_file)
if ext == ".gz":
return gzip.open(in_file, 'rb')
if ext in [".fastq", ".fq"]:
return open(in_file, 'r')
def open_fastq(in_file):
""" open a fastq file, using gzip if it is gzipped
"""
if in_file.startswith("s3:"):
return utils.s3_handle(in_file)
_, ext = os.path.splitext(in_file)
if ext == ".gz":
return gzip.open(in_file, 'rb')
if ext in [".fastq", ".fq"]:
return open(in_file, 'r')
def _pname_and_metadata(in_file):
"""Retrieve metadata and project name from the input metadata CSV file.
Uses the input file name for the project name and
For back compatibility, accepts the project name as an input, providing no metadata.
"""
if os.path.isfile(in_file):
with open(in_file) as in_handle:
md, global_vars = _parse_metadata(in_handle)
base = os.path.splitext(os.path.basename(in_file))[0]
elif in_file.startswith(utils.SUPPORTED_REMOTES):
with utils.s3_handle(in_file) as in_handle:
md, global_vars = _parse_metadata(in_handle)
base = os.path.splitext(os.path.basename(in_file))[0]
else:
base, md, global_vars = _safe_name(in_file), {}, {}
return _safe_name(base), md, global_vars
def _pname_and_metadata(in_file):
"""Retrieve metadata and project name from the input metadata CSV file.
Uses the input file name for the project name and
For back compatibility, accepts the project name as an input, providing no metadata.
"""
if os.path.isfile(in_file):
with open(in_file) as in_handle:
md, global_vars = _parse_metadata(in_handle)
base = os.path.splitext(os.path.basename(in_file))[0]
elif in_file.startswith(utils.SUPPORTED_REMOTES):
with utils.s3_handle(in_file) as in_handle:
md, global_vars = _parse_metadata(in_handle)
base = os.path.splitext(os.path.basename(in_file))[0]
else:
base, md, global_vars = _safe_name(in_file), {}, {}
return _safe_name(base), md, global_vars
def load_s3(sample_config):
"""Move a sample configuration locally, providing remote upload.
"""
with utils.s3_handle(sample_config) as in_handle:
config = yaml.load(in_handle)
bucket, key = utils.s3_bucket_key(sample_config)
config["upload"] = {"method": "s3",
"dir": os.path.join(os.pardir, "final"),
"bucket": bucket,
"folder": os.path.join(os.path.dirname(key), "final")}
if not os.access(os.pardir, os.W_OK | os.X_OK):
raise IOError("Cannot write to the parent directory of work directory %s\n"
"bcbio wants to store prepared uploaded files to %s\n"
"We recommend structuring your project in a project specific directory structure\n"
"with a specific work directory (mkdir -p your-project/work && cd your-project/work)."
% (os.getcwd(), os.path.join(os.pardir, "final")))
config = _add_jar_resources(config, bucket, key)
out_file = os.path.join(utils.safe_makedir(os.path.join(os.getcwd(), "config")),
os.path.basename(key))
with open(out_file, "w") as out_handle:
yaml.dump(config, out_handle, default_flow_style=False, allow_unicode=False)
return out_file