def _remote_fetch(env, url, out_file=None, allow_fail=False):
"""Retrieve url using wget, performing download in a temporary directory.
Provides a central location to handle retrieval issues and avoid
using interrupted downloads.
"""
if out_file is None:
out_file = os.path.basename(url)
if not env.safe_exists(out_file):
orig_dir = env.safe_run_output("pwd").strip()
temp_ext = "/%s" % uuid.uuid3(uuid.NAMESPACE_URL,
str("file://%s/%s/%s/%s" %
(env.host, socket.gethostname(),
datetime.datetime.now().isoformat(), out_file)))
with _make_tmp_dir(ext=temp_ext) as tmp_dir:
with cd(tmp_dir):
with warn_only():
result = env.safe_run("wget --no-check-certificate -O %s '%s'" % (out_file, url))
if result.succeeded:
env.safe_run("mv %s %s" % (out_file, orig_dir))
elif allow_fail:
out_file = None
else:
raise IOError("Failure to retrieve remote file")
return out_file
def _remote_fetch(env, url, out_file=None, allow_fail=False, fix_fn=None):
"""Retrieve url using wget, performing download in a temporary directory.
Provides a central location to handle retrieval issues and avoid
using interrupted downloads.
"""
if out_file is None:
out_file = os.path.basename(url)
if not env.safe_exists(out_file):
orig_dir = env.safe_run_output("pwd").strip()
temp_ext = "/%s" % uuid.uuid3(
uuid.NAMESPACE_URL,
str("file://%s/%s/%s/%s" %
(env.host, socket.gethostname(),
datetime.datetime.now().isoformat(), out_file)))
with _make_tmp_dir(ext=temp_ext) as tmp_dir:
with cd(tmp_dir):
with warn_only():
result = env.safe_run(
"wget --no-check-certificate -O %s '%s'" %
(out_file, url))
if result.succeeded:
if fix_fn:
out_file = fix_fn(env, out_file)
env.safe_run("mv %s %s" % (out_file, orig_dir))
elif allow_fail:
out_file = None
else:
raise IOError("Failure to retrieve remote file")
return out_file
def _download_annotation_bundle(env, url, gid):
"""Download bundle of RNA-seq data from S3 biodata/annotation
"""
tarball = os.path.basename(url)
if not env.safe_exists(tarball):
with warn_only():
env.safe_run("wget %s" % url)
if env.safe_exists(tarball):
env.safe_run("xz -dc %s | tar -xvpf -" % tarball)
env.safe_run("rm -f %s" % tarball)
else:
env.logger.warn("RNA-seq transcripts not available for %s" % gid)
def _download_annotation_bundle(env, url, gid):
"""Download bundle of RNA-seq data from S3 biodata/annotation
"""
tarball = os.path.basename(url)
if not env.safe_exists(tarball):
with warn_only():
env.safe_run("wget %s" % url)
if env.safe_exists(tarball):
env.safe_run("xz -dc %s | tar -xvpf -" % tarball)
env.safe_run("rm -f %s" % tarball)
else:
env.logger.warn("RNA-seq transcripts not available for %s" % gid)