def download_from_url_gatk(job, url, name):
"""
Simple curl request made for a given url
url: str URL to download
name: str Name to give downloaded file
"""
work_dir = job.fileStore.getLocalTempDir()
file_path = os.path.join(work_dir, name)
if not os.path.exists(file_path):
if url.startswith('s3:'):
download_from_s3_url(file_path, url)
else:
try:
if debug:
debug_log = open('download_from_url', 'a')
debug_log.write(file_path + '\n')
debug_log.close()
f = open(file_path, 'w')
f.write('debug')
f.close()
else:
subprocess.check_call(['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path])
except subprocess.CalledProcessError:
raise RuntimeError(
'\nNecessary file could not be acquired: {}. Check input URL'.format(url))
except OSError:
raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
assert os.path.exists(file_path)
return job.fileStore.writeGlobalFile(file_path)
def download_url(job, url, filename):
"""
Downloads a file from a URL and places it in the jobStore
:param job: Job instance
:param url: data url, str
:param filename: name given to downloaded data, str
:return: fileStore promise
"""
work_dir = job.fileStore.getLocalTempDir()
file_path = os.path.join(work_dir, filename)
if not os.path.exists(file_path):
if url.startswith('s3:'):
download_from_s3_url(file_path, url)
else:
try:
subprocess.check_call(['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path])
except subprocess.CalledProcessError as cpe:
raise RuntimeError(
'\nNecessary file could not be acquired: %s. Got error "%s". Check input URL' % (url, cpe))
except OSError:
raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
assert os.path.exists(file_path)
return job.fileStore.writeGlobalFile(file_path)
def download_from_url(job, url, fname):
"""
Downloads a file from a URL and places it in the jobStore
Input1: Toil job instance
Input2: Input arguments
Input3: jobstore id dictionary
Input4: Name of key used to access url in input_args
"""
work_dir = job.fileStore.getLocalTempDir()
file_path = os.path.join(work_dir, fname)
if not os.path.exists(file_path):
if url.startswith('s3:'):
download_from_s3_url(file_path, url)
else:
try:
subprocess.check_call([
'curl', '-fs', '--retry', '5', '--create-dir', url, '-o',
file_path
])
except OSError:
raise RuntimeError(
'Failed to find "curl". Install via "apt-get install curl"'
)
assert os.path.exists(file_path)
return job.fileStore.writeGlobalFile(file_path)
def download_from_url(job, url):
"""
Simple curl request made for a given url
url: str URL to download
"""
work_dir = job.fileStore.getLocalTempDir()
file_path = os.path.join(work_dir, os.path.basename(url))
if url.startswith('s3:'):
download_from_s3_url(file_path, url)
else:
try:
subprocess.check_call(['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path])
except OSError:
raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
assert os.path.exists(file_path)
return job.fileStore.writeGlobalFile(file_path)
def download_from_url(job, url):
"""
Simple curl request made for a given url
url: str URL to download
"""
work_dir = job.fileStore.getLocalTempDir()
file_path = os.path.join(work_dir, os.path.basename(url))
if not os.path.exists(file_path):
if url.startswith('s3:'):
download_from_s3_url(file_path, url)
else:
try:
subprocess.check_call(['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path])
except OSError:
raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
assert os.path.exists(file_path)
return job.fileStore.writeGlobalFile(file_path)
def download_from_url(job, url, fname):
"""
Downloads a file from a URL and places it in the jobStore
Input1: Toil job instance
Input2: Input arguments
Input3: jobstore id dictionary
Input4: Name of key used to access url in input_args
"""
work_dir = job.fileStore.getLocalTempDir()
file_path = os.path.join(work_dir, fname)
if not os.path.exists(file_path):
if url.startswith('s3:'):
download_from_s3_url(file_path, url)
else:
try:
subprocess.check_call(['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path])
except OSError:
raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
assert os.path.exists(file_path)
return job.fileStore.writeGlobalFile(file_path)
def download_from_url(job, url):
"""
Downloads a URL that was supplied as an argument to running this script in LocalTempDir.
After downloading the file, it is stored in the FileStore.
url: str URL to be downloaded. filename is derived from URL
"""
work_dir = job.fileStore.getLocalTempDir()
file_path = os.path.join(work_dir, os.path.basename(url))
if not os.path.exists(file_path):
if url.startswith('s3:'):
download_from_s3_url(file_path, url)
else:
try:
download_cmd = ['curl', '-fs', '--retry', '5', '--create-dir', url, '-o', file_path]
log.info("Downloading file using command %s." % " ".join(download_cmd))
subprocess.check_call(download_cmd)
except OSError:
raise RuntimeError('Failed to find "curl". Install via "apt-get install curl"')
assert os.path.exists(file_path)
return job.fileStore.writeGlobalFile(file_path)
