def _download_https_S3_files(job):
"""
Attempts to download an unencrypted file, a file encrypted with a key.
"""
http_base = 'https://s3-us-west-2.amazonaws.com/cgl-pipeline-inputs/protect/ci_references' \
'/rsem_index_chr6.tar.gz'
get_file_from_url(job, http_base, write_to_jobstore=False)
def _download_http_files(job):
"""
Attempts to download an unencrypted file
"""
http_base = 'http://link.aps.org'
get_file_from_url(job, http_base, write_to_jobstore=False)
def _download_https_files(job):
"""
Attempts to download an unencrypted file, a file encrypted with a key.
"""
http_base = 'https://google.com'
get_file_from_url(job, http_base, write_to_jobstore=False)
def get_pipeline_inputs(job, input_flag, input_file, encryption_key=None, per_file_encryption=False,
gdc_download_token=None):
"""
Get the input file from s3 or disk and write to file store.
:param str input_flag: The name of the flag
:param str input_file: The value passed in the config file
:param str encryption_key: Path to the encryption key if encrypted with sse-c
:param bool per_file_encryption: If encrypted, was the file encrypted using the per-file method?
:param str gdc_download_token: The download token to obtain files from the GDC
:return: fsID for the file
:rtype: toil.fileStore.FileID
"""
work_dir = os.getcwd()
job.fileStore.logToMaster('Obtaining file (%s) to the file job store' % input_flag)
if input_file.startswith(('http', 'https', 'ftp')):
input_file = get_file_from_url(job, input_file, encryption_key=encryption_key,
per_file_encryption=per_file_encryption,
write_to_jobstore=True)
elif input_file.startswith(('S3', 's3')):
input_file = get_file_from_s3(job, input_file, encryption_key=encryption_key,
per_file_encryption=per_file_encryption,
write_to_jobstore=True)
elif input_file.startswith(('GDC', 'gdc')):
input_file = get_file_from_gdc(job, input_file, gdc_download_token=gdc_download_token,
write_to_jobstore=True)
else:
assert os.path.exists(input_file), 'Bogus Input : ' + input_file
input_file = job.fileStore.writeGlobalFile(input_file)
return input_file
def _download_ftp_files(job):
"""
Attempts to download an unencrypted file, a file encrypted with a key.
"""
http_base = 'ftp://ftp.debian.org/debian/README'
get_file_from_url(job, http_base, write_to_jobstore=False)
