def submit_VAMPS(preprocessed_data_id):
"""Submit preprocessed data to VAMPS
Parameters
----------
preprocessed_data_id : int
The preprocesssed data id
"""
preprocessed_data = PreprocessedData(preprocessed_data_id)
study = Study(preprocessed_data.study)
sample_template = SampleTemplate(study.sample_template)
prep_template = PrepTemplate(preprocessed_data.prep_template)
status = preprocessed_data.submitted_to_vamps_status()
if status in ('submitting', 'success'):
raise ValueError("Cannot resubmit! Current status is: %s" % status)
preprocessed_data.update_vamps_status('submitting')
# Generating a tgz
targz_folder = mkdtemp(prefix=qiita_config.working_dir)
targz_fp = join(targz_folder, '%d_%d_%d.tgz' % (study.id,
prep_template.id,
preprocessed_data.id))
targz = taropen(targz_fp, mode='w:gz')
# adding sample/prep
samp_fp = join(targz_folder, 'sample_metadata.txt')
sample_template.to_file(samp_fp)
targz.add(samp_fp, arcname='sample_metadata.txt')
prep_fp = join(targz_folder, 'prep_metadata.txt')
prep_template.to_file(prep_fp)
targz.add(prep_fp, arcname='prep_metadata.txt')
# adding preprocessed data
for _, fp, fp_type in preprocessed_data.get_filepaths():
if fp_type == 'preprocessed_fasta':
targz.add(fp, arcname='preprocessed_fasta.fna')
targz.close()
# submitting
cmd = ("curl -F user=%s -F pass='******' -F uploadFile=@%s -F "
"press=UploadFile %s" % (qiita_config.vamps_user,
qiita_config.vamps_pass,
targz_fp,
qiita_config.vamps_url))
obs, _, _ = system_call(cmd)
exp = ("<html>\n<head>\n<title>Process Uploaded File</title>\n</head>\n"
"<body>\n</body>\n</html>")
if obs != exp:
preprocessed_data.update_vamps_status('failure')
return False
else:
preprocessed_data.update_vamps_status('success')
return True
def submit_VAMPS(preprocessed_data_id):
"""Submit preprocessed data to VAMPS
Parameters
----------
preprocessed_data_id : int
The preprocesssed data id
"""
preprocessed_data = PreprocessedData(preprocessed_data_id)
study = Study(preprocessed_data.study)
sample_template = SampleTemplate(study.sample_template)
prep_template = PrepTemplate(preprocessed_data.prep_template)
status = preprocessed_data.submitted_to_vamps_status()
if status in ('submitting', 'success'):
raise ValueError("Cannot resubmit! Current status is: %s" % status)
preprocessed_data.update_vamps_status('submitting')
# Generating a tgz
targz_folder = mkdtemp(prefix=qiita_config.working_dir)
targz_fp = join(
targz_folder,
'%d_%d_%d.tgz' % (study.id, prep_template.id, preprocessed_data.id))
targz = taropen(targz_fp, mode='w:gz')
# adding sample/prep
samp_fp = join(targz_folder, 'sample_metadata.txt')
sample_template.to_file(samp_fp)
targz.add(samp_fp, arcname='sample_metadata.txt')
prep_fp = join(targz_folder, 'prep_metadata.txt')
prep_template.to_file(prep_fp)
targz.add(prep_fp, arcname='prep_metadata.txt')
# adding preprocessed data
for _, fp, fp_type in preprocessed_data.get_filepaths():
if fp_type == 'preprocessed_fasta':
targz.add(fp, arcname='preprocessed_fasta.fna')
targz.close()
# submitting
cmd = ("curl -F user=%s -F pass='******' -F uploadFile=@%s -F "
"press=UploadFile %s" %
(qiita_config.vamps_user, qiita_config.vamps_pass, targz_fp,
qiita_config.vamps_url))
obs, _, _ = system_call(cmd)
exp = ("<html>\n<head>\n<title>Process Uploaded File</title>\n</head>\n"
"<body>\n</body>\n</html>")
if obs != exp:
preprocessed_data.update_vamps_status('failure')
return False
else:
preprocessed_data.update_vamps_status('success')
return True
def system_call_from_job(job_id, **kwargs):
"""Executes a system call described by a Job
Parameters
----------
job_id : int
The job object ID
"""
job = Job(job_id)
name, command = job.command
options = job.options
cmd = [command]
cmd.extend(flatten(options.items()))
cmd_fmt = ' '.join((str(i) for i in cmd))
try:
so, se, status = system_call(cmd_fmt)
except Exception as e:
job.set_error(str(e))
raise
# FIX THIS add_results should not be hard coded Issue #269
job.add_results([(job.options["--output_dir"], "directory")])
def system_call_from_job(job_id, **kwargs):
"""Executes a system call described by a Job
Parameters
----------
job_id : int
The job object ID
"""
job = Job(job_id)
name, command = job.command
options = job.options
cmd = [command]
cmd.extend(flatten(options.items()))
cmd_fmt = ' '.join((str(i) for i in cmd))
try:
so, se, status = system_call(cmd_fmt)
except Exception as e:
job.set_error(str(e))
raise
# FIX THIS add_results should not be hard coded Issue #269
job.add_results([(job.options["--output_dir"], "directory")])
def submit_EBI(preprocessed_data_id, action, send):
"""Submit a preprocessed data to EBI
Parameters
----------
preprocessed_data_id : int
The preprocesssed data id
action : %s
The action to perform with this data
send : bool
True to actually send the files
"""
# step 1: init and validate
ebi_submission = EBISubmission(preprocessed_data_id, action)
# step 2: generate demux fastq files
ebi_submission.study.ebi_submission_status = 'submitting'
try:
ebi_submission.generate_demultiplexed_fastq()
except:
error_msg = format_exc()
if isdir(ebi_submission.full_ebi_dir):
rmtree(ebi_submission.full_ebi_dir)
ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg
LogEntry.create('Runtime', error_msg,
info={'ebi_submission': preprocessed_data_id})
raise
# step 3: generate and write xml files
ebi_submission.generate_xml_files()
if send:
# step 4: sending sequences
old_ascp_pass = environ.get('ASPERA_SCP_PASS', '')
environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass
LogEntry.create('Runtime',
("Submitting sequences for pre_processed_id: "
"%d" % preprocessed_data_id))
try:
for cmd in ebi_submission.generate_send_sequences_cmd():
try:
stdout, stderr, _ = system_call(cmd)
except Exception as e:
stdout = ''
stderr = str(e)
le = LogEntry.create(
'Fatal', "Command: %s\nError: %s\n" % (cmd, str(e)),
info={'ebi_submission': preprocessed_data_id})
ebi_submission.study.ebi_submission_status = (
"failed: ASCP submission, log id: %d" % le.id)
raise ComputeError("EBI Submission failed! Log id: "
"%d" % le.id)
finally:
open(ebi_submission.ascp_reply, 'a').write(
'stdout:\n%s\n\nstderr: %s' % (stdout, stderr))
finally:
environ['ASPERA_SCP_PASS'] = old_ascp_pass
LogEntry.create('Runtime',
('Submission of sequences of pre_processed_id: '
'%d completed successfully' %
preprocessed_data_id))
# step 5: sending xml and parsing answer
xmls_cmds = ebi_submission.generate_curl_command()
LogEntry.create('Runtime',
("Submitting XMLs for pre_processed_id: "
"%d" % preprocessed_data_id))
try:
xml_content, stderr, _ = system_call(xmls_cmds)
except Exception as e:
xml_content = ''
stderr = str(e)
le = LogEntry.create('Fatal',
"Command: %s\nError: %s\n" % (
cmd, str(e)),
info={'ebi_submission': preprocessed_data_id})
ebi_submission.study.ebi_submission_status = (
"failed: XML submission, log id: %d" % le.id)
raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
else:
LogEntry.create('Runtime',
('Submission of sequences of pre_processed_id: '
'%d completed successfully' %
preprocessed_data_id))
finally:
open(ebi_submission.curl_reply, 'w').write(
'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr))
try:
st_acc, sa_acc, bio_acc, ex_acc, run_acc = \
ebi_submission.parse_EBI_reply(xml_content)
except EBISubmissionError as e:
le = LogEntry.create(
'Fatal', "Command: %s\nError: %s\n" % (xml_content, str(e)),
info={'ebi_submission': preprocessed_data_id})
ebi_submission.study.ebi_submission_status = (
"failed: XML parsing, log id: %d" % le.id)
raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
ebi_submission.study.ebi_submission_status = 'submitted'
if st_acc:
ebi_submission.study.ebi_study_accession = st_acc
if sa_acc:
ebi_submission.sample_template.ebi_sample_accessions = sa_acc
if bio_acc:
ebi_submission.sample_template.biosample_accessions = bio_acc
if ex_acc:
ebi_submission.prep_template.ebi_experiment_accessions = ex_acc
ebi_submission.preprocessed_data.ebi_run_accessions = run_acc
else:
st_acc, sa_acc, bio_acc, ex_acc, run_acc = None, None, None, None, None
return st_acc, sa_acc, bio_acc, ex_acc, run_acc
def submit_EBI(preprocessed_data_id, action, send):
"""Submit a preprocessed data to EBI
Parameters
----------
preprocessed_data_id : int
The preprocesssed data id
action : %s
The action to perform with this data
send : bool
True to actually send the files
"""
# step 1: init and validate
ebi_submission = EBISubmission(preprocessed_data_id, action)
# step 2: generate demux fastq files
ebi_submission.study.ebi_submission_status = 'submitting'
try:
ebi_submission.generate_demultiplexed_fastq()
except:
error_msg = format_exc()
if isdir(ebi_submission.full_ebi_dir):
rmtree(ebi_submission.full_ebi_dir)
ebi_submission.study.ebi_submission_status = 'failed: %s' % error_msg
LogEntry.create('Runtime',
error_msg,
info={'ebi_submission': preprocessed_data_id})
raise
# step 3: generate and write xml files
ebi_submission.generate_xml_files()
if send:
# step 4: sending sequences
if action != 'MODIFY':
old_ascp_pass = environ.get('ASPERA_SCP_PASS', '')
environ['ASPERA_SCP_PASS'] = qiita_config.ebi_seq_xfer_pass
LogEntry.create('Runtime',
("Submitting sequences for pre_processed_id: "
"%d" % preprocessed_data_id))
try:
for cmd in ebi_submission.generate_send_sequences_cmd():
try:
stdout, stderr, _ = system_call(cmd)
except Exception as e:
stdout = ''
stderr = str(e)
le = LogEntry.create(
'Fatal',
"Command: %s\nError: %s\n" % (cmd, str(e)),
info={'ebi_submission': preprocessed_data_id})
ebi_submission.study.ebi_submission_status = (
"failed: ASCP submission, log id: %d" % le.id)
raise ComputeError("EBI Submission failed! Log id: "
"%d" % le.id)
finally:
open(ebi_submission.ascp_reply, 'a').write(
'stdout:\n%s\n\nstderr: %s' % (stdout, stderr))
finally:
environ['ASPERA_SCP_PASS'] = old_ascp_pass
LogEntry.create(
'Runtime',
('Submission of sequences of pre_processed_id: '
'%d completed successfully' % preprocessed_data_id))
# step 5: sending xml and parsing answer
xmls_cmds = ebi_submission.generate_curl_command()
LogEntry.create('Runtime', ("Submitting XMLs for pre_processed_id: "
"%d" % preprocessed_data_id))
try:
xml_content, stderr, _ = system_call(xmls_cmds)
except Exception as e:
xml_content = ''
stderr = str(e)
le = LogEntry.create('Fatal',
"Command: %s\nError: %s\n" % (cmd, str(e)),
info={'ebi_submission': preprocessed_data_id})
ebi_submission.study.ebi_submission_status = (
"failed: XML submission, log id: %d" % le.id)
raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
else:
LogEntry.create(
'Runtime',
('Submission of sequences of pre_processed_id: '
'%d completed successfully' % preprocessed_data_id))
finally:
open(ebi_submission.curl_reply, 'w').write(
'stdout:\n%s\n\nstderr: %s' % (xml_content, stderr))
try:
st_acc, sa_acc, bio_acc, ex_acc, run_acc = \
ebi_submission.parse_EBI_reply(xml_content)
except EBISubmissionError as e:
le = LogEntry.create('Fatal',
"Command: %s\nError: %s\n" %
(xml_content, str(e)),
info={'ebi_submission': preprocessed_data_id})
ebi_submission.study.ebi_submission_status = (
"failed: XML parsing, log id: %d" % le.id)
raise ComputeError("EBI Submission failed! Log id: %d" % le.id)
ebi_submission.study.ebi_submission_status = 'submitted'
if action == 'ADD':
if st_acc:
ebi_submission.study.ebi_study_accession = st_acc
if sa_acc:
ebi_submission.sample_template.ebi_sample_accessions = sa_acc
if bio_acc:
ebi_submission.sample_template.biosample_accessions = bio_acc
if ex_acc:
ebi_submission.prep_template.ebi_experiment_accessions = ex_acc
ebi_submission.artifact.ebi_run_accessions = run_acc
else:
st_acc, sa_acc, bio_acc, ex_acc, run_acc = None, None, None, None, None
return st_acc, sa_acc, bio_acc, ex_acc, run_acc
def submit_VAMPS(artifact_id):
"""Submit artifact to VAMPS
Parameters
----------
artifact_id : int
The artifact id
Raises
------
ComputeError
- If the artifact cannot be submitted to VAMPS
- If the artifact is associated with more than one prep template
"""
artifact = Artifact(artifact_id)
if not artifact.can_be_submitted_to_vamps:
raise ComputeError("Artifact %d cannot be submitted to VAMPS" %
artifact_id)
study = artifact.study
sample_template = study.sample_template
prep_templates = artifact.prep_templates
if len(prep_templates) > 1:
raise ComputeError(
"Multiple prep templates associated with the artifact: %s" %
artifact_id)
prep_template = prep_templates[0]
# Also need to check that is not submitting (see item in #1523)
if artifact.is_submitted_to_vamps:
raise ValueError("Cannot resubmit artifact %s to VAMPS!" % artifact_id)
# Generating a tgz
targz_folder = mkdtemp(prefix=qiita_config.working_dir)
targz_fp = join(targz_folder,
'%d_%d_%d.tgz' % (study.id, prep_template.id, artifact_id))
targz = taropen(targz_fp, mode='w:gz')
# adding sample/prep
samp_fp = join(targz_folder, 'sample_metadata.txt')
sample_template.to_file(samp_fp)
targz.add(samp_fp, arcname='sample_metadata.txt')
prep_fp = join(targz_folder, 'prep_metadata.txt')
prep_template.to_file(prep_fp)
targz.add(prep_fp, arcname='prep_metadata.txt')
# adding preprocessed data
for _, fp, fp_type in artifact.filepaths:
if fp_type == 'preprocessed_fasta':
targz.add(fp, arcname='preprocessed_fasta.fna')
targz.close()
# submitting
cmd = ("curl -F user=%s -F pass='******' -F uploadFile=@%s -F "
"press=UploadFile %s" %
(qiita_config.vamps_user, qiita_config.vamps_pass, targz_fp,
qiita_config.vamps_url))
obs, _, _ = system_call(cmd)
exp = ("<html>\n<head>\n<title>Process Uploaded File</title>\n</head>\n"
"<body>\n</body>\n</html>")
if obs != exp:
return False
else:
artifact.is_submitted_to_vamps = True
return True
def submit_VAMPS(artifact_id):
"""Submit artifact to VAMPS
Parameters
----------
artifact_id : int
The artifact id
Raises
------
ComputeError
- If the artifact cannot be submitted to VAMPS
- If the artifact is associated with more than one prep template
"""
artifact = Artifact(artifact_id)
if not artifact.can_be_submitted_to_vamps:
raise ComputeError("Artifact %d cannot be submitted to VAMPS"
% artifact_id)
study = artifact.study
sample_template = study.sample_template
prep_templates = artifact.prep_templates
if len(prep_templates) > 1:
raise ComputeError(
"Multiple prep templates associated with the artifact: %s"
% artifact_id)
prep_template = prep_templates[0]
# Also need to check that is not submitting (see item in #1523)
if artifact.is_submitted_to_vamps:
raise ValueError("Cannot resubmit artifact %s to VAMPS!" % artifact_id)
# Generating a tgz
targz_folder = mkdtemp(prefix=qiita_config.working_dir)
targz_fp = join(targz_folder, '%d_%d_%d.tgz' % (study.id,
prep_template.id,
artifact_id))
targz = taropen(targz_fp, mode='w:gz')
# adding sample/prep
samp_fp = join(targz_folder, 'sample_metadata.txt')
sample_template.to_file(samp_fp)
targz.add(samp_fp, arcname='sample_metadata.txt')
prep_fp = join(targz_folder, 'prep_metadata.txt')
prep_template.to_file(prep_fp)
targz.add(prep_fp, arcname='prep_metadata.txt')
# adding preprocessed data
for _, fp, fp_type in artifact.filepaths:
if fp_type == 'preprocessed_fasta':
targz.add(fp, arcname='preprocessed_fasta.fna')
targz.close()
# submitting
cmd = ("curl -F user=%s -F pass='******' -F uploadFile=@%s -F "
"press=UploadFile %s" % (qiita_config.vamps_user,
qiita_config.vamps_pass,
targz_fp,
qiita_config.vamps_url))
obs, _, _ = system_call(cmd)
exp = ("<html>\n<head>\n<title>Process Uploaded File</title>\n</head>\n"
"<body>\n</body>\n</html>")
if obs != exp:
return False
else:
artifact.is_submitted_to_vamps = True
return True
