def test_2_make_note(self):
"""Make a note subset by example flowcell and project"""
s_con = SampleRunMetricsConnection(username=self.user, password=self.pw, url=self.url)
fc_con = FlowcellRunMetricsConnection(username=self.user, password=self.pw, url=self.url)
p_con = ProjectSummaryConnection(username=self.user, password=self.pw, url=self.url)
paragraphs = sample_note_paragraphs()
headers = sample_note_headers()
samples = s_con.get_samples(self.examples["flowcell"], self.examples["project"])
project = p_con.get_entry(self.examples["project"])
samples = p_con.map_srm_to_name(self.examples["project"], fc_id=self.examples["flowcell"], use_bc_map=True)
for k,v in samples.items():
s_param = parameters
s = s_con.get_entry(k)
s_param.update({key:s[srm_to_parameter[key]] for key in srm_to_parameter.keys()})
fc = "{}_{}".format(s["date"], s["flowcell"])
s_param["phix_error_rate"] = fc_con.get_phix_error_rate(str(fc), s["lane"])
s_param['avg_quality_score'] = s_con.calc_avg_qv(s["name"])
s_param['rounded_read_count'] = round(float(s_param['rounded_read_count'])/1e6,1) if s_param['rounded_read_count'] else None
s_param['customer_name'] = project['samples'][v["sample"]].get('customer_name', None)
if project:
s_param['ordered_amount'] = p_con.get_ordered_amount(self.examples["project"])
s_param['customer_reference'] = s_param.get('customer_reference', project['customer_reference'])
s_param['uppnex_project_id'] = s_param.get('uppnex_project_id', project['uppnex_id'])
s_param['success'] = sequencing_success(s_param, cutoffs)
s_param.update({k:"N/A" for k in s_param.keys() if s_param[k] is None})
make_note("{}.pdf".format(s["barcode_name"]), headers, paragraphs, **s_param)
def test_2_make_project_note(self):
"""Make a project note subset by flowcell and project"""
s_con = SampleRunMetricsConnection(username=self.user, password=self.pw, url=self.url)
fc_con = FlowcellRunMetricsConnection(username=self.user, password=self.pw, url=self.url)
p_con = ProjectSummaryConnection(username=self.user, password=self.pw, url=self.url)
paragraphs = project_note_paragraphs()
headers = project_note_headers()
param = parameters
project = p_con.get_entry(self.examples["project"])
if not project:
print "No project named {}".format(self.examples["project"])
return
if project:
ordered_amount = p_con.get_ordered_amount(self.examples["project"])
else:
return
ordered_amount = self.pargs.ordered_million_reads
## Start collecting the data
sample_table = []
sample_list = project['samples']
param.update({key:project.get(ps_to_parameter[key], None) for key in ps_to_parameter.keys()})
samples = p_con.map_name_to_srm(self.examples["project"], check_consistency=True, use_bc_map=True)
all_passed = True
for k,v in samples.items():
if k=="Unexpected":
continue
project_sample = sample_list[k]
vals = {x:project_sample.get(prjs_to_table[x], None) for x in prjs_to_table.keys()}
vals['MOrdered'] = ordered_amount
vals['BarcodeSeq'] = s_con.get_entry(v.keys()[0], "sequence")
## Set status
vals['Status'] = set_status(vals) if vals['Status'] is None else vals['Status']
vals.update({k:"N/A" for k in vals.keys() if vals[k] is None})
if vals['Status']=="N/A" or vals['Status']=="NP": all_passed = False
sample_table.append([vals[k] for k in table_keys])
if all_passed: param["finished"] = 'Project finished.'
sample_table.sort()
sample_table = list(sample_table for sample_table,_ in itertools.groupby(sample_table))
sample_table.insert(0, ['ScilifeID', 'CustomerID', 'BarcodeSeq', 'MSequenced', 'MOrdered', 'Status'])
paragraphs["Samples"]["tpl"] = make_sample_table(sample_table)
make_note("{}.pdf".format(self.examples["project"]), headers, paragraphs, **param)
def _project_status_note_table(project_name=None,
username=None,
password=None,
url=None,
use_ps_map=True,
use_bc_map=False,
check_consistency=False,
ordered_million_reads=None,
uppnex_id=None,
customer_reference=None,
exclude_sample_ids={},
project_alias=None,
sample_aliases={},
projectdb="projects",
samplesdb="samples",
flowcelldb="flowcells",
include_all_samples=False,
param={},
**kw):
# mapping project_summary to parameter keys
ps_to_parameter = {
"scilife_name": "scilife_name",
"customer_name": "customer_name",
"project_name": "project_name"
}
# mapping project sample to table
table_keys = [
'ScilifeID', 'SubmittedID', 'BarcodeSeq', 'MSequenced', 'MOrdered'
]
output_data = {
'stdout': StringIO(),
'stderr': StringIO(),
'debug': StringIO()
}
# Connect and run
s_con = SampleRunMetricsConnection(dbname=samplesdb,
username=username,
password=password,
url=url)
fc_con = FlowcellRunMetricsConnection(dbname=flowcelldb,
username=username,
password=password,
url=url)
p_con = ProjectSummaryConnection(dbname=projectdb,
username=username,
password=password,
url=url)
#Get the information source for this project
source = p_con.get_info_source(project_name)
# Get project summary from project database
sample_aliases = _literal_eval_option(sample_aliases, default={})
prj_summary = p_con.get_entry(project_name)
if not prj_summary:
LOG.warn("No such project '{}'".format(project_name))
return
LOG.debug("Working on project '{}'.".format(project_name))
# Determine if project is finished by getting all samples sequenced date
try:
all_samples_sequenced = prj_summary['project_summary'][
'all_samples_sequenced']
except (TypeError, KeyError):
all_samples_sequenced = False
# Get sample run list and loop samples to make mapping sample -> {sampleruns}
sample_run_list = _set_sample_run_list(project_name,
flowcell=None,
project_alias=project_alias,
s_con=s_con)
samples = {}
for s in sample_run_list:
prj_sample = p_con.get_project_sample(
project_name, s.get("project_sample_name", None))
if prj_sample:
sample_name = prj_sample['project_sample'].get(
"scilife_name", None)
s_d = {s["name"]: {'sample': sample_name, 'id': s["_id"]}}
samples.update(s_d)
else:
if s["barcode_name"] in sample_aliases:
s_d = {
sample_aliases[s["barcode_name"]]: {
'sample': sample_aliases[s["barcode_name"]],
'id': s["_id"]
}
}
samples.update(s_d)
else:
s_d = {
s["name"]: {
'sample': s["name"],
'id': s["_id"],
'barcode_name': s["barcode_name"]
}
}
LOG.warn(
"No mapping found for sample run:\n '{}'".format(s_d))
# Convert to mapping from desired sample name to list of aliases
# Less important for the moment; one solution is to update the
# Google docs summary table to use the P names
sample_dict = prj_summary['samples']
param.update({
key: prj_summary.get(ps_to_parameter[key], None)
for key in ps_to_parameter.keys()
})
param["ordered_amount"] = param.get(
"ordered_amount",
p_con.get_ordered_amount(project_name, samples=sample_dict))
if not param.get('customer_reference'):
try:
param['customer_reference'] = prj_summary['details'][
'customer_project_reference']
except (TypeError, KeyError):
param['customer_reference'] = prj_summary.get('customer_reference')
param['uppnex_project_id'] = param.get('uppnex_project_id',
prj_summary.get('uppnex_id'))
# Override database values if options passed at command line
if uppnex_id:
param["uppnex_project_id"] = uppnex_id
if customer_reference:
param["customer_reference"] = customer_reference
# Process options
ordered_million_reads = _literal_eval_option(ordered_million_reads)
exclude_sample_ids = _literal_eval_option(exclude_sample_ids, default={})
## Start collecting the data
sample_table = []
samples_excluded = []
last_library_preps = p_con.get_latest_library_prep(project_name)
last_library_preps_srm = [
x for l in last_library_preps.values() for x in l
]
LOG.debug(
"Looping through sample map that maps project sample names to sample run metrics ids"
)
for k, v in samples.items():
LOG.debug("project sample '{}' maps to '{}'".format(k, v))
if not include_all_samples:
if v['sample'] not in last_library_preps.keys():
LOG.info(
"No library prep information for sample {}; keeping in report"
.format(v['sample']))
else:
if k not in last_library_preps_srm:
LOG.info(
"Sample run {} ('{}') is not latest library prep ({}) for project sample {}: excluding from report"
.format(
k, v["id"], ",".join(
list(
set(last_library_preps[
v['sample']].values()))), v['sample']))
continue
else:
pass
if re.search("Unexpected", k):
continue
barcode_seq = s_con.get_entry(k, "sequence")
# Exclude sample id?
if _exclude_sample_id(exclude_sample_ids, v['sample'], barcode_seq):
samples_excluded.append(v['sample'])
continue
# Get the project sample name from the sample run and set table values
project_sample = sample_dict[v['sample']]
vals = _set_sample_table_values(v['sample'], project_sample,
barcode_seq, ordered_million_reads,
param)
sample_table.append([vals[k] for k in table_keys])
# Loop through samples in sample_dict for which there is no sample run information
samples_in_table_or_excluded = list(set([x[0] for x in sample_table
])) + samples_excluded
samples_not_in_table = list(
set(sample_dict.keys()) - set(samples_in_table_or_excluded))
for sample in samples_not_in_table:
if re.search("Unexpected", sample):
continue
project_sample = sample_dict[sample]
# Set project_sample_d: a dictionary mapping from sample run metrics name to sample run metrics database id
project_sample_d = _set_project_sample_dict(project_sample, source)
if project_sample_d:
for k, v in project_sample_d.iteritems():
barcode_seq = s_con.get_entry(k, "sequence")
vals = _set_sample_table_values(sample, project_sample,
barcode_seq,
ordered_million_reads, param)
sample_table.append([vals[k] for k in table_keys])
else:
barcode_seq = None
vals = _set_sample_table_values(sample, project_sample,
barcode_seq, ordered_million_reads,
param)
sample_table.append([vals[k] for k in table_keys])
if all_samples_sequenced:
param["finished"] = 'All samples for this project have been sequenced.'
sample_table.sort()
sample_table = list(sample_table
for sample_table, _ in itertools.groupby(sample_table))
sample_table.insert(
0,
['ScilifeID', 'SubmittedID', 'BarcodeSeq', 'MSequenced', 'MOrdered'])
return output_data, sample_table, param
def sample_status_note(project_name=None,
flowcell=None,
username=None,
password=None,
url=None,
ordered_million_reads=None,
uppnex_id=None,
customer_reference=None,
bc_count=None,
project_alias=[],
projectdb="projects",
samplesdb="samples",
flowcelldb="flowcells",
phix=None,
is_paired=True,
**kw):
"""Make a sample status note. Used keywords:
:param project_name: project name
:param flowcell: flowcell id
:param username: db username
:param password: db password
:param url: db url
:param ordered_million_reads: number of ordered reads in millions
:param uppnex_id: the uppnex id
:param customer_reference: customer project name
:param project_alias: project alias name
:param phix: phix error rate
:param is_paired: True if run is paired-end, False for single-end
"""
# Cutoffs
cutoffs = {
"phix_err_cutoff": 2.0,
"qv_cutoff": 30,
}
instrument = _parse_instrument_config(
os.path.expanduser(kw.get("instrument_config", "")))
instrument_dict = {i['instrument_id']: i for i in instrument}
# parameters
parameters = {
"project_name": None,
"start_date": None,
"FC_id": None,
"scilifelab_name": None,
"rounded_read_count": None,
"phix_error_rate": None,
"avg_quality_score": None,
"pct_q30_bases": None,
"success": None,
"run_mode": None,
"is_paired": True
}
# key mapping from sample_run_metrics to parameter keys
srm_to_parameter = {
"project_name": "sample_prj",
"FC_id": "flowcell",
"scilifelab_name": "barcode_name",
"start_date": "date",
"rounded_read_count": "bc_count",
"lane": "lane"
}
LOG.debug("got parameters {}".format(parameters))
output_data = {
'stdout': StringIO(),
'stderr': StringIO(),
'debug': StringIO()
}
if not _assert_flowcell_format(flowcell):
LOG.warn(
"Wrong flowcell format {}; skipping. Please use the flowcell id (format \"[A-Z0-9\-]+\")"
.format(flowcell))
return output_data
output_data = _update_sample_output_data(output_data, cutoffs)
# Connect and run
s_con = SampleRunMetricsConnection(dbname=samplesdb,
username=username,
password=password,
url=url)
fc_con = FlowcellRunMetricsConnection(dbname=flowcelldb,
username=username,
password=password,
url=url)
p_con = ProjectSummaryConnection(dbname=projectdb,
username=username,
password=password,
url=url)
# Set up paragraphs
paragraphs = sample_note_paragraphs()
headers = sample_note_headers()
# Get project
project = p_con.get_entry(project_name)
source = p_con.get_info_source(project_name)
if not project:
LOG.warn("No such project '{}'".format(project_name))
return output_data
# Set samples list
sample_run_list = _set_sample_run_list(project_name, flowcell,
project_alias, s_con)
if len(sample_run_list) == 0:
LOG.warn(
"No samples for project '{}', flowcell '{}'. Maybe there are no sample run metrics in statusdb?"
.format(project_name, flowcell))
return output_data
# Set options
ordered_million_reads = _literal_eval_option(ordered_million_reads)
bc_count = _literal_eval_option(bc_count)
phix = _literal_eval_option(phix)
# Count number of times a sample has been run on a flowcell; if several, make lane-specific reports
sample_count = Counter([x.get("barcode_name") for x in sample_run_list])
# Loop samples and collect information
s_param_out = []
fcdoc = None
for s in sample_run_list:
s_param = {}
LOG.debug(
"working on sample '{}', sample run metrics name '{}', id '{}'".
format(s.get("barcode_name", None), s.get("name", None),
s.get("_id", None)))
s_param.update(parameters)
s_param.update(
{key: s[srm_to_parameter[key]]
for key in srm_to_parameter.keys()})
fc = "{}_{}".format(s.get("date"), s.get("flowcell"))
# Get instrument
try:
s_param.update(instrument_dict[fc_con.get_instrument(str(fc))])
except:
LOG.warn(
"Failed to set instrument and software versions for flowcell {} in report due to missing RunInfo -> Instrument field in statusdb. Either rerun 'pm qc update-qc' or search-and-replace 'NN' in the sample report."
.format(fc))
s_param.update(instrument_dict['default'])
# Get run mode
if not fcdoc or fcdoc.get("name") != fc:
fcdoc = fc_con.get_entry(fc)
runp = fcdoc.get("RunParameters", {})
s_param[
"sequencing_platform"] = "MiSeq" if "MCSVersion" in runp else "HiSeq2500"
s_param["clustering_method"] = "onboard clustering" if runp.get(
"ClusteringChoice", "") == "OnBoardClustering" or s_param[
"sequencing_platform"] == "MiSeq" else "cBot"
s_param["sequencing_setup"] = fcdoc.get("run_setup")
s_param["sequencing_mode"] = runp.get("RunMode", "High Output")
s_param["sequencing_software"] = "RTA {}".format(
runp.get("RTAVersion"))
if s_param["sequencing_platform"] == "MiSeq":
s_param["sequencing_software"] = "MCS {}/{}".format(
runp.get("MCSVersion"), s_param["sequencing_software"])
else:
s_param["sequencing_software"] = "{} {}/{}".format(
runp.get("ApplicationName"), runp.get("ApplicationVersion"),
s_param["sequencing_software"])
s_param["is_paired"] = fc_con.is_paired_end(str(fc))
if s_param["is_paired"] is None:
LOG.warn(
"Could not determine run setup for flowcell {}. Will assume paired-end."
.format(fc))
s_param["is_paired"] = True
s_param.update(software_versions)
s_param["phix_error_rate"] = fc_con.get_phix_error_rate(
str(fc), s["lane"])
if phix:
s_param["phix_error_rate"] = _get_phix_error_rate(s["lane"], phix)
# Get quality score from demultiplex stats, if that fails
# (which it shouldn't), fall back on fastqc data.
(avg_quality_score,
pct_q30_bases) = fc_con.get_barcode_lane_statistics(
project_name, s.get("barcode_name"), fc, s["lane"])
s_param[
'avg_quality_score'] = avg_quality_score if avg_quality_score else calc_avg_qv(
s)
if not s_param['avg_quality_score']:
LOG.warn(
"Setting average quality failed for sample {}, id {}".format(
s.get("name"), s.get("_id")))
s_param['pct_q30_bases'] = pct_q30_bases
if not s_param['pct_q30_bases']:
LOG.warn(
"Setting % of >= Q30 Bases (PF) failed for sample {}, id {}".
format(s.get("name"), s.get("_id")))
# Compare phix error and qv to cutoffs
err_stat = "OK"
qv_stat = "OK"
if s_param["phix_error_rate"] > cutoffs["phix_err_cutoff"]:
err_stat = "HIGH"
elif s_param["phix_error_rate"] == -1:
err_stat = "N/A"
if s_param["avg_quality_score"] < cutoffs["qv_cutoff"]:
qv_stat = "LOW"
output_data["stdout"].write(
"{:>18}\t{:>6}\t{:>12}\t{:>12}\t{:>12}\t{:>12}\n".format(
s["barcode_name"], s["lane"], s_param["phix_error_rate"],
err_stat, s_param["avg_quality_score"], qv_stat))
# Update/set remaning sample run parameters, falling back on project defaults if *key* is missing
s_param['ordered_amount'] = s_param.get(
'ordered_amount',
p_con.get_ordered_amount(project_name,
samples=p_con.get_entry(
project_name, 'samples')))
s_param['customer_reference'] = s_param.get(
'customer_reference', project.get('customer_reference'))
s_param['uppnex_project_id'] = s_param.get('uppnex_project_id',
project.get('uppnex_id'))
# Override database settings if options passed at command line
if ordered_million_reads:
s_param["ordered_amount"] = _get_ordered_million_reads(
s["barcode_name"], ordered_million_reads)
if bc_count:
s_param["rounded_read_count"] = _round_read_count_in_millions(
_get_bc_count(s["barcode_name"], bc_count, s))
else:
s_param["rounded_read_count"] = _round_read_count_in_millions(
s_param["rounded_read_count"])
if uppnex_id:
s_param["uppnex_project_id"] = uppnex_id
if customer_reference:
s_param["customer_reference"] = customer_reference
# Get the project sample name corresponding to the sample run
project_sample = p_con.get_project_sample(
project_name, s.get("project_sample_name", None))
if project_sample:
LOG.debug(
"project sample run metrics mapping found: '{}' : '{}'".format(
s["name"], project_sample["sample_name"]))
project_sample_item = project_sample['project_sample']
# Set project_sample_d: a dictionary mapping from sample run metrics name to sample run metrics database id
project_sample_d = _set_project_sample_dict(
project_sample_item, source)
if not project_sample_d:
LOG.warn(
"No sample_run_metrics information for sample '{}', barcode name '{}', id '{}'\n\tProject summary information {}"
.format(s["name"], s["barcode_name"], s["_id"],
project_sample))
# Check if sample run metrics name present in project database: if so, verify that database ids are consistent
if s["name"] not in project_sample_d.keys():
LOG.warn(
"no such sample run metrics '{}' in project sample run metrics dictionary"
.format(s["name"]))
else:
if s["_id"] == project_sample_d[s["name"]]:
LOG.debug(
"project sample run metrics mapping found: '{}' : '{}'"
.format(s["name"], project_sample_d[s["name"]]))
else:
LOG.warn(
"inconsistent mapping for '{}': '{}' != '{}' (project summary id)"
.format(s["name"], s["_id"],
project_sample_d[s["name"]]))
s_param['customer_name'] = project_sample_item.get(
"customer_name", None)
# Always normalize submitted id, since module textttable does not support unicode
if type(s_param['customer_name']) is unicode:
s_param['customer_name'] = unicodedata.normalize(
'NFKD',
s_param['customer_name']).encode('ascii', 'ignore')
# No project sample found. Manual upload to database necessary.
else:
s_param['customer_name'] = None
LOG.warn(
"No project sample name found for sample run name '{}'".format(
s["barcode_name"]))
LOG.info(
"Please run 'pm qc upload-qc FLOWCELL_ID --extensive-matching' to update project sample names "
)
LOG.info(
"or 'pm qc update --sample_prj PROJECT_NAME --names BARCODE_TO_SAMPLE_MAP to update project sample names."
)
LOG.info("Please refer to the pm documentation for examples.")
query_ok(force=kw.get("force", False))
# Finally assess sequencing success, update parameters and set outputs
s_param['success'] = sequencing_success(s_param, cutoffs)
s_param.update({
k: "N/A"
for k in s_param.keys()
if s_param[k] is None or s_param[k] == "" or s_param[k] == -1.0
})
if sample_count[s.get("barcode_name")] > 1:
outfile = "{}_{}_{}_{}.pdf".format(s["barcode_name"], s["date"],
s["flowcell"], s["lane"])
else:
outfile = "{}_{}_{}.pdf".format(s["barcode_name"], s["date"],
s["flowcell"])
s_param["outfile"] = outfile
s_param_out.append(s_param)
# Write final output to reportlab and rst files
output_data["debug"].write(
json.dumps({
's_param': s_param_out,
'sample_runs':
{s["name"]: s["barcode_name"]
for s in sample_run_list}
}))
notes = [
make_note(headers=headers, paragraphs=paragraphs, **sp)
for sp in s_param_out
]
rest_notes = make_sample_rest_notes(
"{}_{}_{}_sample_summary.rst".format(project_name, s.get("date", None),
s.get("flowcell", None)),
s_param_out)
concatenate_notes(
notes, "{}_{}_{}_sample_summary.pdf".format(project_name,
s.get("date", None),
s.get("flowcell", None)))
return output_data
def _project_status_note_table(project_name=None, username=None, password=None, url=None,
use_ps_map=True, use_bc_map=False, check_consistency=False,
ordered_million_reads=None, uppnex_id=None, customer_reference=None,
exclude_sample_ids={}, project_alias=None, sample_aliases={},
projectdb="projects", samplesdb="samples", flowcelldb="flowcells",
include_all_samples=False, param={}, **kw):
# mapping project_summary to parameter keys
ps_to_parameter = {"scilife_name":"scilife_name", "customer_name":"customer_name", "project_name":"project_name"}
# mapping project sample to table
table_keys = ['ScilifeID', 'SubmittedID', 'BarcodeSeq', 'MSequenced', 'MOrdered']
output_data = {'stdout':StringIO(), 'stderr':StringIO(), 'debug':StringIO()}
# Connect and run
s_con = SampleRunMetricsConnection(dbname=samplesdb, username=username, password=password, url=url)
fc_con = FlowcellRunMetricsConnection(dbname=flowcelldb, username=username, password=password, url=url)
p_con = ProjectSummaryConnection(dbname=projectdb, username=username, password=password, url=url)
#Get the information source for this project
source = p_con.get_info_source(project_name)
# Get project summary from project database
sample_aliases = _literal_eval_option(sample_aliases, default={})
prj_summary = p_con.get_entry(project_name)
if not prj_summary:
LOG.warn("No such project '{}'".format(project_name))
return
LOG.debug("Working on project '{}'.".format(project_name))
# Determine if project is finished by getting all samples sequenced date
try:
all_samples_sequenced = prj_summary['project_summary']['all_samples_sequenced']
except (TypeError,KeyError):
all_samples_sequenced = False
# Get sample run list and loop samples to make mapping sample -> {sampleruns}
sample_run_list = _set_sample_run_list(project_name, flowcell=None, project_alias=project_alias, s_con=s_con)
samples = {}
for s in sample_run_list:
prj_sample = p_con.get_project_sample(project_name, s.get("project_sample_name", None))
if prj_sample:
sample_name = prj_sample['project_sample'].get("scilife_name", None)
s_d = {s["name"] : {'sample':sample_name, 'id':s["_id"]}}
samples.update(s_d)
else:
if s["barcode_name"] in sample_aliases:
s_d = {sample_aliases[s["barcode_name"]] : {'sample':sample_aliases[s["barcode_name"]], 'id':s["_id"]}}
samples.update(s_d)
else:
s_d = {s["name"]:{'sample':s["name"], 'id':s["_id"], 'barcode_name':s["barcode_name"]}}
LOG.warn("No mapping found for sample run:\n '{}'".format(s_d))
# Convert to mapping from desired sample name to list of aliases
# Less important for the moment; one solution is to update the
# Google docs summary table to use the P names
sample_dict = prj_summary['samples']
param.update({key:prj_summary.get(ps_to_parameter[key], None) for key in ps_to_parameter.keys()})
param["ordered_amount"] = param.get("ordered_amount", p_con.get_ordered_amount(project_name, samples=sample_dict))
if not param.get('customer_reference') :
try:
param['customer_reference'] = prj_summary['details']['customer_project_reference']
except (TypeError,KeyError):
param['customer_reference'] = prj_summary.get('customer_reference')
param['uppnex_project_id'] = param.get('uppnex_project_id', prj_summary.get('uppnex_id'))
# Override database values if options passed at command line
if uppnex_id:
param["uppnex_project_id"] = uppnex_id
if customer_reference:
param["customer_reference"] = customer_reference
# Process options
ordered_million_reads = _literal_eval_option(ordered_million_reads)
exclude_sample_ids = _literal_eval_option(exclude_sample_ids, default={})
## Start collecting the data
sample_table = []
samples_excluded = []
last_library_preps = p_con.get_latest_library_prep(project_name)
last_library_preps_srm = [x for l in last_library_preps.values() for x in l]
LOG.debug("Looping through sample map that maps project sample names to sample run metrics ids")
for k,v in samples.items():
LOG.debug("project sample '{}' maps to '{}'".format(k, v))
if not include_all_samples:
if v['sample'] not in last_library_preps.keys():
LOG.info("No library prep information for sample {}; keeping in report".format(v['sample']))
else:
if k not in last_library_preps_srm:
LOG.info("Sample run {} ('{}') is not latest library prep ({}) for project sample {}: excluding from report".format(k, v["id"], ",".join(list(set(last_library_preps[v['sample']].values()))), v['sample']))
continue
else:
pass
if re.search("Unexpected", k):
continue
barcode_seq = s_con.get_entry(k, "sequence")
# Exclude sample id?
if _exclude_sample_id(exclude_sample_ids, v['sample'], barcode_seq):
samples_excluded.append(v['sample'])
continue
# Get the project sample name from the sample run and set table values
project_sample = sample_dict[v['sample']]
vals = _set_sample_table_values(v['sample'], project_sample, barcode_seq, ordered_million_reads, param)
sample_table.append([vals[k] for k in table_keys])
# Loop through samples in sample_dict for which there is no sample run information
samples_in_table_or_excluded = list(set([x[0] for x in sample_table])) + samples_excluded
samples_not_in_table = list(set(sample_dict.keys()) - set(samples_in_table_or_excluded))
for sample in samples_not_in_table:
if re.search("Unexpected", sample):
continue
project_sample = sample_dict[sample]
# Set project_sample_d: a dictionary mapping from sample run metrics name to sample run metrics database id
project_sample_d = _set_project_sample_dict(project_sample, source)
if project_sample_d:
for k,v in project_sample_d.iteritems():
barcode_seq = s_con.get_entry(k, "sequence")
vals = _set_sample_table_values(sample, project_sample, barcode_seq, ordered_million_reads, param)
sample_table.append([vals[k] for k in table_keys])
else:
barcode_seq = None
vals = _set_sample_table_values(sample, project_sample, barcode_seq, ordered_million_reads, param)
sample_table.append([vals[k] for k in table_keys])
if all_samples_sequenced: param["finished"] = 'All samples for this project have been sequenced.'
sample_table.sort()
sample_table = list(sample_table for sample_table,_ in itertools.groupby(sample_table))
sample_table.insert(0, ['ScilifeID', 'SubmittedID', 'BarcodeSeq', 'MSequenced', 'MOrdered'])
return output_data, sample_table, param
def sample_status_note(project_name=None, flowcell=None, username=None, password=None, url=None,
ordered_million_reads=None, uppnex_id=None, customer_reference=None, bc_count=None,
project_alias=[], projectdb="projects", samplesdb="samples", flowcelldb="flowcells",
phix=None, is_paired=True, **kw):
"""Make a sample status note. Used keywords:
:param project_name: project name
:param flowcell: flowcell id
:param username: db username
:param password: db password
:param url: db url
:param ordered_million_reads: number of ordered reads in millions
:param uppnex_id: the uppnex id
:param customer_reference: customer project name
:param project_alias: project alias name
:param phix: phix error rate
:param is_paired: True if run is paired-end, False for single-end
"""
# Cutoffs
cutoffs = {
"phix_err_cutoff" : 2.0,
"qv_cutoff" : 30,
}
instrument = _parse_instrument_config(os.path.expanduser(kw.get("instrument_config","")))
instrument_dict = {i['instrument_id']: i for i in instrument}
# parameters
parameters = {
"project_name" : None,
"start_date" : None,
"FC_id" : None,
"scilifelab_name" : None,
"rounded_read_count" : None,
"phix_error_rate" : None,
"avg_quality_score" : None,
"pct_q30_bases" : None,
"success" : None,
"run_mode":None,
"is_paired":True
}
# key mapping from sample_run_metrics to parameter keys
srm_to_parameter = {"project_name":"sample_prj", "FC_id":"flowcell",
"scilifelab_name":"barcode_name", "start_date":"date",
"rounded_read_count":"bc_count", "lane": "lane"}
LOG.debug("got parameters {}".format(parameters))
output_data = {'stdout':StringIO(), 'stderr':StringIO(), 'debug':StringIO()}
if not _assert_flowcell_format(flowcell):
LOG.warn("Wrong flowcell format {}; skipping. Please use the flowcell id (format \"[A-Z0-9\-]+\")".format(flowcell) )
return output_data
output_data = _update_sample_output_data(output_data, cutoffs)
# Connect and run
s_con = SampleRunMetricsConnection(dbname=samplesdb, username=username, password=password, url=url)
fc_con = FlowcellRunMetricsConnection(dbname=flowcelldb, username=username, password=password, url=url)
p_con = ProjectSummaryConnection(dbname=projectdb, username=username, password=password, url=url)
# Set up paragraphs
paragraphs = sample_note_paragraphs()
headers = sample_note_headers()
# Get project
project = p_con.get_entry(project_name)
source = p_con.get_info_source(project_name)
if not project:
LOG.warn("No such project '{}'".format(project_name))
return output_data
# Set samples list
sample_run_list = _set_sample_run_list(project_name, flowcell, project_alias, s_con)
if len(sample_run_list) == 0:
LOG.warn("No samples for project '{}', flowcell '{}'. Maybe there are no sample run metrics in statusdb?".format(project_name, flowcell))
return output_data
# Set options
ordered_million_reads = _literal_eval_option(ordered_million_reads)
bc_count = _literal_eval_option(bc_count)
phix = _literal_eval_option(phix)
# Count number of times a sample has been run on a flowcell; if several, make lane-specific reports
sample_count = Counter([x.get("barcode_name") for x in sample_run_list])
# Loop samples and collect information
s_param_out = []
fcdoc = None
for s in sample_run_list:
s_param = {}
LOG.debug("working on sample '{}', sample run metrics name '{}', id '{}'".format(s.get("barcode_name", None), s.get("name", None), s.get("_id", None)))
s_param.update(parameters)
s_param.update({key:s[srm_to_parameter[key]] for key in srm_to_parameter.keys()})
fc = "{}_{}".format(s.get("date"), s.get("flowcell"))
# Get instrument
try:
s_param.update(instrument_dict[fc_con.get_instrument(str(fc))])
except:
LOG.warn("Failed to set instrument and software versions for flowcell {} in report due to missing RunInfo -> Instrument field in statusdb. Either rerun 'pm qc update-qc' or search-and-replace 'NN' in the sample report.".format(fc))
s_param.update(instrument_dict['default'])
# Get run mode
if not fcdoc or fcdoc.get("name") != fc:
fcdoc = fc_con.get_entry(fc)
runp = fcdoc.get("RunParameters",{})
s_param["sequencing_platform"] = "MiSeq" if "MCSVersion" in runp else "HiSeq2500"
s_param["clustering_method"] = "onboard clustering" if runp.get("ClusteringChoice","") == "OnBoardClustering" or s_param["sequencing_platform"] == "MiSeq" else "cBot"
s_param["sequencing_setup"] = fcdoc.get("run_setup")
s_param["sequencing_mode"] = runp.get("RunMode","High Output")
s_param["sequencing_software"] = "RTA {}".format(runp.get("RTAVersion"))
if s_param["sequencing_platform"] == "MiSeq":
s_param["sequencing_software"] = "MCS {}/{}".format(runp.get("MCSVersion"),s_param["sequencing_software"])
else:
s_param["sequencing_software"] = "{} {}/{}".format(runp.get("ApplicationName"),runp.get("ApplicationVersion"),s_param["sequencing_software"])
s_param["is_paired"] = fc_con.is_paired_end(str(fc))
if s_param["is_paired"] is None:
LOG.warn("Could not determine run setup for flowcell {}. Will assume paired-end.".format(fc))
s_param["is_paired"] = True
s_param.update(software_versions)
s_param["phix_error_rate"] = fc_con.get_phix_error_rate(str(fc), s["lane"])
if phix:
s_param["phix_error_rate"] = _get_phix_error_rate(s["lane"], phix)
# Get quality score from demultiplex stats, if that fails
# (which it shouldn't), fall back on fastqc data.
(avg_quality_score, pct_q30_bases) = fc_con.get_barcode_lane_statistics(project_name, s.get("barcode_name"), fc, s["lane"])
s_param['avg_quality_score'] = avg_quality_score if avg_quality_score else calc_avg_qv(s)
if not s_param['avg_quality_score']:
LOG.warn("Setting average quality failed for sample {}, id {}".format(s.get("name"), s.get("_id")))
s_param['pct_q30_bases'] = pct_q30_bases
if not s_param['pct_q30_bases']:
LOG.warn("Setting % of >= Q30 Bases (PF) failed for sample {}, id {}".format(s.get("name"), s.get("_id")))
# Compare phix error and qv to cutoffs
err_stat = "OK"
qv_stat = "OK"
if s_param["phix_error_rate"] > cutoffs["phix_err_cutoff"]:
err_stat = "HIGH"
elif s_param["phix_error_rate"] == -1:
err_stat = "N/A"
if s_param["avg_quality_score"] < cutoffs["qv_cutoff"]:
qv_stat = "LOW"
output_data["stdout"].write("{:>18}\t{:>6}\t{:>12}\t{:>12}\t{:>12}\t{:>12}\n".format(s["barcode_name"], s["lane"], s_param["phix_error_rate"], err_stat, s_param["avg_quality_score"], qv_stat))
# Update/set remaning sample run parameters, falling back on project defaults if *key* is missing
s_param['ordered_amount'] = s_param.get('ordered_amount',
p_con.get_ordered_amount(project_name,
samples=p_con.get_entry(project_name,'samples')))
s_param['customer_reference'] = s_param.get('customer_reference', project.get('customer_reference'))
s_param['uppnex_project_id'] = s_param.get('uppnex_project_id', project.get('uppnex_id'))
# Override database settings if options passed at command line
if ordered_million_reads:
s_param["ordered_amount"] = _get_ordered_million_reads(s["barcode_name"], ordered_million_reads)
if bc_count:
s_param["rounded_read_count"] = _round_read_count_in_millions(_get_bc_count(s["barcode_name"], bc_count, s))
else:
s_param["rounded_read_count"] = _round_read_count_in_millions(s_param["rounded_read_count"])
if uppnex_id:
s_param["uppnex_project_id"] = uppnex_id
if customer_reference:
s_param["customer_reference"] = customer_reference
# Get the project sample name corresponding to the sample run
project_sample = p_con.get_project_sample(project_name, s.get("project_sample_name", None))
if project_sample:
LOG.debug("project sample run metrics mapping found: '{}' : '{}'".format(s["name"], project_sample["sample_name"]))
project_sample_item = project_sample['project_sample']
# Set project_sample_d: a dictionary mapping from sample run metrics name to sample run metrics database id
project_sample_d = _set_project_sample_dict(project_sample_item, source)
if not project_sample_d:
LOG.warn("No sample_run_metrics information for sample '{}', barcode name '{}', id '{}'\n\tProject summary information {}".format(s["name"], s["barcode_name"], s["_id"], project_sample))
# Check if sample run metrics name present in project database: if so, verify that database ids are consistent
if s["name"] not in project_sample_d.keys():
LOG.warn("no such sample run metrics '{}' in project sample run metrics dictionary".format(s["name"]) )
else:
if s["_id"] == project_sample_d[s["name"]]:
LOG.debug("project sample run metrics mapping found: '{}' : '{}'".format(s["name"], project_sample_d[s["name"]]))
else:
LOG.warn("inconsistent mapping for '{}': '{}' != '{}' (project summary id)".format(s["name"], s["_id"], project_sample_d[s["name"]]))
s_param['customer_name'] = project_sample_item.get("customer_name", None)
# Always normalize submitted id, since module textttable does not support unicode
if type(s_param['customer_name']) is unicode:
s_param['customer_name'] = unicodedata.normalize('NFKD', s_param['customer_name']).encode('ascii', 'ignore')
# No project sample found. Manual upload to database necessary.
else:
s_param['customer_name'] = None
LOG.warn("No project sample name found for sample run name '{}'".format(s["barcode_name"]))
LOG.info("Please run 'pm qc upload-qc FLOWCELL_ID --extensive-matching' to update project sample names ")
LOG.info("or 'pm qc update --sample_prj PROJECT_NAME --names BARCODE_TO_SAMPLE_MAP to update project sample names.")
LOG.info("Please refer to the pm documentation for examples.")
query_ok(force=kw.get("force", False))
# Finally assess sequencing success, update parameters and set outputs
s_param['success'] = sequencing_success(s_param, cutoffs)
s_param.update({k:"N/A" for k in s_param.keys() if s_param[k] is None or s_param[k] == "" or s_param[k] == -1.0})
if sample_count[s.get("barcode_name")] > 1:
outfile = "{}_{}_{}_{}.pdf".format(s["barcode_name"], s["date"], s["flowcell"], s["lane"])
else:
outfile = "{}_{}_{}.pdf".format(s["barcode_name"], s["date"], s["flowcell"])
s_param["outfile"] = outfile
s_param_out.append(s_param)
# Write final output to reportlab and rst files
output_data["debug"].write(json.dumps({'s_param': s_param_out, 'sample_runs':{s["name"]:s["barcode_name"] for s in sample_run_list}}))
notes = [make_note(headers=headers, paragraphs=paragraphs, **sp) for sp in s_param_out]
rest_notes = make_sample_rest_notes("{}_{}_{}_sample_summary.rst".format(project_name, s.get("date", None), s.get("flowcell", None)), s_param_out)
concatenate_notes(notes, "{}_{}_{}_sample_summary.pdf".format(project_name, s.get("date", None), s.get("flowcell", None)))
return output_data
def project_status_note(project_name=None,
username=None,
password=None,
url=None,
use_ps_map=True,
use_bc_map=False,
check_consistency=False,
ordered_million_reads=None,
uppnex_id=None,
customer_reference=None,
exclude_sample_ids={},
project_alias=None,
sample_aliases={},
projectdb="projects",
samplesdb="samples",
flowcelldb="flowcells",
include_all_samples=False,
**kw):
"""Make a project status note. Used keywords:
:param project_name: project name
:param user: db user name
:param password: db password
:param url: db url
:param use_ps_map: use project summary mapping
:param use_bc_map: use project to barcode name mapping
:param check_consistency: check consistency between mappings
:param ordered_million_reads: number of ordered reads in millions
:param uppnex_id: the uppnex id
:param customer_reference: customer project name
:param exclude_sample_ids: exclude some sample ids from project note
:param project_alias: project alias name
:param sample_aliases: sample alias names
:param projectdb: project db name
:param samplesdb: samples db name
:param flowcelldb: flowcells db name
:param include_all_samples: include all samples in report
"""
# parameters
parameters = {
"project_name": project_name,
"finished": "Not finished, or cannot yet assess if finished.",
}
# mapping project_summary to parameter keys
ps_to_parameter = {
"scilife_name": "scilife_name",
"customer_name": "customer_name",
"project_name": "project_name"
}
# mapping project sample to table
table_keys = [
'ScilifeID', 'CustomerID', 'BarcodeSeq', 'MSequenced', 'MOrdered',
'Status'
]
output_data = {
'stdout': StringIO(),
'stderr': StringIO(),
'debug': StringIO()
}
# Connect and run
s_con = SampleRunMetricsConnection(dbname=samplesdb,
username=username,
password=password,
url=url)
fc_con = FlowcellRunMetricsConnection(dbname=flowcelldb,
username=username,
password=password,
url=url)
p_con = ProjectSummaryConnection(dbname=projectdb,
username=username,
password=password,
url=url)
# Set report paragraphs
paragraphs = project_note_paragraphs()
headers = project_note_headers()
# Set local param variable
param = parameters
# Get project summary from project database
sample_aliases = _literal_eval_option(sample_aliases, default={})
prj_summary = p_con.get_entry(project_name)
if not prj_summary:
LOG.warn("No such project '{}'".format(project_name))
return
LOG.debug("Working on project '{}'.".format(project_name))
# Get sample run list and loop samples to make mapping sample -> {sampleruns}
sample_run_list = _set_sample_run_list(project_name,
flowcell=None,
project_alias=project_alias,
s_con=s_con)
samples = {}
for s in sample_run_list:
prj_sample = p_con.get_project_sample(
project_name, s.get("project_sample_name", None))
if prj_sample:
sample_name = prj_sample['project_sample'].get(
"scilife_name", None)
s_d = {s["name"]: {'sample': sample_name, 'id': s["_id"]}}
samples.update(s_d)
else:
if s["barcode_name"] in sample_aliases:
s_d = {
sample_aliases[s["barcode_name"]]: {
'sample': sample_aliases[s["barcode_name"]],
'id': s["_id"]
}
}
samples.update(s_d)
else:
s_d = {
s["name"]: {
'sample': s["name"],
'id': s["_id"],
'barcode_name': s["barcode_name"]
}
}
LOG.warn(
"No mapping found for sample run:\n '{}'".format(s_d))
# Convert to mapping from desired sample name to list of aliases
# Less important for the moment; one solution is to update the
# Google docs summary table to use the P names
sample_dict = prj_summary['samples']
param.update({
key: prj_summary.get(ps_to_parameter[key], None)
for key in ps_to_parameter.keys()
})
param["ordered_amount"] = param.get("ordered_amount",
p_con.get_ordered_amount(project_name))
param['customer_reference'] = param.get(
'customer_reference', prj_summary.get('customer_reference'))
param['uppnex_project_id'] = param.get('uppnex_project_id',
prj_summary.get('uppnex_id'))
# Override database values if options passed at command line
if uppnex_id:
param["uppnex_project_id"] = uppnex_id
if customer_reference:
param["customer_reference"] = customer_reference
# Process options
ordered_million_reads = _literal_eval_option(ordered_million_reads)
exclude_sample_ids = _literal_eval_option(exclude_sample_ids, default={})
## Start collecting the data
sample_table = []
samples_excluded = []
all_passed = True
last_library_preps = p_con.get_latest_library_prep(project_name)
last_library_preps_srm = [
x for l in last_library_preps.values() for x in l
]
LOG.debug(
"Looping through sample map that maps project sample names to sample run metrics ids"
)
for k, v in samples.items():
LOG.debug("project sample '{}' maps to '{}'".format(k, v))
if not include_all_samples:
if v['sample'] not in last_library_preps.keys():
LOG.info(
"No library prep information for sample {}; keeping in report"
.format(v['sample']))
else:
if k not in last_library_preps_srm:
LOG.info(
"Sample run {} ('{}') is not latest library prep ({}) for project sample {}: excluding from report"
.format(k, v["id"],
last_library_preps[v['sample']].values()[0],
v['sample']))
continue
else:
pass
if re.search("Unexpected", k):
continue
barcode_seq = s_con.get_entry(k, "sequence")
# Exclude sample id?
if _exclude_sample_id(exclude_sample_ids, v['sample'], barcode_seq):
samples_excluded.append(v['sample'])
continue
# Get the project sample name from the sample run and set table values
project_sample = sample_dict[v['sample']]
vals = _set_sample_table_values(v['sample'], project_sample,
barcode_seq, ordered_million_reads,
param)
if vals['Status'] == "N/A" or vals['Status'] == "NP":
all_passed = False
sample_table.append([vals[k] for k in table_keys])
# Loop through samples in sample_dict for which there is no sample run information
samples_in_table_or_excluded = list(set([x[0] for x in sample_table
])) + samples_excluded
samples_not_in_table = list(
set(sample_dict.keys()) - set(samples_in_table_or_excluded))
for sample in samples_not_in_table:
if re.search("Unexpected", sample):
continue
project_sample = sample_dict[sample]
# Set project_sample_d: a dictionary mapping from sample run metrics name to sample run metrics database id
project_sample_d = _set_project_sample_dict(project_sample)
if project_sample_d:
for k, v in project_sample_d.iteritems():
barcode_seq = s_con.get_entry(k, "sequence")
vals = _set_sample_table_values(sample, project_sample,
barcode_seq,
ordered_million_reads, param)
if vals['Status'] == "N/A" or vals['Status'] == "NP":
all_passed = False
sample_table.append([vals[k] for k in table_keys])
else:
barcode_seq = None
vals = _set_sample_table_values(sample, project_sample,
barcode_seq, ordered_million_reads,
param)
if vals['Status'] == "N/A" or vals['Status'] == "NP":
all_passed = False
sample_table.append([vals[k] for k in table_keys])
if all_passed: param["finished"] = 'Project finished.'
sample_table.sort()
sample_table = list(sample_table
for sample_table, _ in itertools.groupby(sample_table))
sample_table.insert(0, [
'ScilifeID', 'CustomerID', 'BarcodeSeq', 'MSequenced', 'MOrdered',
'Status'
])
paragraphs["Samples"]["tpl"] = make_sample_table(sample_table)
make_note("{}_project_summary.pdf".format(project_name), headers,
paragraphs, **param)
make_rest_note("{}_project_summary.rst".format(project_name),
sample_table=sample_table,
report="project_report",
**param)
param.update(
{k: "N/A"
for k in param.keys() if param[k] is None or param[k] == ""})
output_data["debug"].write(
json.dumps({
'param': param,
'table': sample_table
}))
return output_data
def sample_status_note(project_name=None,
flowcell=None,
username=None,
password=None,
url=None,
ordered_million_reads=None,
uppnex_id=None,
customer_reference=None,
bc_count=None,
project_alias=[],
projectdb="projects",
samplesdb="samples",
flowcelldb="flowcells",
phix=None,
**kw):
"""Make a sample status note. Used keywords:
:param project_name: project name
:param flowcell: flowcell id
:param username: db username
:param password: db password
:param url: db url
:param ordered_million_reads: number of ordered reads in millions
:param uppnex_id: the uppnex id
:param customer_reference: customer project name
:param project_alias: project alias name
:param phix: phix error rate
"""
# Cutoffs
cutoffs = {
"phix_err_cutoff": 2.0,
"qv_cutoff": 30,
}
# parameters
parameters = {
"project_name": None,
"start_date": None,
"FC_id": None,
"scilifelab_name": None,
"rounded_read_count": None,
"phix_error_rate": None,
"avg_quality_score": None,
"success": None,
"run_mode": None,
}
# key mapping from sample_run_metrics to parameter keys
srm_to_parameter = {
"project_name": "sample_prj",
"FC_id": "flowcell",
"scilifelab_name": "barcode_name",
"start_date": "date",
"rounded_read_count": "bc_count"
}
LOG.debug("got parameters {}".format(parameters))
output_data = {
'stdout': StringIO(),
'stderr': StringIO(),
'debug': StringIO()
}
if not _assert_flowcell_format(flowcell):
LOG.warn(
"Wrong flowcell format {}; skipping. Please use the flowcell id (format \"[A-Z0-9]+XX\")"
.format(flowcell))
return output_data
output_data = _update_sample_output_data(output_data, cutoffs)
# Connect and run
s_con = SampleRunMetricsConnection(dbname=samplesdb,
username=username,
password=password,
url=url)
fc_con = FlowcellRunMetricsConnection(dbname=flowcelldb,
username=username,
password=password,
url=url)
p_con = ProjectSummaryConnection(dbname=projectdb,
username=username,
password=password,
url=url)
# Set up paragraphs
paragraphs = sample_note_paragraphs()
headers = sample_note_headers()
# Get project
project = p_con.get_entry(project_name)
if not project:
LOG.warn("No such project '{}'".format(project_name))
return output_data
# Set samples list
sample_run_list = _set_sample_run_list(project_name, flowcell,
project_alias, s_con)
if len(sample_run_list) == 0:
LOG.warn(
"No samples for project '{}', flowcell '{}'. Maybe there are no sample run metrics in statusdb?"
.format(project_name, flowcell))
return output_data
# Set options
ordered_million_reads = _literal_eval_option(ordered_million_reads)
bc_count = _literal_eval_option(bc_count)
phix = _literal_eval_option(phix)
# Count number of times a sample has been run on a flowcell; if several, make lane-specific reports
sample_count = Counter([x.get("barcode_name") for x in sample_run_list])
# Loop samples and collect information
s_param_out = []
for s in sample_run_list:
s_param = {}
LOG.debug(
"working on sample '{}', sample run metrics name '{}', id '{}'".
format(s.get("barcode_name", None), s.get("name", None),
s.get("_id", None)))
s_param.update(parameters)
s_param.update(
{key: s[srm_to_parameter[key]]
for key in srm_to_parameter.keys()})
fc = "{}_{}".format(s.get("date"), s.get("flowcell"))
# Get instrument
try:
s_param.update(instrument[fc_con.get_instrument(str(fc))])
except:
LOG.warn(
"Failed to set instrument and software versions for flowcell {} in report due to missing RunInfo -> Instrument field in statusdb. Either rerun 'pm qc update-qc' or search-and-replace 'NN' in the sample report."
.format(fc))
s_param.update(instrument['default'])
# Get run mode
s_param["run_mode"] = fc_con.get_run_mode(str(fc))
s_param.update(software_versions)
s_param["phix_error_rate"] = fc_con.get_phix_error_rate(
str(fc), s["lane"])
if phix:
s_param["phix_error_rate"] = _get_phix_error_rate(s["lane"], phix)
s_param['avg_quality_score'] = calc_avg_qv(s)
if not s_param['avg_quality_score']:
LOG.warn(
"Calculation of average quality failed for sample {}, id {}".
format(s.get("name"), s.get("_id")))
# Compare phix error and qv to cutoffs
err_stat = "OK"
qv_stat = "OK"
if s_param["phix_error_rate"] > cutoffs["phix_err_cutoff"]:
err_stat = "HIGH"
elif s_param["phix_error_rate"] == -1:
err_stat = "N/A"
if s_param["avg_quality_score"] < cutoffs["qv_cutoff"]:
qv_stat = "LOW"
output_data["stdout"].write(
"{:>18}\t{:>6}\t{:>12}\t{:>12}\t{:>12}\t{:>12}\n".format(
s["barcode_name"], s["lane"], s_param["phix_error_rate"],
err_stat, s_param["avg_quality_score"], qv_stat))
# Update/set remaning sample run parameters, falling back on project defaults if *key* is missing
s_param['ordered_amount'] = s_param.get(
'ordered_amount', p_con.get_ordered_amount(project_name))
s_param['customer_reference'] = s_param.get(
'customer_reference', project.get('customer_reference'))
s_param['uppnex_project_id'] = s_param.get('uppnex_project_id',
project.get('uppnex_id'))
# Override database settings if options passed at command line
if ordered_million_reads:
s_param["ordered_amount"] = _get_ordered_million_reads(
s["barcode_name"], ordered_million_reads)
if bc_count:
s_param["rounded_read_count"] = _round_read_count_in_millions(
_get_bc_count(s["barcode_name"], bc_count, s))
else:
s_param["rounded_read_count"] = _round_read_count_in_millions(
s_param["rounded_read_count"])
if uppnex_id:
s_param["uppnex_project_id"] = uppnex_id
if customer_reference:
s_param["customer_reference"] = customer_reference
# Get the project sample name corresponding to the sample run
project_sample = p_con.get_project_sample(
project_name, s.get("project_sample_name", None))
if project_sample:
LOG.debug(
"project sample run metrics mapping found: '{}' : '{}'".format(
s["name"], project_sample["sample_name"]))
project_sample_item = project_sample['project_sample']
# Set project_sample_d: a dictionary mapping from sample run metrics name to sample run metrics database id
project_sample_d = _set_project_sample_dict(project_sample_item)
if not project_sample_d:
LOG.warn(
"No sample_run_metrics information for sample '{}', barcode name '{}', id '{}'\n\tProject summary information {}"
.format(s["name"], s["barcode_name"], s["_id"],
project_sample))
# Check if sample run metrics name present in project database: if so, verify that database ids are consistent
if s["name"] not in project_sample_d.keys():
LOG.warn(
"no such sample run metrics '{}' in project sample run metrics dictionary"
.format(s["name"]))
else:
if s["_id"] == project_sample_d[s["name"]]:
LOG.debug(
"project sample run metrics mapping found: '{}' : '{}'"
.format(s["name"], project_sample_d[s["name"]]))
else:
LOG.warn(
"inconsistent mapping for '{}': '{}' != '{}' (project summary id)"
.format(s["name"], s["_id"],
project_sample_d[s["name"]]))
s_param['customer_name'] = project_sample_item.get(
"customer_name", None)
# No project sample found. Manual upload to database necessary.
else:
s_param['customer_name'] = None
LOG.warn(
"No project sample name found for sample run name '{}'".format(
s["barcode_name"]))
LOG.info(
"Please run 'pm qc upload-qc FLOWCELL_ID --extensive-matching' to update project sample names "
)
LOG.info(
"or 'pm qc update --sample_prj PROJECT_NAME --names BARCODE_TO_SAMPLE_MAP to update project sample names."
)
LOG.info("Please refer to the pm documentation for examples.")
query_ok(force=kw.get("force", False))
# Finally assess sequencing success, update parameters and set outputs
s_param['success'] = sequencing_success(s_param, cutoffs)
s_param.update({
k: "N/A"
for k in s_param.keys()
if s_param[k] is None or s_param[k] == "" or s_param[k] == -1.0
})
if sample_count[s.get("barcode_name")] > 1:
outfile = "{}_{}_{}_{}.pdf".format(s["barcode_name"], s["date"],
s["flowcell"], s["lane"])
else:
outfile = "{}_{}_{}.pdf".format(s["barcode_name"], s["date"],
s["flowcell"])
s_param["outfile"] = outfile
s_param_out.append(s_param)
# Write final output to reportlab and rst files
output_data["debug"].write(
json.dumps({
's_param': s_param_out,
'sample_runs':
{s["name"]: s["barcode_name"]
for s in sample_run_list}
}))
notes = [
make_note(headers=headers, paragraphs=paragraphs, **sp)
for sp in s_param_out
]
rest_notes = make_sample_rest_notes(
"{}_{}_{}_sample_summary.rst".format(project_name, s.get("date", None),
s.get("flowcell", None)),
s_param_out)
concatenate_notes(
notes, "{}_{}_{}_sample_summary.pdf".format(project_name,
s.get("date", None),
s.get("flowcell", None)))
return output_data
def project_status_note(project_name=None, username=None, password=None, url=None,
use_ps_map=True, use_bc_map=False, check_consistency=False,
ordered_million_reads=None, uppnex_id=None, customer_reference=None,
exclude_sample_ids={}, project_alias=None, sample_aliases={},
projectdb="projects", samplesdb="samples", flowcelldb="flowcells",
include_all_samples=False, **kw):
"""Make a project status note. Used keywords:
:param project_name: project name
:param user: db user name
:param password: db password
:param url: db url
:param use_ps_map: use project summary mapping
:param use_bc_map: use project to barcode name mapping
:param check_consistency: check consistency between mappings
:param ordered_million_reads: number of ordered reads in millions
:param uppnex_id: the uppnex id
:param customer_reference: customer project name
:param exclude_sample_ids: exclude some sample ids from project note
:param project_alias: project alias name
:param sample_aliases: sample alias names
:param projectdb: project db name
:param samplesdb: samples db name
:param flowcelldb: flowcells db name
:param include_all_samples: include all samples in report
"""
# parameters
parameters = {
"project_name" : project_name,
"finished" : "Not finished, or cannot yet assess if finished.",
}
# mapping project_summary to parameter keys
ps_to_parameter = {"scilife_name":"scilife_name", "customer_name":"customer_name", "project_name":"project_name"}
# mapping project sample to table
table_keys = ['ScilifeID', 'CustomerID', 'BarcodeSeq', 'MSequenced', 'MOrdered', 'Status']
output_data = {'stdout':StringIO(), 'stderr':StringIO(), 'debug':StringIO()}
# Connect and run
s_con = SampleRunMetricsConnection(dbname=samplesdb, username=username, password=password, url=url)
fc_con = FlowcellRunMetricsConnection(dbname=flowcelldb, username=username, password=password, url=url)
p_con = ProjectSummaryConnection(dbname=projectdb, username=username, password=password, url=url)
# Set report paragraphs
paragraphs = project_note_paragraphs()
headers = project_note_headers()
# Set local param variable
param = parameters
# Get project summary from project database
sample_aliases = _literal_eval_option(sample_aliases, default={})
prj_summary = p_con.get_entry(project_name)
if not prj_summary:
LOG.warn("No such project '{}'".format(project_name))
return
LOG.debug("Working on project '{}'.".format(project_name))
# Get sample run list and loop samples to make mapping sample -> {sampleruns}
sample_run_list = _set_sample_run_list(project_name, flowcell=None, project_alias=project_alias, s_con=s_con)
samples = {}
for s in sample_run_list:
prj_sample = p_con.get_project_sample(project_name, s.get("project_sample_name", None))
if prj_sample:
sample_name = prj_sample['project_sample'].get("scilife_name", None)
s_d = {s["name"] : {'sample':sample_name, 'id':s["_id"]}}
samples.update(s_d)
else:
if s["barcode_name"] in sample_aliases:
s_d = {sample_aliases[s["barcode_name"]] : {'sample':sample_aliases[s["barcode_name"]], 'id':s["_id"]}}
samples.update(s_d)
else:
s_d = {s["name"]:{'sample':s["name"], 'id':s["_id"], 'barcode_name':s["barcode_name"]}}
LOG.warn("No mapping found for sample run:\n '{}'".format(s_d))
# Convert to mapping from desired sample name to list of aliases
# Less important for the moment; one solution is to update the
# Google docs summary table to use the P names
sample_dict = prj_summary['samples']
param.update({key:prj_summary.get(ps_to_parameter[key], None) for key in ps_to_parameter.keys()})
param["ordered_amount"] = param.get("ordered_amount", p_con.get_ordered_amount(project_name))
param['customer_reference'] = param.get('customer_reference', prj_summary.get('customer_reference'))
param['uppnex_project_id'] = param.get('uppnex_project_id', prj_summary.get('uppnex_id'))
# Override database values if options passed at command line
if uppnex_id:
param["uppnex_project_id"] = uppnex_id
if customer_reference:
param["customer_reference"] = customer_reference
# Process options
ordered_million_reads = _literal_eval_option(ordered_million_reads)
exclude_sample_ids = _literal_eval_option(exclude_sample_ids, default={})
## Start collecting the data
sample_table = []
samples_excluded = []
all_passed = True
last_library_preps = p_con.get_latest_library_prep(project_name)
last_library_preps_srm = [x for l in last_library_preps.values() for x in l]
LOG.debug("Looping through sample map that maps project sample names to sample run metrics ids")
for k,v in samples.items():
LOG.debug("project sample '{}' maps to '{}'".format(k, v))
if not include_all_samples:
if v['sample'] not in last_library_preps.keys():
LOG.info("No library prep information for sample {}; keeping in report".format(v['sample']))
else:
if k not in last_library_preps_srm:
LOG.info("Sample run {} ('{}') is not latest library prep ({}) for project sample {}: excluding from report".format(k, v["id"], last_library_preps[v['sample']].values()[0], v['sample']))
continue
else:
pass
if re.search("Unexpected", k):
continue
barcode_seq = s_con.get_entry(k, "sequence")
# Exclude sample id?
if _exclude_sample_id(exclude_sample_ids, v['sample'], barcode_seq):
samples_excluded.append(v['sample'])
continue
# Get the project sample name from the sample run and set table values
project_sample = sample_dict[v['sample']]
vals = _set_sample_table_values(v['sample'], project_sample, barcode_seq, ordered_million_reads, param)
if vals['Status']=="N/A" or vals['Status']=="NP": all_passed = False
sample_table.append([vals[k] for k in table_keys])
# Loop through samples in sample_dict for which there is no sample run information
samples_in_table_or_excluded = list(set([x[0] for x in sample_table])) + samples_excluded
samples_not_in_table = list(set(sample_dict.keys()) - set(samples_in_table_or_excluded))
for sample in samples_not_in_table:
if re.search("Unexpected", sample):
continue
project_sample = sample_dict[sample]
# Set project_sample_d: a dictionary mapping from sample run metrics name to sample run metrics database id
project_sample_d = _set_project_sample_dict(project_sample)
if project_sample_d:
for k,v in project_sample_d.iteritems():
barcode_seq = s_con.get_entry(k, "sequence")
vals = _set_sample_table_values(sample, project_sample, barcode_seq, ordered_million_reads, param)
if vals['Status']=="N/A" or vals['Status']=="NP": all_passed = False
sample_table.append([vals[k] for k in table_keys])
else:
barcode_seq = None
vals = _set_sample_table_values(sample, project_sample, barcode_seq, ordered_million_reads, param)
if vals['Status']=="N/A" or vals['Status']=="NP": all_passed = False
sample_table.append([vals[k] for k in table_keys])
if all_passed: param["finished"] = 'Project finished.'
sample_table.sort()
sample_table = list(sample_table for sample_table,_ in itertools.groupby(sample_table))
sample_table.insert(0, ['ScilifeID', 'CustomerID', 'BarcodeSeq', 'MSequenced', 'MOrdered', 'Status'])
paragraphs["Samples"]["tpl"] = make_sample_table(sample_table)
make_note("{}_project_summary.pdf".format(project_name), headers, paragraphs, **param)
make_rest_note("{}_project_summary.rst".format(project_name), sample_table=sample_table, report="project_report", **param)
param.update({k:"N/A" for k in param.keys() if param[k] is None or param[k] == ""})
output_data["debug"].write(json.dumps({'param':param, 'table':sample_table}))
return output_data
def sample_status_note(project_name=None, flowcell=None, username=None, password=None, url=None,
ordered_million_reads=None, uppnex_id=None, customer_reference=None, bc_count=None,
project_alias=[], projectdb="projects", samplesdb="samples", flowcelldb="flowcells",
phix=None, **kw):
"""Make a sample status note. Used keywords:
:param project_name: project name
:param flowcell: flowcell id
:param username: db username
:param password: db password
:param url: db url
:param ordered_million_reads: number of ordered reads in millions
:param uppnex_id: the uppnex id
:param customer_reference: customer project name
:param project_alias: project alias name
:param phix: phix error rate
"""
# Cutoffs
cutoffs = {
"phix_err_cutoff" : 2.0,
"qv_cutoff" : 30,
}
# parameters
parameters = {
"project_name" : None,
"start_date" : None,
"FC_id" : None,
"scilifelab_name" : None,
"rounded_read_count" : None,
"phix_error_rate" : None,
"avg_quality_score" : None,
"success" : None,
"run_mode":None,
}
# key mapping from sample_run_metrics to parameter keys
srm_to_parameter = {"project_name":"sample_prj", "FC_id":"flowcell",
"scilifelab_name":"barcode_name", "start_date":"date", "rounded_read_count":"bc_count"}
LOG.debug("got parameters {}".format(parameters))
output_data = {'stdout':StringIO(), 'stderr':StringIO(), 'debug':StringIO()}
if not _assert_flowcell_format(flowcell):
LOG.warn("Wrong flowcell format {}; skipping. Please use the flowcell id (format \"[A-Z0-9]+XX\")".format(flowcell) )
return output_data
output_data = _update_sample_output_data(output_data, cutoffs)
# Connect and run
s_con = SampleRunMetricsConnection(dbname=samplesdb, username=username, password=password, url=url)
fc_con = FlowcellRunMetricsConnection(dbname=flowcelldb, username=username, password=password, url=url)
p_con = ProjectSummaryConnection(dbname=projectdb, username=username, password=password, url=url)
# Set up paragraphs
paragraphs = sample_note_paragraphs()
headers = sample_note_headers()
# Get project
project = p_con.get_entry(project_name)
if not project:
LOG.warn("No such project '{}'".format(project_name))
return output_data
# Set samples list
sample_run_list = _set_sample_run_list(project_name, flowcell, project_alias, s_con)
if len(sample_run_list) == 0:
LOG.warn("No samples for project '{}', flowcell '{}'. Maybe there are no sample run metrics in statusdb?".format(project_name, flowcell))
return output_data
# Set options
ordered_million_reads = _literal_eval_option(ordered_million_reads)
bc_count = _literal_eval_option(bc_count)
phix = _literal_eval_option(phix)
# Count number of times a sample has been run on a flowcell; if several, make lane-specific reports
sample_count = Counter([x.get("barcode_name") for x in sample_run_list])
# Loop samples and collect information
s_param_out = []
for s in sample_run_list:
s_param = {}
LOG.debug("working on sample '{}', sample run metrics name '{}', id '{}'".format(s.get("barcode_name", None), s.get("name", None), s.get("_id", None)))
s_param.update(parameters)
s_param.update({key:s[srm_to_parameter[key]] for key in srm_to_parameter.keys()})
fc = "{}_{}".format(s.get("date"), s.get("flowcell"))
# Get instrument
try:
s_param.update(instrument[fc_con.get_instrument(str(fc))])
except:
LOG.warn("Failed to set instrument and software versions for flowcell {} in report due to missing RunInfo -> Instrument field in statusdb. Either rerun 'pm qc update-qc' or search-and-replace 'NN' in the sample report.".format(fc))
s_param.update(instrument['default'])
# Get run mode
s_param["run_mode"] = fc_con.get_run_mode(str(fc))
s_param.update(software_versions)
s_param["phix_error_rate"] = fc_con.get_phix_error_rate(str(fc), s["lane"])
if phix:
s_param["phix_error_rate"] = _get_phix_error_rate(s["lane"], phix)
s_param['avg_quality_score'] = calc_avg_qv(s)
if not s_param['avg_quality_score']:
LOG.warn("Calculation of average quality failed for sample {}, id {}".format(s.get("name"), s.get("_id")))
# Compare phix error and qv to cutoffs
err_stat = "OK"
qv_stat = "OK"
if s_param["phix_error_rate"] > cutoffs["phix_err_cutoff"]:
err_stat = "HIGH"
elif s_param["phix_error_rate"] == -1:
err_stat = "N/A"
if s_param["avg_quality_score"] < cutoffs["qv_cutoff"]:
qv_stat = "LOW"
output_data["stdout"].write("{:>18}\t{:>6}\t{:>12}\t{:>12}\t{:>12}\t{:>12}\n".format(s["barcode_name"], s["lane"], s_param["phix_error_rate"], err_stat, s_param["avg_quality_score"], qv_stat))
# Update/set remaning sample run parameters, falling back on project defaults if *key* is missing
s_param['ordered_amount'] = s_param.get('ordered_amount', p_con.get_ordered_amount(project_name))
s_param['customer_reference'] = s_param.get('customer_reference', project.get('customer_reference'))
s_param['uppnex_project_id'] = s_param.get('uppnex_project_id', project.get('uppnex_id'))
# Override database settings if options passed at command line
if ordered_million_reads:
s_param["ordered_amount"] = _get_ordered_million_reads(s["barcode_name"], ordered_million_reads)
if bc_count:
s_param["rounded_read_count"] = _round_read_count_in_millions(_get_bc_count(s["barcode_name"], bc_count, s))
else:
s_param["rounded_read_count"] = _round_read_count_in_millions(s_param["rounded_read_count"])
if uppnex_id:
s_param["uppnex_project_id"] = uppnex_id
if customer_reference:
s_param["customer_reference"] = customer_reference
# Get the project sample name corresponding to the sample run
project_sample = p_con.get_project_sample(project_name, s.get("project_sample_name", None))
if project_sample:
LOG.debug("project sample run metrics mapping found: '{}' : '{}'".format(s["name"], project_sample["sample_name"]))
project_sample_item = project_sample['project_sample']
# Set project_sample_d: a dictionary mapping from sample run metrics name to sample run metrics database id
project_sample_d = _set_project_sample_dict(project_sample_item)
if not project_sample_d:
LOG.warn("No sample_run_metrics information for sample '{}', barcode name '{}', id '{}'\n\tProject summary information {}".format(s["name"], s["barcode_name"], s["_id"], project_sample))
# Check if sample run metrics name present in project database: if so, verify that database ids are consistent
if s["name"] not in project_sample_d.keys():
LOG.warn("no such sample run metrics '{}' in project sample run metrics dictionary".format(s["name"]) )
else:
if s["_id"] == project_sample_d[s["name"]]:
LOG.debug("project sample run metrics mapping found: '{}' : '{}'".format(s["name"], project_sample_d[s["name"]]))
else:
LOG.warn("inconsistent mapping for '{}': '{}' != '{}' (project summary id)".format(s["name"], s["_id"], project_sample_d[s["name"]]))
s_param['customer_name'] = project_sample_item.get("customer_name", None)
# No project sample found. Manual upload to database necessary.
else:
s_param['customer_name'] = None
LOG.warn("No project sample name found for sample run name '{}'".format(s["barcode_name"]))
LOG.info("Please run 'pm qc upload-qc FLOWCELL_ID --extensive-matching' to update project sample names ")
LOG.info("or 'pm qc update --sample_prj PROJECT_NAME --names BARCODE_TO_SAMPLE_MAP to update project sample names.")
LOG.info("Please refer to the pm documentation for examples.")
query_ok(force=kw.get("force", False))
# Finally assess sequencing success, update parameters and set outputs
s_param['success'] = sequencing_success(s_param, cutoffs)
s_param.update({k:"N/A" for k in s_param.keys() if s_param[k] is None or s_param[k] == "" or s_param[k] == -1.0})
if sample_count[s.get("barcode_name")] > 1:
outfile = "{}_{}_{}_{}.pdf".format(s["barcode_name"], s["date"], s["flowcell"], s["lane"])
else:
outfile = "{}_{}_{}.pdf".format(s["barcode_name"], s["date"], s["flowcell"])
s_param["outfile"] = outfile
s_param_out.append(s_param)
# Write final output to reportlab and rst files
output_data["debug"].write(json.dumps({'s_param': s_param_out, 'sample_runs':{s["name"]:s["barcode_name"] for s in sample_run_list}}))
notes = [make_note(headers=headers, paragraphs=paragraphs, **sp) for sp in s_param_out]
rest_notes = make_sample_rest_notes("{}_{}_{}_sample_summary.rst".format(project_name, s.get("date", None), s.get("flowcell", None)), s_param_out)
concatenate_notes(notes, "{}_{}_{}_sample_summary.pdf".format(project_name, s.get("date", None), s.get("flowcell", None)))
return output_data
def test_2_make_project_note(self):
"""Make a project note subset by flowcell and project"""
s_con = SampleRunMetricsConnection(username=self.user,
password=self.pw,
url=self.url)
fc_con = FlowcellRunMetricsConnection(username=self.user,
password=self.pw,
url=self.url)
p_con = ProjectSummaryConnection(username=self.user,
password=self.pw,
url=self.url)
paragraphs = project_note_paragraphs()
headers = project_note_headers()
param = parameters
project = p_con.get_entry(self.examples["project"])
if not project:
print "No project named {}".format(self.examples["project"])
return
if project:
ordered_amount = p_con.get_ordered_amount(self.examples["project"])
else:
return
ordered_amount = self.pargs.ordered_million_reads
## Start collecting the data
sample_table = []
sample_list = project['samples']
param.update({
key: project.get(ps_to_parameter[key], None)
for key in ps_to_parameter.keys()
})
samples = p_con.map_name_to_srm(self.examples["project"],
check_consistency=True,
use_bc_map=True)
all_passed = True
for k, v in samples.items():
if k == "Unexpected":
continue
project_sample = sample_list[k]
vals = {
x: project_sample.get(prjs_to_table[x], None)
for x in prjs_to_table.keys()
}
vals['MOrdered'] = ordered_amount
vals['BarcodeSeq'] = s_con.get_entry(v.keys()[0], "sequence")
## Set status
vals['Status'] = set_status(
vals) if vals['Status'] is None else vals['Status']
vals.update({k: "N/A" for k in vals.keys() if vals[k] is None})
if vals['Status'] == "N/A" or vals['Status'] == "NP":
all_passed = False
sample_table.append([vals[k] for k in table_keys])
if all_passed: param["finished"] = 'Project finished.'
sample_table.sort()
sample_table = list(
sample_table
for sample_table, _ in itertools.groupby(sample_table))
sample_table.insert(0, [
'ScilifeID', 'CustomerID', 'BarcodeSeq', 'MSequenced', 'MOrdered',
'Status'
])
paragraphs["Samples"]["tpl"] = make_sample_table(sample_table)
make_note("{}.pdf".format(self.examples["project"]), headers,
paragraphs, **param)
