def get_and_set_fluor_int(self, target_file):
"""Copies "End RFU" values from "Quant-iT Result File 1" and
"Quant-iT Result File 2" (if provided) to udfs "Fluorescence intensity 1"
and "Fluorescence intensity 2. Calculates and returns Relative
fluorescence intensity standards:
rel_fluor_int = The End RFU of standards - Background fluorescence
intensity"""
sample = target_file.samples[0].name
fluor_int = []
target_udfs = target_file.udf
# For the moment we dont know ofa way to delete udfs. Should be solved.
#if dict(target_udfs.items()).has_key('Fluorescence intensity 1'):
# del target_udfs['Fluorescence intensity 1']
#if dict(target_udfs.items()).has_key('Fluorescence intensity 2'):
# del target_udfs['Fluorescence intensity 2']
target_file.udf = target_udfs
for udf_name, formated_file in self.result_files.items():
if sample in formated_file.keys():
fluor_int.append(float(formated_file[sample]['End RFU']))
target_file.udf[udf_name] = float(
formated_file[sample]['End RFU'])
else:
self.missing_samps.append(sample)
set_field(target_file)
rel_fluor_int = np.mean(fluor_int) - self.standards[1]
return rel_fluor_int
def get_and_set_fluor_int(self, target_file):
"""Copies "End RFU" values from "Quant-iT Result File 1" and
"Quant-iT Result File 2" (if provided) to udfs "Fluorescence intensity 1"
and "Fluorescence intensity 2. Calculates and returns Relative
fluorescence intensity standards:
rel_fluor_int = The End RFU of standards - Background fluorescence
intensity"""
sample = target_file.samples[0].name
fluor_int = []
target_udfs = target_file.udf
# For the moment we dont know ofa way to delete udfs. Should be solved.
#if dict(target_udfs.items()).has_key('Fluorescence intensity 1'):
# del target_udfs['Fluorescence intensity 1']
#if dict(target_udfs.items()).has_key('Fluorescence intensity 2'):
# del target_udfs['Fluorescence intensity 2']
target_file.udf = target_udfs
for udf_name ,formated_file in self.result_files.items():
if sample in formated_file.keys():
fluor_int.append(float(formated_file[sample]['End RFU']))
target_file.udf[udf_name] = float(formated_file[sample]['End RFU'])
else:
self.missing_samps.append(sample)
set_field(target_file)
rel_fluor_int = np.mean(fluor_int) - self.standards[1]
return rel_fluor_int
def _set_udfs(self, samp_name, target_file):
if samp_name in self.QF_from_file.keys():
s_inf = self.QF_from_file[samp_name]
target_file.udf['# Reads'] = int(s_inf['# Reads'])
target_file.udf['% Bases >=Q30'] = float(s_inf['% Bases >=Q30'])
self.nr_samps_updat += 1
else:
self.missing_samps.append(samp_name)
set_field(target_file)
def get_qbit_csv_data(process):
#samples missing from the qubit csv file
missing_samples = 0
low_conc = 0
bad_format = 0
#strings returned to the EPP user
log = []
#get file path by parsing lims artifacts
file_path=get_qbit_file(process)
#parse the qubit file and get the interesting data out
data=get_data(file_path, log)
if "Minimum required concentration (ng/ul)" in process.udf:
min_conc=process.udf['Minimum required concentration (ng/ul)']
else:
min_conc=None
log.append("Set 'Minimum required concentration (ng/ul)' to get qc-flags based on this threshold!")
for target_file in process.result_files():
conc=None
new_conc=None
file_sample=target_file.samples[0].name
if file_sample in data:
try:
conc=float(data[file_sample]['concentration'])
except ValueError:
#concentration is not a float
target_file.qc_flag = "FAILED"
if data[file_sample]['concentration'] != 'Out of range':
#Out of range is a valid value, the others are not.
bad_format+=1
else:
new_conc=convert_to_ng_ul(conc, data[file_sample]['unit'])
if new_conc is not None :
target_file.udf['Concentration'] = new_conc
target_file.udf['Conc. Units'] = 'ng/ul'
if new_conc < min_conc:
target_file.qc_flag = "FAILED"
low_conc +=1
else:
target_file.qc_flag = "PASSED"
#actually set the data
target_file.put()
set_field(target_file)
else:
missing_samples += 1
if low_conc:
log.append('{0}/{1} samples have low concentration.'.format(low_conc, len(process.result_files())))
if missing_samples:
log.append('{0}/{1} samples are missing in the Qubit Result File.'.format(missing_samples, len(process.result_files())))
if bad_format:
log.append('There are {0} badly formatted samples in Qubit Result File. Please fix these to get proper results.'.format(bad_format))
print(''.join(log), file=sys.stderr)
def main(lims, pid, epp_logger):
process = Process(lims,id = pid)
sample_names = map(lambda a: a.name, process.analytes()[0])
target_files = process.result_files()
file_handler = ReadResultFiles(process)
files = file_handler.shared_files['Qubit Result File']
qubit_result_file = file_handler.format_file(files,
name = 'Qubit Result File',
first_header = 'Sample',
find_keys = sample_names)
missing_samples = 0
low_conc = 0
bad_formated = 0
abstract = []
udfs = dict(process.udf.items())
if udfs.has_key("Minimum required concentration (ng/ul)"):
min_conc = udfs["Minimum required concentration (ng/ul)"]
else:
min_conc = None
abstract.append("Set 'Minimum required concentration (ng/ul)' to get qc-flaggs based on this treshold!")
for target_file in target_files:
sample = target_file.samples[0].name
if qubit_result_file.has_key(sample):
sample_mesurements = qubit_result_file[sample]
if "Sample Concentration" in sample_mesurements.keys():
conc, unit = sample_mesurements["Sample Concentration"]
if conc == 'Out Of Range':
target_file.qc_flag = "FAILED"
elif conc.replace('.','').isdigit():
conc = float(conc)
if unit == 'ng/mL':
conc = np.true_divide(conc, 1000)
if min_conc:
if conc < min_conc:
target_file.qc_flag = "FAILED"
low_conc +=1
else:
target_file.qc_flag = "PASSED"
target_file.udf['Concentration'] = conc
target_file.udf['Conc. Units'] = 'ng/ul'
else:
bad_formated += 1
set_field(target_file)
else:
missing_samples += 1
if low_conc:
abstract.append('{0}/{1} samples have low concentration.'.format(low_conc, len(target_files)))
if missing_samples:
abstract.append('{0}/{1} samples are missing in Qubit Result File.'.format(missing_samples, len(target_files)))
if bad_formated:
abstract.append('There are {0} badly formated samples in Qubit Result File. Please fix these to get proper results.'.format(bad_formated))
print >> sys.stderr, ' '.join(abstract)
def set_result_file_udfs(self):
"""populates the target file App QC udf"""
for samp_name, target_file in self.target_files.items():
if samp_name in self.app_QC.keys():
qc_passed = str(self.app_QC[samp_name]['automated_qc']['qc_passed'])
sample = target_file.samples[0]
sample.udf['App QC'] = qc_passed
set_field(sample)
self.nr_samps_updat += 1
else:
self.missing_samps.append(samp_name)
def _set_udfs(self, samp_name, target_file, lane):
if lane in self.QF_from_file.keys():
if samp_name in self.QF_from_file[lane].keys():
s_inf = self.QF_from_file[lane][samp_name]
target_file.udf['# Reads'] = int(s_inf['# Reads'])
target_file.udf['% Bases >=Q30'] = float(
s_inf['% Bases >=Q30'])
self.nr_samps_updat.append(samp_name)
else:
self.missing_samps.append(samp_name)
set_field(target_file)
def set_qc(self):
if self.amount_treshold:
for artifact in self.correct_artifacts:
if 'Amount (ng)' in artifact.udf and artifact.udf[
'Amount (ng)']:
if artifact.udf['Amount (ng)'] < self.amount_treshold:
artifact.qc_flag = "FAILED"
self.low_conc += 1
else:
artifact.qc_flag = "PASSED"
artifact.put()
set_field(artifact)
def assign_QC_flag(self):
if self.required_udfs.issubset(self.udfs.keys()):
for result_file in self.result_files:
result_file_udfs = dict(result_file.udf.items())
QC_conc = self.concentration_QC(result_file, result_file_udfs)
QC_sat = self.saturation_QC(result_file, result_file_udfs)
if QC_conc and QC_sat:
QC = QC_conc if QC_conc == QC_sat else "FAILED"
self.no_failed += 1 if QC == "FAILED" else 0
result_file.qc_flag = QC
set_field(result_file)
else:
self.missing_udfs = ', '.join(list(self.required_udfs))
def assign_QC_flag(self):
if self.required_udfs.issubset(self.udfs.keys()):
for result_file in self.result_files:
result_file_udfs = dict(result_file.udf.items())
QC_conc = self.concentration_QC(result_file, result_file_udfs)
QC_sat = self.saturation_QC(result_file, result_file_udfs)
if QC_conc and QC_sat:
QC = QC_conc if QC_conc == QC_sat else "FAILED"
self.no_failed +=1 if QC == "FAILED" else 0
result_file.qc_flag = QC
set_field(result_file)
else:
self.missing_udfs = ', '.join(list(self.required_udfs))
def set_qc(self):
if self.conc_treshold:
for artifact in self.correct_artifacts:
if 'Concentration' in artifact.udf and artifact.udf[
'Concentration']:
if artifact.udf['Concentration'] < float(
self.conc_treshold):
artifact.qc_flag = "FAILED"
self.low_conc += 1
else:
artifact.qc_flag = "PASSED"
artifact.put()
set_field(artifact)
def calc_and_set_conc(self, target_file, rel_fluor_int):
"""Concentrations are calculated based on the linear regression
parametersand copied to the "Concentration"-udf of the target_file. The
"Conc. Units"-udf is set to "ng/ul"""
requiered_udfs = set(['Sample volume','Standard dilution','WS volume'])
if requiered_udfs.issubset(self.udfs.keys()) and self.model:
conc = np.true_divide((self.model[1] * rel_fluor_int * (
self.udfs['WS volume'] + self.udfs['Sample volume'])),
self.udfs['Sample volume']*(self.udfs['WS volume'] +
self.udfs['Standard volume']))
target_file.udf['Concentration'] = conc
target_file.udf['Conc. Units'] = 'ng/ul'
set_field(target_file)
self.no_samps +=1
elif not requiered_udfs.issubset(self.missing_udfs):
self.missing_udfs += requiered_udfs
def calc_and_set_conc(self, target_file, rel_fluor_int):
"""Concentrations are calculated based on the linear regression
parametersand copied to the "Concentration"-udf of the target_file. The
"Conc. Units"-udf is set to "ng/ul"""
requiered_udfs = set(
['Sample volume', 'Standard dilution', 'WS volume'])
if requiered_udfs.issubset(self.udfs.keys()) and self.model:
conc = np.true_divide(
(self.model[1] * rel_fluor_int *
(self.udfs['WS volume'] + self.udfs['Sample volume'])),
self.udfs['Sample volume'] *
(self.udfs['WS volume'] + self.udfs['Standard volume']))
target_file.udf['Concentration'] = conc
target_file.udf['Conc. Units'] = 'ng/ul'
set_field(target_file)
self.no_samps += 1
elif not requiered_udfs.issubset(self.missing_udfs):
self.missing_udfs += requiered_udfs
def lane_QC(self):
for target_file in self.out_arts:
samp_name = target_file.samples[0].name
for lane_samp in self.BLS:
if self.lane == lane_samp['Lane']:
samp = lane_samp['Sample ID']
if samp == samp_name:
IQC = IndexQC(target_file, lane_samp)
IQC.set_target_file_udfs()
IQC.set_read_pairs(self.single)
try:
IQC.lane_index_QC(self.reads_threshold, self.Q30_treshold)
if IQC.html_file_error:
self.html_file_error = IQC.html_file_error
set_field(IQC.t_file)
self.nr_samps_updat +=1
except:
self.QC_fail.append(samp)
self._check_un_exp_lane_yield()
for index_count in self.counts:
self._check_un_exp_ind_yield(index_count)
def lane_QC(self):
for target_file in self.out_arts:
samp_name = target_file.samples[0].name
for lane_samp in self.BLS:
if self.lane == lane_samp['Lane']:
samp = lane_samp['Sample ID']
if samp == samp_name:
IQC = IndexQC(target_file, lane_samp)
IQC.set_target_file_udfs()
IQC.set_read_pairs(self.single)
try:
IQC.lane_index_QC(self.reads_threshold,
self.Q30_treshold)
if IQC.html_file_error:
self.html_file_error = IQC.html_file_error
set_field(IQC.t_file)
self.nr_samps_updat += 1
except:
self.QC_fail.append(samp)
self._check_un_exp_lane_yield()
for index_count in self.counts:
self._check_un_exp_ind_yield(index_count)
def get_qbit_csv_data(process):
#samples missing from the qubit csv file
missing_samples = 0
low_conc = 0
bad_format = 0
#strings returned to the EPP user
log = []
#get file path by parsing lims artifacts
file_path = get_qbit_file(process)
#parse the qubit file and get the interesting data out
data = get_data(file_path, log)
if "Minimum required concentration (ng/ul)" in process.udf:
min_conc = process.udf['Minimum required concentration (ng/ul)']
else:
min_conc = None
log.append(
"Set 'Minimum required concentration (ng/ul)' to get qc-flags based on this threshold!"
)
for target_file in process.result_files():
conc = None
new_conc = None
file_sample = target_file.samples[0].name
if file_sample in data:
try:
conc = float(data[file_sample]['concentration'])
except ValueError:
#concentration is not a float
target_file.qc_flag = "FAILED"
if data[file_sample]['concentration'] != 'Out of range':
#Out of range is a valid value, the others are not.
bad_format += 1
else:
new_conc = convert_to_ng_ul(conc, data[file_sample]['unit'])
if new_conc is not None:
target_file.udf['Concentration'] = new_conc
target_file.udf['Conc. Units'] = 'ng/ul'
if new_conc < min_conc:
target_file.qc_flag = "FAILED"
low_conc += 1
else:
target_file.qc_flag = "PASSED"
#actually set the data
target_file.put()
set_field(target_file)
else:
missing_samples += 1
if low_conc:
log.append('{0}/{1} samples have low concentration.'.format(
low_conc, len(process.result_files())))
if missing_samples:
log.append(
'{0}/{1} samples are missing in the Qubit Result File.'.format(
missing_samples, len(process.result_files())))
if bad_format:
log.append(
'There are {0} badly formatted samples in Qubit Result File. Please fix these to get proper results.'
.format(bad_format))
print(''.join(log), file=sys.stderr)
def _get_run_id(self):
if self.run_udfs.has_key('Run ID'):
self.process.udf['Run ID'] = self.run_udfs['Run ID']
set_field(self.process)
def _get_run_id(self):
if self.run_udfs.has_key('Run ID'):
self.process.udf['Run ID'] = self.run_udfs['Run ID']
set_field(self.process)
def old_main(lims, pid, epp_logger):
process = Process(lims, id=pid)
sample_names = map(lambda a: a.name, process.analytes()[0])
target_files = process.result_files()
file_handler = ReadResultFiles(process)
files = file_handler.shared_files['Qubit Result File']
qubit_result_file = file_handler.format_file(
files,
name='Qubit Result File',
first_header=['Test', 'Sample'],
find_keys=sample_names)
missing_samples = 0
low_conc = 0
bad_formated = 0
abstract = []
udfs = dict(process.udf.items())
if udfs.has_key("Minimum required concentration (ng/ul)"):
min_conc = udfs["Minimum required concentration (ng/ul)"]
else:
min_conc = None
abstract.append(
"Set 'Minimum required concentration (ng/ul)' to get qc-flaggs based on this treshold!"
)
for target_file in target_files:
sample = target_file.samples[0].name
if qubit_result_file.has_key(sample):
sample_mesurements = qubit_result_file[sample]
if "Sample Concentration" in sample_mesurements.keys():
conc, unit = sample_mesurements["Sample Concentration"]
if conc == 'Out Of Range':
target_file.qc_flag = "FAILED"
elif conc.replace('.', '').isdigit():
conc = float(conc)
if unit == 'ng/mL':
conc = np.true_divide(conc, 1000)
if min_conc:
if conc < min_conc:
target_file.qc_flag = "FAILED"
low_conc += 1
else:
target_file.qc_flag = "PASSED"
target_file.udf['Concentration'] = conc
target_file.udf['Conc. Units'] = 'ng/ul'
else:
bad_formated += 1
set_field(target_file)
else:
missing_samples += 1
if low_conc:
abstract.append('{0}/{1} samples have low concentration.'.format(
low_conc, len(target_files)))
if missing_samples:
abstract.append(
'{0}/{1} samples are missing in Qubit Result File.'.format(
missing_samples, len(target_files)))
if bad_formated:
abstract.append(
'There are {0} badly formated samples in Qubit Result File. Please fix these to get proper results.'
.format(bad_formated))
print >> sys.stderr, ' '.join(abstract)