def command(self):
app = self.load_wsgi_app()
storage = QLStorageSource(app.config)
if len(self.args) < 2:
print self.__class__.usage
return
for i in range(0,len(self.args)-1):
plate_id = int(self.args[i])
plate = dbplate_tree(plate_id)
try:
plate_path = storage.plate_path(plate)
except Exception, e:
print "Could not read plate: %s" % plate_id
continue
qlplate = get_plate(plate_path)
if not qlplate:
raise ValueError, "Could not read plate: %s" % plate_path
else:
print "Processing %s" % plate_path
try:
from qtools.lib.metrics import DEFAULT_CNV_CALC, compute_metric_foreach_qlwell
plate_metrics = plate.metrics[0]
compute_metric_foreach_qlwell(qlplate, plate_metrics, DEFAULT_CNV_CALC)
except Exception, e:
import sys, traceback
traceback.print_exc(file=sys.stdout)
Session.rollback()
return
def update_droplet_generators(self):
dgs = [DropletGenerator(name=u'DG #01'),
DropletGenerator(name=u'DG #02'),
DropletGenerator(name=u'DG #03'),
DropletGenerator(name=u'DG #04'),
DropletGenerator(name=u'DG #05'),
DropletGenerator(name=u'DG #06'),
DropletGenerator(name=u'DG #07'),
DropletGenerator(name=u'DG #08'),
DropletGenerator(name=u'DG #09'),
DropletGenerator(name=u'DG #10'),
DropletGenerator(name=u'DG #11'),
DropletGenerator(name=u'DG #12'),
DropletGenerator(name=u'DG #13'),
DropletGenerator(name=u'DG #14'),
DropletGenerator(name=u'DG #15'),
DropletGenerator(name=u'DG #16'),
DropletGenerator(name=u'DG #17'),
DropletGenerator(id=101, name=u'Pilot #1'),
DropletGenerator(id=102, name=u'Pilot #2'),
DropletGenerator(id=103, name=u'Pilot #3'),
DropletGenerator(id=104, name=u'Pilot #4'),
DropletGenerator(id=105, name=u'Pilot #5'),
DropletGenerator(id=201, name=u'Groove DG #1'),
DropletGenerator(id=202, name=u'Groove DG #2'),
DropletGenerator(id=203, name=u'Groove DG #3'),
DropletGenerator(id=204, name=u'Groove DG #4')]
for dg in dgs:
db_dg = Session.query(DropletGenerator).filter_by(name=dg.name).first()
if not db_dg:
Session.add(dg)
Session.commit()
def command(self):
app = self.load_wsgi_app()
# enforce config.ini
if len(self.args) < 2:
raise ValueError, self.__class__.usage
plate_id = int(self.args[0])
plate = dbplate_tree(plate_id)
if not plate:
raise ValueError, "Invalid plate id: %s" % plate_id
storage = QLStorageSource(app.config)
# for now, no reprocessing.
plate_path = storage.plate_path(plate)
qlplate = get_plate(plate_path)
if not qlplate:
raise ValueError, "Could not read plate: %s" % plate_path
try:
plate_metrics = plate.metrics[0]
self.process_plate( qlplate, plate_metrics)
except Exception, e:
import sys, traceback
traceback.print_exc(file=sys.stdout)
Session.rollback()
return
def retreive_and_validate_inputs( job, job_queue, logger):
"""
retreives and validates analysis group and reprocessor from a job
"""
struct = JSONMessage.unserialize(job.input_message)
## get IDs
analysis_group_id = struct.analysis_group_id
reprocess_config_id = struct.reprocess_config_id
analysis_group = Session.query(AnalysisGroup).get(analysis_group_id)
reprocessor = Session.query(ReprocessConfig).get(reprocess_config_id)
#check if analysis group exists in DB
if not analysis_group:
logger.error("Reprocess job: Analysis group id: %s not found[job %s]" % (analysis_group_id, job.id))
job_queue.abort(job, JSONErrorMessage("Unknown analysis group id: %s" % analysis_group_id))
#check if reprocessor exists in DB
if not reprocessor:
logger.error("Reprocess job: reprocessor id: %s not found[job %s]" % (reprocess_config_id, job.id))
job_queue.abort(job, JSONErrorMessage("Unknown reprocessor: %s" % reprocess_config_id))
return analysis_group, reprocessor
def process_plate(self, qlplate, plate_metric):
well_dict = plate_metric.well_metric_name_dict
# TODO move this into a metrics lib instead?
for name, qlwell in qlplate.analyzed_wells.items():
well_metric = well_dict[name]
for qwc, wcm in zip(qlwell.channels, well_metric.well_channel_metrics):
DEFAULT_NTC_POSITIVE_CALCULATOR.compute(qlwell, qwc, wcm)
Session.commit()
def process_plate(self, qlplate, plate_metric):
from qtools.lib.metrics import NEW_DROPLET_CLUSTER_METRICS_CALCULATOR, compute_metric_foreach_qlwell_channel
from qtools.lib.metrics import NEW_DROPLET_CLUSTER_WELL_METRICS_CALCULATOR, compute_metric_foreach_qlwell
compute_metric_foreach_qlwell_channel(qlplate, plate_metric, NEW_DROPLET_CLUSTER_METRICS_CALCULATOR)
compute_metric_foreach_qlwell( qlplate, plate_metric, NEW_DROPLET_CLUSTER_WELL_METRICS_CALCULATOR)
Session.commit()
def __call__(self, environ, start_response):
"""Invoke the Controller"""
# WSGIController.__call__ dispatches to the Controller method
# the request is routed to. This routing information is
# available in environ['pylons.routes_dict']
try:
return WSGIController.__call__(self, environ, start_response)
finally:
Session.remove()
def setup_buffers(self):
Session.add_all([Buffer(name=u"NEBuffer 1"),
Buffer(name=u"NEBuffer 2"),
Buffer(name=u"NEBuffer 3"),
Buffer(name=u"NEBuffer 4"),
Buffer(name=u'NEBuffer DpnII'),
Buffer(name=u'NEBuffer EcoRI'),
Buffer(name=u'NEBuffer SspI')])
Session.commit()
def update_dgs_used(self):
dgu = [DGUsed(name=u'Unicorn'),
DGUsed(name=u'Yeti'),
DGUsed(name=u'Nessie')]
for dgs in dgu:
db_dgu = Session.query(DGUsed).filter_by(name=dgs.name).first()
if not db_dgu:
Session.add(dgs)
Session.commit()
def command(self):
app = self.load_wsgi_app()
# enforce at least one plate
if len(self.args) < 2:
raise ValueError, self.__class__.usage
for i in range(0,len(self.args)-1):
print "Deleting plate #%s" % self.args[i]
delete_plate_recursive(int(self.args[i]))
Session.commit()
def command(self):
self.load_wsgi_app()
projects = [Project(name=u'Beta Test',code=u'Beta'),
Project(name=u'Validation',code=u'Validation')]
for project in projects:
dp = Session.query(Project).filter_by(name=project.name).first()
if not dp:
Session.add(project)
Session.commit()
def update_dr_groups(self):
dr_groups = [DRGroup(name='Apps DRs', active=True),
DRGroup(name='Groove DRs', active=True),
DRGroup(name='Golden DRs', active=True),
DRGroup(name='Rev Z DRs', active=True),
DRGroup(name='Custom DRs', active=True),
DRGroup(name='DNA 2.0', active=True),
DRGroup(name='Retired', active=False)]
for group in dr_groups:
drg = Session.query(DRGroup).filter_by(name=group.name).first()
if not drg:
Session.add(group)
Session.commit()
def command(self):
app = self.load_wsgi_app()
# enforce name.ini
if len(self.args) < 2:
raise ValueError, self.__class__.usage
name = self.args[0]
ag = AnalysisGroup(name=name)
Session.add(ag)
Session.commit()
print "Created analysis group %s with id=%s" % (ag.name, ag.id)
def command(self):
app = self.load_wsgi_app()
print "Clearing buffer tables..."
Session.execute('DELETE from vendor_enzyme')
Session.execute('DELETE from buffer')
Session.execute('DELETE from vendor')
Session.execute('DELETE from enzyme')
Session.commit()
self.setup_buffers()
self.setup_vendors()
def command(self):
self.load_wsgi_app()
Session.execute('DELETE from plate_tag_plate')
Session.execute('DELETE from plate_tag')
Session.commit()
Session.add_all([PlateTag(name=u'Clog'),
PlateTag(name=u'Failure'),
PlateTag(name=u'Highlight')])
Session.commit()
def command(self):
app = self.load_wsgi_app()
if len(self.args) > 1:
plate_id = int(self.args[0])
else:
raise Exception, "Not enough arguments: %s" % self.__class__.usage
plate = Session.query(Plate).get(plate_id)
if not plate:
raise Exception, "Could not find plate #: %s" % plate_id
plate_type = Session.query(PlateType).filter_by(code=u'bexp').first()
plate.plate_type = plate_type
Session.commit()
def update_physical_plates(self):
pps = [PhysicalPlate(name='Eppendorf twin.tec PCR plate 96'),
PhysicalPlate(name='Bio-Rad Hard-Shell White/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Red/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Black/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Yellow/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Blue/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Green/Clear')]
for pp in pps:
dbpp = Session.query(PhysicalPlate).filter_by(name=pp.name).first()
if not dbpp:
Session.add(pp)
Session.commit()
def recent_validation_plates(self, eagerload_plates=False):
"""
Returns the most recent tests for this spec.
"""
if eagerload_plates:
query = Session.query(ProductValidationPlate)\
.join(ProductValidationPlate.plate)\
.options(contains_eager(ProductValidationPlate.plate))
else:
query = Session.query(ProductValidationPlate).join(Plate)
query = query.filter(ProductValidationPlate.spec_id==self.id)\
.order_by(Plate.run_time.desc())
return query
def command(self):
self.load_wsgi_app()
Session.execute('DELETE from well_tag_qlbwell')
Session.execute('DELETE from well_tag')
Session.commit()
Session.add_all([WellTag(name=u'Clog'),
WellTag(name=u'NoClog Control'),
WellTag(name=u'Rain'),
WellTag(name=u'Negative Rain'),
WellTag(name=u'Bad Threshold'),
WellTag(name=u'Good Well')])
Session.commit()
def update_thermal_cyclers(self):
tcs = [ThermalCycler(name=u'Eppendorf 1'),
ThermalCycler(name=u'Eppendorf 2'),
ThermalCycler(name=u'Eppendorf 3'),
ThermalCycler(name=u'Eppendorf 4'),
ThermalCycler(name=u'Eppendorf 5'),
ThermalCycler(name=u'Eppendorf 6'),
ThermalCycler(name=u'Bio-Rad T100 1'),
ThermalCycler(name=u'Bio-Rad C1000 1'),
ThermalCycler(name=u'Eppendorf 8')]
for tc in tcs:
db_tc = Session.query(ThermalCycler).filter_by(name=tc.name).first()
if not db_tc:
Session.add(tc)
Session.commit()
def process_amplicon_job(job_queue, sequence_source, dg_calc):
remaining = job_queue.remaining(job_type=(JOB_ID_PROCESS_TAQMAN_AMPLICON,
JOB_ID_PROCESS_LOCATION_AMPLICON,
JOB_ID_PROCESS_SNP_AMPLICON,
JOB_ID_PROCESS_GEX_TAQMAN_TRANSCRIPT))
for job in remaining:
if job.type == JOB_ID_PROCESS_TAQMAN_AMPLICON:
process_taqman_job(job, job_queue, sequence_source, dg_calc)
elif job.type == JOB_ID_PROCESS_LOCATION_AMPLICON:
process_location_job(job, job_queue, sequence_source, dg_calc)
elif job.type == JOB_ID_PROCESS_SNP_AMPLICON:
process_snp_job(job, job_queue, sequence_source, dg_calc)
elif job.type == JOB_ID_PROCESS_GEX_TAQMAN_TRANSCRIPT:
process_transcript_job(job, job_queue, sequence_source, dg_calc)
Session.close()
def _to_python(self, value, state):
# value is gonna be a dict
for name, spec in self.spec.items():
obj, id_col, disp_col, additional_cols = spec
dict_val = value.get(name)
if not dict_val:
continue
if dict_val and dict_val.isdigit():
if Session.query(obj).filter(getattr(obj, id_col) == int(dict_val)).one():
continue
additional_cols[disp_col] = dict_val
newobj = obj(**additional_cols)
Session.add(newobj)
Session.commit()
value[name] = getattr(newobj, id_col)
return value
def command(self):
app = self.load_wsgi_app()
# TODO: abstract into analysis group/reprocess config extraction code?
analysis_group_id = int(self.args[0])
if len(self.args) == 3:
reprocess_config = Session.query(ReprocessConfig).filter_by(code=self.args[1]).one()
reprocess_config_id = reprocess_config.id
else:
reprocess_config = None
reprocess_config_id = None
analysis_group = Session.query(AnalysisGroup).get(analysis_group_id)
if not analysis_group:
raise ValueError, "No analysis group for id %s" % analysis_group_id
self.process_plates(app, analysis_group, reprocess_config)
def update_plate_types(self):
plate_types = [PlateType(name=u'CNV',code=u'bcnv'),
PlateType(name=u'Dye',code=u'bdye'),
PlateType(name=u'Singleplex',code=u'bsplex'),
PlateType(name=u'Duplex',code=u'bdplex'),
PlateType(name=u'FP/FN',code=u'bfpfn'),
PlateType(name=u'HighDNR',code=u'bdnr'),
PlateType(name=u'RED',code=u'bred'),
PlateType(name=u'Other',code=u'bapp'),
PlateType(name=u'Exception', code=u'bexp'),
PlateType(name=u'Dye v2',code=u'bdye2'),
PlateType(name=u'EventCount', code=u'betaec'),
PlateType(name=u'Carryover (ENG)', code=u'bcarry'),
PlateType(name=u'4-Well ColorComp (ENG)', code=u'bcc'),
PlateType(name=u'Cal Verification (CSFV)', code=u'fvtitr'),
PlateType(name=u'4-Well ColorComp (MFG)', code=u'mfgcc'),
PlateType(name=u'Dye Calibration', code=u'scc'),
PlateType(name=u'Fluidics Verification', code=u'mfgco'),
PlateType(name=u'FAM/VIC Titration (Groove)', code=u'gfv'),
PlateType(name=u'EvaGreen DNR (Staph/high BG)', code=u'2x10'),
PlateType(name=u'DG Pressure Volume', code=u'dgvol'),
PlateType(name=u'CNV (Groove)', code=u'gcnv'),
PlateType(name=u'EvaGreen DNR (Staph/no BG)', code=u'gdnr'),
PlateType(name=u'EvaGreen Assay DNA Load', code=u'gload'),
PlateType(name=u'EvaGreen DNR (RNaseP)', code=u'egdnr'),
PlateType(name=u'FAM350 no DNA', code=u'fam350'),
PlateType(name=u'FAM350 DNA Load', code=u'fm350l'),
PlateType(name=u'Auto Validation', code=u'av'),
PlateType(name=u'Qx200 DNR validation', code=u'dnr200'),
PlateType(name=u'Qx200 Duplex validation', code=u'dplex200'),
PlateType(name=u'Qx200 Broad Panel EG val', code=u'eg200'),
PlateType(name=u'Qx200 Broad Panel TQ val', code=u'tq200'),
PlateType(name=u'Qx200 CNV validation', code=u'cnv200'),
PlateType(name=u'Qx200 EvaGreen CNV val', code=u'cnv200e'),
PlateType(name=u'Probe EventCount', code=u'probeec'),
PlateType(name=u'Eva EventCount', code=u'evaec')
]
for pt in plate_types:
dpt = Session.query(PlateType).filter_by(code=pt.code).first()
if not dpt:
Session.add(pt)
elif pt.name != dpt.name:
dpt.name = pt.name
Session.commit()
def command(self):
app = self.load_wsgi_app()
# enforce name.ini
if len(self.args) < 3:
raise ValueError, self.__class__.usage
plate_id = int(self.args[0])
ag_id = int(self.args[1])
plate = Session.query(Plate).get(plate_id)
if not plate:
raise ValueError, "Invalid plate id: %s" % plate_id
ag = Session.query(AnalysisGroup).get(ag_id)
if not ag:
raise ValueError, "Invalid analysis group id: %s" % ag_id
ag.plates.append(plate)
Session.commit()
def process_plates(self, app, analysis_group, reprocess_config):
storage = QLStorageSource(app.config)
plates = analysis_group.plates
for plate in plates:
pms = [pm for pm in plate.metrics if pm.reprocess_config_id == reprocess_config.id]
if not pms:
print "Cannot find analysis group for plate %s" % plate.id
else:
pm = pms[0]
dbplate = dbplate_tree(plate.id)
if reprocess_config:
data_root = app.config['qlb.reprocess_root']
storage = QLPReprocessedFileSource(data_root, reprocess_config)
else:
storage = QLStorageSource(app.config)
try:
if reprocess_config:
plate_path = storage.full_path(analysis_group, dbplate)
else:
plate_path = storage.plate_path(dbplate)
except Exception, e:
print "Could not read plate: %s" % plate.id
continue
qlplate = get_plate(plate_path)
if not qlplate:
raise ValueError, "Could not read plate: %s" % plate_path
else:
print "Processing %s" % plate_path
try:
self.backfill_plate(qlplate, pm)
Session.commit()
except Exception, e:
print "Could not process plate %s" % dbplate.id
import sys, traceback
traceback.print_exc(file=sys.stdout)
Session.rollback()
continue
def command(self):
app = self.load_wsgi_app()
# enforce at least one plate
if len(self.args) < 3:
raise ValueError, self.__class__.usage
group_id = int(self.args[0])
ag = Session.query(AnalysisGroup).get(group_id)
if not ag:
raise ValueError, "Invalid analysis group id: %s" % group_id
for i in range(1,len(self.args)-1):
plate_id = int(self.args[i])
plate = Session.query(Plate).get(plate_id)
if not plate:
print "Could not find plate with id %s" % plate_id
continue
ag.plates.append(plate)
Session.commit()
def update_system_versions(self):
""" Double check if values are inserted correctly if you re-enable this. mysql seesm to auto inc when id = 0....
"""
#system_versions = [SystemVersion(id=-1 ,type=u'QX100',desc=u'Unknown Hardware version'),
# SystemVersion(id=0 ,type=u'QX100',desc=u'QX100 - HW Rev A/B'),
system_versions = [SystemVersion(id=1 ,type=u'QX100', desc=u'QX100 - HW Rev A/B bigger detector cap differences'),
SystemVersion(id=2 ,type=u'QX100', desc=u'QX100 - HW Rev C'),
SystemVersion(id=3 ,type=u'QX150', desc=u'QX150 - HW Rev Z Upgrade'),
SystemVersion(id=4 ,type=u'QX200', desc=u'QX200 - HW Rev Z'),
SystemVersion(id=5 ,type=u'QX201', desc=u'QX200 - HW with BR built Detector'),
SystemVersion(id=6 ,type=u'QX150L', desc=u'QX150 - HW Rev Z Upgrade with LED'),
SystemVersion(id=7 ,type=u'QX201L', desc=u'QX201 - HW with BR built LED Detector'),
SystemVersion(id=200,type=u'QX200', desc=u'QX200 - Pre-Beta HW')]
for sv in system_versions:
dbsv = Session.query(SystemVersion).filter_by(id=sv.id).first()
if not dbsv:
Session.add(sv)
else:
if (dbsv.type != sv.type):
dbsv.type = sv.type
if( dbsv.desc != sv.desc):
dbsv.desc = sv.desc
Session.commit()
def process_taqman_job(job, job_queue, sequence_source, dg_calc):
logger = logging.getLogger(LOGGER_NAME)
struct = JSONMessage.unserialize(job.input_message)
forward_primer_id = struct.forward_primer_id
reverse_primer_id = struct.reverse_primer_id
probe_ids = struct.probe_ids
sequence_group = Session.query(SequenceGroup).get(struct.sequence_group_id)
fp_sequence = Session.query(SequenceGroupComponent).get(forward_primer_id).sequence
rp_sequence = Session.query(SequenceGroupComponent).get(reverse_primer_id).sequence
probes = Session.query(SequenceGroupComponent).filter(SequenceGroupComponent.id.in_(probe_ids)).all()
probe_seqs = [p.sequence for p in probes]
sequences = sequence_source.sequences_for_primers(fp_sequence.sequence, rp_sequence.sequence,
fwd_prefix_length=MAX_CACHE_PADDING,
rev_suffix_length=MAX_CACHE_PADDING)
if sequences is None:
logger.error("Amplicon TaqMan: Could not get response from server [job %s]" % job.id)
job_queue.abort(job, JSONErrorMessage('Could not get response from server.'))
Session.commit()
return
amplicons = create_amplicons_from_pcr_sequences(sequence_group,
forward_primer=fp_sequence,
reverse_primer=rp_sequence,
probes=probe_seqs,
pcr_sequences=sequences)
for amp in amplicons:
populate_amplicon_dgs(amp, dg_calc)
logger.info("Taqman job completed [job %s]" % job.id)
Session.commit()
# TODO: add finish message
# now add SNPs to cached sequences
for amp in amplicons:
for cseq in amp.cached_sequences:
job_queue.add(JOB_ID_PROCESS_SNPS, ProcessSNPMessage(cached_sequence_id=cseq.id),
parent_job=job.parent)
# avoid condition where job completed -- clear child job first
job_queue.finish(job, None)
def update_reprocess_analysis_group_data(analysis_group, reprocess_config, config, logger):
"""
Given an analysis_gropu and repocessor, relaod each into qtools
"""
update_status = 0
plates = analysis_group.plates
# remove old metrics if present
for plate in plates:
pm = [pm for pm in plate.metrics if pm.reprocess_config_id == reprocess_config.id]
# should only be of length 1, but just to be safe
for p in pm:
Session.delete(p)
# TODO: how to make this whole operation transactional
Session.commit()
plate_ids = [plate.id for plate in plates]
data_root = config['qlb.reprocess_root']
file_source = QLPReprocessedFileSource(data_root, reprocess_config)
for id in plate_ids:
dbplate = dbplate_tree(id)
# TODO: right abstraction?
plate_path = file_source.full_path(analysis_group, dbplate)
print "Reading/updating metrics for %s" % plate_path
qlplate = get_plate(plate_path)
if not qlplate:
print "Could not read plate: %s" % plate_path
continue
plate_metrics = get_beta_plate_metrics(dbplate, qlplate, reprocess_config)
Session.add(plate_metrics)
del qlplate
Session.commit()
return update_status
