def update_droplet_generators(self):
dgs = [DropletGenerator(name=u'DG #01'),
DropletGenerator(name=u'DG #02'),
DropletGenerator(name=u'DG #03'),
DropletGenerator(name=u'DG #04'),
DropletGenerator(name=u'DG #05'),
DropletGenerator(name=u'DG #06'),
DropletGenerator(name=u'DG #07'),
DropletGenerator(name=u'DG #08'),
DropletGenerator(name=u'DG #09'),
DropletGenerator(name=u'DG #10'),
DropletGenerator(name=u'DG #11'),
DropletGenerator(name=u'DG #12'),
DropletGenerator(name=u'DG #13'),
DropletGenerator(name=u'DG #14'),
DropletGenerator(name=u'DG #15'),
DropletGenerator(name=u'DG #16'),
DropletGenerator(name=u'DG #17'),
DropletGenerator(id=101, name=u'Pilot #1'),
DropletGenerator(id=102, name=u'Pilot #2'),
DropletGenerator(id=103, name=u'Pilot #3'),
DropletGenerator(id=104, name=u'Pilot #4'),
DropletGenerator(id=105, name=u'Pilot #5'),
DropletGenerator(id=201, name=u'Groove DG #1'),
DropletGenerator(id=202, name=u'Groove DG #2'),
DropletGenerator(id=203, name=u'Groove DG #3'),
DropletGenerator(id=204, name=u'Groove DG #4')]
for dg in dgs:
db_dg = Session.query(DropletGenerator).filter_by(name=dg.name).first()
if not db_dg:
Session.add(dg)
Session.commit()
def process_plate(self, qlplate, plate_metric):
from qtools.lib.metrics import NEW_DROPLET_CLUSTER_METRICS_CALCULATOR, compute_metric_foreach_qlwell_channel
from qtools.lib.metrics import NEW_DROPLET_CLUSTER_WELL_METRICS_CALCULATOR, compute_metric_foreach_qlwell
compute_metric_foreach_qlwell_channel(qlplate, plate_metric, NEW_DROPLET_CLUSTER_METRICS_CALCULATOR)
compute_metric_foreach_qlwell( qlplate, plate_metric, NEW_DROPLET_CLUSTER_WELL_METRICS_CALCULATOR)
Session.commit()
def process_plate(self, qlplate, plate_metric):
well_dict = plate_metric.well_metric_name_dict
# TODO move this into a metrics lib instead?
for name, qlwell in qlplate.analyzed_wells.items():
well_metric = well_dict[name]
for qwc, wcm in zip(qlwell.channels, well_metric.well_channel_metrics):
DEFAULT_NTC_POSITIVE_CALCULATOR.compute(qlwell, qwc, wcm)
Session.commit()
def setup_buffers(self):
Session.add_all([Buffer(name=u"NEBuffer 1"),
Buffer(name=u"NEBuffer 2"),
Buffer(name=u"NEBuffer 3"),
Buffer(name=u"NEBuffer 4"),
Buffer(name=u'NEBuffer DpnII'),
Buffer(name=u'NEBuffer EcoRI'),
Buffer(name=u'NEBuffer SspI')])
Session.commit()
def update_dgs_used(self):
dgu = [DGUsed(name=u'Unicorn'),
DGUsed(name=u'Yeti'),
DGUsed(name=u'Nessie')]
for dgs in dgu:
db_dgu = Session.query(DGUsed).filter_by(name=dgs.name).first()
if not db_dgu:
Session.add(dgs)
Session.commit()
def command(self):
app = self.load_wsgi_app()
# enforce at least one plate
if len(self.args) < 2:
raise ValueError, self.__class__.usage
for i in range(0,len(self.args)-1):
print "Deleting plate #%s" % self.args[i]
delete_plate_recursive(int(self.args[i]))
Session.commit()
def command(self):
self.load_wsgi_app()
projects = [Project(name=u'Beta Test',code=u'Beta'),
Project(name=u'Validation',code=u'Validation')]
for project in projects:
dp = Session.query(Project).filter_by(name=project.name).first()
if not dp:
Session.add(project)
Session.commit()
def update_dr_groups(self):
dr_groups = [DRGroup(name='Apps DRs', active=True),
DRGroup(name='Groove DRs', active=True),
DRGroup(name='Golden DRs', active=True),
DRGroup(name='Rev Z DRs', active=True),
DRGroup(name='Custom DRs', active=True),
DRGroup(name='DNA 2.0', active=True),
DRGroup(name='Retired', active=False)]
for group in dr_groups:
drg = Session.query(DRGroup).filter_by(name=group.name).first()
if not drg:
Session.add(group)
Session.commit()
def command(self):
app = self.load_wsgi_app()
print "Clearing buffer tables..."
Session.execute('DELETE from vendor_enzyme')
Session.execute('DELETE from buffer')
Session.execute('DELETE from vendor')
Session.execute('DELETE from enzyme')
Session.commit()
self.setup_buffers()
self.setup_vendors()
def command(self):
app = self.load_wsgi_app()
# enforce name.ini
if len(self.args) < 2:
raise ValueError, self.__class__.usage
name = self.args[0]
ag = AnalysisGroup(name=name)
Session.add(ag)
Session.commit()
print "Created analysis group %s with id=%s" % (ag.name, ag.id)
def command(self):
self.load_wsgi_app()
Session.execute('DELETE from plate_tag_plate')
Session.execute('DELETE from plate_tag')
Session.commit()
Session.add_all([PlateTag(name=u'Clog'),
PlateTag(name=u'Failure'),
PlateTag(name=u'Highlight')])
Session.commit()
def update_physical_plates(self):
pps = [PhysicalPlate(name='Eppendorf twin.tec PCR plate 96'),
PhysicalPlate(name='Bio-Rad Hard-Shell White/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Red/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Black/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Yellow/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Blue/Clear'),
PhysicalPlate(name='Bio-Rad Hard-Shell Green/Clear')]
for pp in pps:
dbpp = Session.query(PhysicalPlate).filter_by(name=pp.name).first()
if not dbpp:
Session.add(pp)
Session.commit()
def command(self):
app = self.load_wsgi_app()
if len(self.args) > 1:
plate_id = int(self.args[0])
else:
raise Exception, "Not enough arguments: %s" % self.__class__.usage
plate = Session.query(Plate).get(plate_id)
if not plate:
raise Exception, "Could not find plate #: %s" % plate_id
plate_type = Session.query(PlateType).filter_by(code=u'bexp').first()
plate.plate_type = plate_type
Session.commit()
def update_thermal_cyclers(self):
tcs = [ThermalCycler(name=u'Eppendorf 1'),
ThermalCycler(name=u'Eppendorf 2'),
ThermalCycler(name=u'Eppendorf 3'),
ThermalCycler(name=u'Eppendorf 4'),
ThermalCycler(name=u'Eppendorf 5'),
ThermalCycler(name=u'Eppendorf 6'),
ThermalCycler(name=u'Bio-Rad T100 1'),
ThermalCycler(name=u'Bio-Rad C1000 1'),
ThermalCycler(name=u'Eppendorf 8')]
for tc in tcs:
db_tc = Session.query(ThermalCycler).filter_by(name=tc.name).first()
if not db_tc:
Session.add(tc)
Session.commit()
def command(self):
self.load_wsgi_app()
Session.execute('DELETE from well_tag_qlbwell')
Session.execute('DELETE from well_tag')
Session.commit()
Session.add_all([WellTag(name=u'Clog'),
WellTag(name=u'NoClog Control'),
WellTag(name=u'Rain'),
WellTag(name=u'Negative Rain'),
WellTag(name=u'Bad Threshold'),
WellTag(name=u'Good Well')])
Session.commit()
def _to_python(self, value, state):
# value is gonna be a dict
for name, spec in self.spec.items():
obj, id_col, disp_col, additional_cols = spec
dict_val = value.get(name)
if not dict_val:
continue
if dict_val and dict_val.isdigit():
if Session.query(obj).filter(getattr(obj, id_col) == int(dict_val)).one():
continue
additional_cols[disp_col] = dict_val
newobj = obj(**additional_cols)
Session.add(newobj)
Session.commit()
value[name] = getattr(newobj, id_col)
return value
def process_taqman_job(job, job_queue, sequence_source, dg_calc):
logger = logging.getLogger(LOGGER_NAME)
struct = JSONMessage.unserialize(job.input_message)
forward_primer_id = struct.forward_primer_id
reverse_primer_id = struct.reverse_primer_id
probe_ids = struct.probe_ids
sequence_group = Session.query(SequenceGroup).get(struct.sequence_group_id)
fp_sequence = Session.query(SequenceGroupComponent).get(forward_primer_id).sequence
rp_sequence = Session.query(SequenceGroupComponent).get(reverse_primer_id).sequence
probes = Session.query(SequenceGroupComponent).filter(SequenceGroupComponent.id.in_(probe_ids)).all()
probe_seqs = [p.sequence for p in probes]
sequences = sequence_source.sequences_for_primers(fp_sequence.sequence, rp_sequence.sequence,
fwd_prefix_length=MAX_CACHE_PADDING,
rev_suffix_length=MAX_CACHE_PADDING)
if sequences is None:
logger.error("Amplicon TaqMan: Could not get response from server [job %s]" % job.id)
job_queue.abort(job, JSONErrorMessage('Could not get response from server.'))
Session.commit()
return
amplicons = create_amplicons_from_pcr_sequences(sequence_group,
forward_primer=fp_sequence,
reverse_primer=rp_sequence,
probes=probe_seqs,
pcr_sequences=sequences)
for amp in amplicons:
populate_amplicon_dgs(amp, dg_calc)
logger.info("Taqman job completed [job %s]" % job.id)
Session.commit()
# TODO: add finish message
# now add SNPs to cached sequences
for amp in amplicons:
for cseq in amp.cached_sequences:
job_queue.add(JOB_ID_PROCESS_SNPS, ProcessSNPMessage(cached_sequence_id=cseq.id),
parent_job=job.parent)
# avoid condition where job completed -- clear child job first
job_queue.finish(job, None)
def update_plate_types(self):
plate_types = [PlateType(name=u'CNV',code=u'bcnv'),
PlateType(name=u'Dye',code=u'bdye'),
PlateType(name=u'Singleplex',code=u'bsplex'),
PlateType(name=u'Duplex',code=u'bdplex'),
PlateType(name=u'FP/FN',code=u'bfpfn'),
PlateType(name=u'HighDNR',code=u'bdnr'),
PlateType(name=u'RED',code=u'bred'),
PlateType(name=u'Other',code=u'bapp'),
PlateType(name=u'Exception', code=u'bexp'),
PlateType(name=u'Dye v2',code=u'bdye2'),
PlateType(name=u'EventCount', code=u'betaec'),
PlateType(name=u'Carryover (ENG)', code=u'bcarry'),
PlateType(name=u'4-Well ColorComp (ENG)', code=u'bcc'),
PlateType(name=u'Cal Verification (CSFV)', code=u'fvtitr'),
PlateType(name=u'4-Well ColorComp (MFG)', code=u'mfgcc'),
PlateType(name=u'Dye Calibration', code=u'scc'),
PlateType(name=u'Fluidics Verification', code=u'mfgco'),
PlateType(name=u'FAM/VIC Titration (Groove)', code=u'gfv'),
PlateType(name=u'EvaGreen DNR (Staph/high BG)', code=u'2x10'),
PlateType(name=u'DG Pressure Volume', code=u'dgvol'),
PlateType(name=u'CNV (Groove)', code=u'gcnv'),
PlateType(name=u'EvaGreen DNR (Staph/no BG)', code=u'gdnr'),
PlateType(name=u'EvaGreen Assay DNA Load', code=u'gload'),
PlateType(name=u'EvaGreen DNR (RNaseP)', code=u'egdnr'),
PlateType(name=u'FAM350 no DNA', code=u'fam350'),
PlateType(name=u'FAM350 DNA Load', code=u'fm350l'),
PlateType(name=u'Auto Validation', code=u'av'),
PlateType(name=u'Qx200 DNR validation', code=u'dnr200'),
PlateType(name=u'Qx200 Duplex validation', code=u'dplex200'),
PlateType(name=u'Qx200 Broad Panel EG val', code=u'eg200'),
PlateType(name=u'Qx200 Broad Panel TQ val', code=u'tq200'),
PlateType(name=u'Qx200 CNV validation', code=u'cnv200'),
PlateType(name=u'Qx200 EvaGreen CNV val', code=u'cnv200e'),
PlateType(name=u'Probe EventCount', code=u'probeec'),
PlateType(name=u'Eva EventCount', code=u'evaec')
]
for pt in plate_types:
dpt = Session.query(PlateType).filter_by(code=pt.code).first()
if not dpt:
Session.add(pt)
elif pt.name != dpt.name:
dpt.name = pt.name
Session.commit()
def command(self):
app = self.load_wsgi_app()
# enforce name.ini
if len(self.args) < 3:
raise ValueError, self.__class__.usage
plate_id = int(self.args[0])
ag_id = int(self.args[1])
plate = Session.query(Plate).get(plate_id)
if not plate:
raise ValueError, "Invalid plate id: %s" % plate_id
ag = Session.query(AnalysisGroup).get(ag_id)
if not ag:
raise ValueError, "Invalid analysis group id: %s" % ag_id
ag.plates.append(plate)
Session.commit()
def process_plates(self, app, analysis_group, reprocess_config):
storage = QLStorageSource(app.config)
plates = analysis_group.plates
for plate in plates:
pms = [pm for pm in plate.metrics if pm.reprocess_config_id == reprocess_config.id]
if not pms:
print "Cannot find analysis group for plate %s" % plate.id
else:
pm = pms[0]
dbplate = dbplate_tree(plate.id)
if reprocess_config:
data_root = app.config['qlb.reprocess_root']
storage = QLPReprocessedFileSource(data_root, reprocess_config)
else:
storage = QLStorageSource(app.config)
try:
if reprocess_config:
plate_path = storage.full_path(analysis_group, dbplate)
else:
plate_path = storage.plate_path(dbplate)
except Exception, e:
print "Could not read plate: %s" % plate.id
continue
qlplate = get_plate(plate_path)
if not qlplate:
raise ValueError, "Could not read plate: %s" % plate_path
else:
print "Processing %s" % plate_path
try:
self.backfill_plate(qlplate, pm)
Session.commit()
except Exception, e:
print "Could not process plate %s" % dbplate.id
import sys, traceback
traceback.print_exc(file=sys.stdout)
Session.rollback()
continue
def update_reprocess_analysis_group_data(analysis_group, reprocess_config, config, logger):
"""
Given an analysis_gropu and repocessor, relaod each into qtools
"""
update_status = 0
plates = analysis_group.plates
# remove old metrics if present
for plate in plates:
pm = [pm for pm in plate.metrics if pm.reprocess_config_id == reprocess_config.id]
# should only be of length 1, but just to be safe
for p in pm:
Session.delete(p)
# TODO: how to make this whole operation transactional
Session.commit()
plate_ids = [plate.id for plate in plates]
data_root = config['qlb.reprocess_root']
file_source = QLPReprocessedFileSource(data_root, reprocess_config)
for id in plate_ids:
dbplate = dbplate_tree(id)
# TODO: right abstraction?
plate_path = file_source.full_path(analysis_group, dbplate)
print "Reading/updating metrics for %s" % plate_path
qlplate = get_plate(plate_path)
if not qlplate:
print "Could not read plate: %s" % plate_path
continue
plate_metrics = get_beta_plate_metrics(dbplate, qlplate, reprocess_config)
Session.add(plate_metrics)
del qlplate
Session.commit()
return update_status
def process_transcript_job(job, job_queue, sequence_source, dg_calc):
logger = logging.getLogger(LOGGER_NAME)
struct = JSONMessage.unserialize(job.input_message)
forward_primer_id = struct.forward_primer_id
reverse_primer_id = struct.reverse_primer_id
probe_ids = struct.probe_ids
sequence_group = Session.query(SequenceGroup).get(struct.sequence_group_id)
fp_sequence = Session.query(SequenceGroupComponent).get(forward_primer_id).sequence
rp_sequence = Session.query(SequenceGroupComponent).get(reverse_primer_id).sequence
probes = Session.query(SequenceGroupComponent).filter(SequenceGroupComponent.id.in_(probe_ids)).all()
probe_seqs = [p.sequence for p in probes]
transcripts = sequence_source.transcript_sequences_for_primers(fp_sequence.sequence,
rp_sequence.sequence)
if transcripts is None:
logger.error("GEX TaqMan: Could not get response from server [job %s]" % job.id)
job_queue.abort(job, JSONErrorMessage("Could not get response from server."))
Session.commit()
return
db_transcripts = create_db_transcripts_from_pcr_transcripts(sequence_group,
forward_primer=fp_sequence,
reverse_primer=rp_sequence,
probes=probe_seqs,
pcr_gene_sequences=transcripts)
sequence_group.transcripts = db_transcripts
for trans in sequence_group.transcripts:
populate_transcript_dgs(trans, dg_calc)
logger.info("GEX TaqMan job completed [job %s]" % job.id)
Session.commit()
for trans in db_transcripts:
job_queue.add(JOB_ID_PROCESS_GEX_SNPS, ProcessGEXSNPMessage(trans.id),
parent_job=job.parent)
job_queue.finish(job, None)
def command(self):
app = self.load_wsgi_app()
# enforce at least one plate
if len(self.args) < 3:
raise ValueError, self.__class__.usage
group_id = int(self.args[0])
ag = Session.query(AnalysisGroup).get(group_id)
if not ag:
raise ValueError, "Invalid analysis group id: %s" % group_id
for i in range(1,len(self.args)-1):
plate_id = int(self.args[i])
plate = Session.query(Plate).get(plate_id)
if not plate:
print "Could not find plate with id %s" % plate_id
continue
ag.plates.append(plate)
Session.commit()
def update_system_versions(self):
""" Double check if values are inserted correctly if you re-enable this. mysql seesm to auto inc when id = 0....
"""
#system_versions = [SystemVersion(id=-1 ,type=u'QX100',desc=u'Unknown Hardware version'),
# SystemVersion(id=0 ,type=u'QX100',desc=u'QX100 - HW Rev A/B'),
system_versions = [SystemVersion(id=1 ,type=u'QX100', desc=u'QX100 - HW Rev A/B bigger detector cap differences'),
SystemVersion(id=2 ,type=u'QX100', desc=u'QX100 - HW Rev C'),
SystemVersion(id=3 ,type=u'QX150', desc=u'QX150 - HW Rev Z Upgrade'),
SystemVersion(id=4 ,type=u'QX200', desc=u'QX200 - HW Rev Z'),
SystemVersion(id=5 ,type=u'QX201', desc=u'QX200 - HW with BR built Detector'),
SystemVersion(id=6 ,type=u'QX150L', desc=u'QX150 - HW Rev Z Upgrade with LED'),
SystemVersion(id=7 ,type=u'QX201L', desc=u'QX201 - HW with BR built LED Detector'),
SystemVersion(id=200,type=u'QX200', desc=u'QX200 - Pre-Beta HW')]
for sv in system_versions:
dbsv = Session.query(SystemVersion).filter_by(id=sv.id).first()
if not dbsv:
Session.add(sv)
else:
if (dbsv.type != sv.type):
dbsv.type = sv.type
if( dbsv.desc != sv.desc):
dbsv.desc = sv.desc
Session.commit()
def command(self):
app = self.load_wsgi_app()
print "Clearing NEB buffer tables..."
# make this reversible...
neb = Session.query(Vendor).filter_by(name='NEB').first()
if neb:
enzymes = neb.enzymes
for enz in enzymes:
Session.delete(enz)
neb.enzymes = []
Session.commit()
# now go with the vendors (question: is it vendor enzyme or enzyme
# in general subject to methylation)
source = NEBEnzymeSource()
if not neb:
neb = Vendor(name="NEB", website="http://www.neb.com")
Session.add(neb)
Session.commit()
try:
for name, page_uri in source.iter_restriction_enzymes():
print "Processing %s..." % name
enz = Session.query(Enzyme).filter_by(name=name).first()
if not enz:
enz = Enzyme(name=name)
Session.add(enz)
details = source.enzyme_details(name, page_uri)
if details:
enz.cutseq = details['sequence']
enz.methylation = details['methylation_sensitivity']
buf = Session.query(Buffer).filter_by(name=details['buffer']).first()
if not buf:
buf = Buffer(name=details['buffer'])
Session.add(buf)
ve = VendorEnzyme(enzyme=enz, vendor=neb, buffer=buf)
ve.unit_cost_1 = details.get('unit_cost_1', None)
ve.unit_cost_2 = details.get('unit_cost_2', None)
ve.unit_serial_1 = details.get('unit_serial_1', None)
ve.unit_serial_2 = details.get('unit_serial_2', None)
ve.vendor_serial = details.get('vendor_serial', None)
if ve.unit_cost_1 is not None:
Session.add(ve)
# save per enzyme, interruption should be logged
Session.commit()
except IOError, e:
Session.rollback()
print "Could not communicate with NEB server"
def process_location_job(job, job_queue, sequence_source, dg_calc):
logger = logging.getLogger(LOGGER_NAME)
struct = JSONMessage.unserialize(job.input_message)
sequence_group = Session.query(SequenceGroup).get(struct.sequence_group_id)
sequence = None
try:
sequence = sequence_source.sequence_around_loc(sequence_group.location_chromosome,
sequence_group.location_base,
sequence_group.amplicon_length,
prefix_length=MAX_CACHE_PADDING,
suffix_length=MAX_CACHE_PADDING)
except Exception:
logger.exception("Could not retrieve sequence for assay location [job %s]: " % job.id)
job_queue.abort(job, JSONErrorMessage('Could not retrieve the sequence for the specified amplicon location.'))
Session.commit()
return
if sequence is None:
logger.error("Amplicon Location: could not get response from server [job %s]: " % job.id)
job_queue.abort(job, JSONErrorMessage('Could not get response from server.'))
Session.commit()
return
amplicons = create_amplicons_from_pcr_sequences(sequence_group,
pcr_sequences=[sequence])
for amp in amplicons:
populate_amplicon_dgs(amp, dg_calc)
logger.info("Location job completed [job %s]" % job.id)
Session.commit()
for amp in amplicons:
for cseq in amp.cached_sequences:
job_queue.add(JOB_ID_PROCESS_SNPS, ProcessSNPMessage(cached_sequence_id=cseq.id),
parent_job=job.parent)
job_queue.finish(job, None)