def command(self): app = self.load_wsgi_app() min_id = 0 if len(self.args) > 1: min_id = self.args[0] storage = QLStorageSource(app.config) qlbplates = Session.query(QLBPlate).filter(QLBPlate.id > min_id).order_by('id').all() for qlbplate in qlbplates: try: path = storage.qlbplate_path(qlbplate) except Exception: print "Could not find plate: %s (%s)" % (qlbplate.plate.name if qlbplate.plate else 'Name unknown', qlbplate.id) continue try: qlplate = get_plate(path) except Exception: print "Could not read plate: %s (%s)" % (qlbplate.plate.name if qlbplate.plate else 'Name Unknown', qlbplate.id) continue if qlplate.is_fam_vic: qlbplate.dyeset = QLBPlate.DYESET_FAM_VIC elif qlplate.is_fam_hex: qlbplate.dyeset = QLBPlate.DYESET_FAM_HEX elif qlplate.is_eva_green: qlbplate.dyeset = QLBPlate.DYESET_EVA else: qlbplate.dyeset = QLBPlate.DYESET_UNKNOWN print "Assigned dye %s - %s (%s)" % (qlbplate.dyeset, qlbplate.plate.name if qlbplate.plate else 'Name Unknown', qlbplate.id) Session.commit()
def command(self): app = self.load_wsgi_app() storage = QLStorageSource(app.config) if len(self.args) < 2: print self.__class__.usage return for i in range(0,len(self.args)-1): plate_id = int(self.args[i]) plate = dbplate_tree(plate_id) try: plate_path = storage.plate_path(plate) except Exception, e: print "Could not read plate: %s" % plate_id continue qlplate = get_plate(plate_path) if not qlplate: raise ValueError, "Could not read plate: %s" % plate_path else: print "Processing %s" % plate_path try: from qtools.lib.metrics import DEFAULT_CNV_CALC, compute_metric_foreach_qlwell plate_metrics = plate.metrics[0] compute_metric_foreach_qlwell(qlplate, plate_metrics, DEFAULT_CNV_CALC) except Exception, e: import sys, traceback traceback.print_exc(file=sys.stdout) Session.rollback() return
def command(self): app = self.load_wsgi_app() # enforce config.ini if len(self.args) < 2: raise ValueError, self.__class__.usage plate_id = int(self.args[0]) plate = dbplate_tree(plate_id) if not plate: raise ValueError, "Invalid plate id: %s" % plate_id storage = QLStorageSource(app.config) # for now, no reprocessing. plate_path = storage.plate_path(plate) qlplate = get_plate(plate_path) if not qlplate: raise ValueError, "Could not read plate: %s" % plate_path try: plate_metrics = plate.metrics[0] self.process_plate( qlplate, plate_metrics) except Exception, e: import sys, traceback traceback.print_exc(file=sys.stdout) Session.rollback() return
def do_register_lab_reader(self): LAB_FILEROOT = 'main' storage = QLStorageSource(config) path = self.form_result['path'] name = self.form_result['path'][3:] # sans DR new_reader_path = storage.real_path(LAB_FILEROOT, self.form_result['path']) new_reader = Box2(name=name, code=name, src_dir=path, reader_type=Box2.READER_TYPE_WHOLE, active=True, fileroot=LAB_FILEROOT) Session.add(new_reader) Session.commit() session['flash'] = 'Reader %s added.' % name session.save() return redirect(url(controller='admin', action='register_lab'))
def from_plate(self, id=None): if not id: abort(404) plate = Session.query(Plate).get(id) if not plate: abort(404) # ignore possibility of reprocess config for now storage = QLStorageSource(config) path = storage.plate_path(plate) factory = ExperimentMetadataObjectFactory() qlp = factory.parse_plate(path) c.wells = qlp.analyzed_wells c.original_source = plate.name c.plate_template_id = None c.plate_setup_id = None c.id = id return self.__display(rowcol_order=qlp.acquisition_order)
def process_plates(self, app, analysis_group, reprocess_config): storage = QLStorageSource(app.config) plates = analysis_group.plates for plate in plates: pms = [pm for pm in plate.metrics if pm.reprocess_config_id == reprocess_config.id] if not pms: print "Cannot find analysis group for plate %s" % plate.id else: pm = pms[0] dbplate = dbplate_tree(plate.id) if reprocess_config: data_root = app.config['qlb.reprocess_root'] storage = QLPReprocessedFileSource(data_root, reprocess_config) else: storage = QLStorageSource(app.config) try: if reprocess_config: plate_path = storage.full_path(analysis_group, dbplate) else: plate_path = storage.plate_path(dbplate) except Exception, e: print "Could not read plate: %s" % plate.id continue qlplate = get_plate(plate_path) if not qlplate: raise ValueError, "Could not read plate: %s" % plate_path else: print "Processing %s" % plate_path try: self.backfill_plate(qlplate, pm) Session.commit() except Exception, e: print "Could not process plate %s" % dbplate.id import sys, traceback traceback.print_exc(file=sys.stdout) Session.rollback() continue
def _register_lab_base(self): """ This works in reverse of create(). Instead, require that the user create a folder on HyperV, and then only show the new folders that have not yet been registered. """ LAB_FILEROOT = 'main' file_source = QLStorageSource(config) dir_root = file_source.real_path(LAB_FILEROOT,'') rel_dirs = [os.path.relpath(path, dir_root) for path in glob.glob('%s/DR*' % dir_root)] # for now, assume that a lab reader will be at the top level lab_drs = [record[0].split('/')[0] for record in Session.query(Box2.src_dir).all()] new_dirs = set(rel_dirs) - set(lab_drs) # what should remain are new directories and misnamed directories c.reader_field = {'value': '', 'options': [(d, d[3:]) for d in new_dirs]} return render('/admin/register_lab.html')
def command(self): app = self.load_wsgi_app() storage = QLStorageSource(app.config) # enforce config.ini if len(self.args) > 1: plate_id = int(self.args[0]) else: plate_id = 1 ## default start.... plate_metrics = Session.query(PlateMetric).filter(PlateMetric.plate_id >= plate_id)\ .options(joinedload(PlateMetric.plate, innerjoin=True))\ .options(joinedload(PlateMetric.reprocess_config)) # TODO come up with version that takes care of reprocessed plates as well # (by iterating through analysis groups, most likely) for pm in plate_metrics: if pm.from_reprocessed: continue plate = dbplate_tree(pm.plate_id) try: plate_path = storage.plate_path(plate) except Exception: print "Could not read plate: %s" % pm.plate_id continue try: qlplate = get_plate(plate_path) except Exception: print "Could not read plate: %s" % plate_path continue if not qlplate: print "Could not read plate: %s" % plate_path continue else: print "Processing %s: %s..." % (pm.plate_id, plate_path) self.process_plate(qlplate, pm)
def fft(self, id=None): from qtools.lib.mplot import cleanup, render as plt_render # make it better? if not deps_loaded: abort(500) if not id: abort(404) well = Session.query(QLBWell).get(id) if not well: abort(404) file = well.file if not file: abort(404) storage = QLStorageSource(config) path = storage.qlbwell_path(well) wellobj = get_well(path) fam_samples = wellobj.samples[:,0][::FFT_DOWNSAMPLE].astype('float') vic_samples = wellobj.samples[:,1][::FFT_DOWNSAMPLE].astype('float') # normalize to 0->1 fam_samples /= np.max(fam_samples) vic_samples /= np.max(vic_samples) Sf = np.fft.fft(fam_samples) numpy.fft.fftpack._fft_cache = {} ff = np.fft.fftfreq(len(Sf), d=FFT_DOWNSAMPLE/wellobj.data_acquisition_params.sample_rate) # only consider positive half of frequency domain, normalize by num of samples Yf = abs(Sf[0:len(ff)/2])/len(fam_samples) # make X-axis in terms of Hz Wf = int((len(ff)/2)/max(ff)) # group by 1 Hz Sv = np.fft.fft(vic_samples) numpy.fft.fftpack._fft_cache = {} fv = np.fft.fftfreq(len(Sv), d=FFT_DOWNSAMPLE/wellobj.data_acquisition_params.sample_rate) Yv = abs(Sv[0:len(fv)/2])/len(vic_samples) Wv = int((len(fv)/2)/max(fv)) # group by 1 Hz # find peaks fam_peak_indices = fft.find_peak_indices(Yf, Wf*2) # group by 2Hz vic_peak_indices = fft.find_peak_indices(Yv, Wv*2) # group by 2Hz ftops = ff.take(fam_peak_indices)[0:10] vtops = fv.take(vic_peak_indices)[0:10] fig = plt.figure() fig.set_figwidth(8) fig.set_figheight(6) plt.subplot(211) plt.title('FAM Channel FFT (0-150Hz)') plt.plot(ff[0:len(ff)/2], Yf) plt.axis([-10, 150, -0.002, 0.04]) plt.text(70, 0.03, "Top Peaks (2 Hz windows):", weight='bold') for i, val in enumerate(ftops): plt.text(70 if i < 5 else 90, 0.027-(0.003*(i % 5)), "%.2f" % val, size=10) # todo render text func? plt.subplot(212) plt.title('VIC Channel FFT (0-150Hz)') plt.plot(fv[0:len(fv)/2], Yv) plt.axis([-10, 150, -0.002, 0.04]) plt.text(70, 0.03, "Top Peaks (2 Hz windows):", weight='bold') for i, val in enumerate(vtops): plt.text(70 if i < 5 else 90, 0.027-(0.003*(i % 5)), "%.2f" % val, size=10) # fft leaves cache-- not good for threaded server imgdata = self.__render(fig) cleanup(fig) del fig, wellobj, fam_samples, vic_samples, Sf, ff, Yf, Wf, Sv, fv, Yv, Wv, ftops, vtops gc.collect() return imgdata
def command(self): app = self.load_wsgi_app() # enforce config.ini if len(self.args) < 2: raise ValueError, self.__class__.usage analysis_group_id = int(self.args[0]) if len(self.args) == 3: reprocess_config = Session.query(ReprocessConfig).filter_by(code=self.args[1]).one() reprocess_config_id = reprocess_config.id else: reprocess_config = None reprocess_config_id = None analysis_group = Session.query(AnalysisGroup).get(analysis_group_id) if not analysis_group: raise ValueError, "No analysis group for id %s" % analysis_group_id plates = analysis_group.plates # todo: add in reprocess config id for plate in plates: pm = [pm for pm in plate.metrics if pm.reprocess_config_id == reprocess_config_id] # should only be of length 1, but just to be safe for p in pm: Session.delete(p) # TODO: how to make this whole operation transactional Session.commit() # this is a little tricky in the ORM world. only get the # ids of the analysis_group plates, so that you can load the plate # and all the necessary children plate_ids = [plate.id for plate in plates] if reprocess_config_id is None: storage = QLStorageSource(app.config) for id in plate_ids: dbplate = dbplate_tree(id) plate_path = storage.plate_path(dbplate) print "Reading/updating metrics for %s" % plate_path qlplate = get_plate(plate_path) if not qlplate: print "Could not read plate: %s" % plate_path continue plate_metrics = get_beta_plate_metrics(dbplate, qlplate) Session.add(plate_metrics) del qlplate else: data_root = app.config['qlb.reprocess_root'] file_source = QLPReprocessedFileSource(data_root, reprocess_config) for id in plate_ids: dbplate = dbplate_tree(id) # TODO: right abstraction? plate_path = file_source.full_path(analysis_group, dbplate) print "Reading/updating metrics for %s" % plate_path qlplate = get_plate(plate_path) if not qlplate: print "Could not read plate: %s" % plate_path continue plate_metrics = get_beta_plate_metrics(dbplate, qlplate, reprocess_config) Session.add(plate_metrics) del qlplate Session.commit()
def __upload_file_path(self, box2, path): source = QLStorageSource(config) return source.box2_relative_path(box2, 'qtools_attachments', path)