def command(self):
app = self.load_wsgi_app()
min_id = 0
if len(self.args) > 1:
min_id = self.args[0]
storage = QLStorageSource(app.config)
qlbplates = Session.query(QLBPlate).filter(QLBPlate.id > min_id).order_by('id').all()
for qlbplate in qlbplates:
try:
path = storage.qlbplate_path(qlbplate)
except Exception:
print "Could not find plate: %s (%s)" % (qlbplate.plate.name if qlbplate.plate else 'Name unknown', qlbplate.id)
continue
try:
qlplate = get_plate(path)
except Exception:
print "Could not read plate: %s (%s)" % (qlbplate.plate.name if qlbplate.plate else 'Name Unknown', qlbplate.id)
continue
if qlplate.is_fam_vic:
qlbplate.dyeset = QLBPlate.DYESET_FAM_VIC
elif qlplate.is_fam_hex:
qlbplate.dyeset = QLBPlate.DYESET_FAM_HEX
elif qlplate.is_eva_green:
qlbplate.dyeset = QLBPlate.DYESET_EVA
else:
qlbplate.dyeset = QLBPlate.DYESET_UNKNOWN
print "Assigned dye %s - %s (%s)" % (qlbplate.dyeset, qlbplate.plate.name if qlbplate.plate else 'Name Unknown', qlbplate.id)
Session.commit()
def command(self):
app = self.load_wsgi_app()
storage = QLStorageSource(app.config)
if len(self.args) < 2:
print self.__class__.usage
return
for i in range(0,len(self.args)-1):
plate_id = int(self.args[i])
plate = dbplate_tree(plate_id)
try:
plate_path = storage.plate_path(plate)
except Exception, e:
print "Could not read plate: %s" % plate_id
continue
qlplate = get_plate(plate_path)
if not qlplate:
raise ValueError, "Could not read plate: %s" % plate_path
else:
print "Processing %s" % plate_path
try:
from qtools.lib.metrics import DEFAULT_CNV_CALC, compute_metric_foreach_qlwell
plate_metrics = plate.metrics[0]
compute_metric_foreach_qlwell(qlplate, plate_metrics, DEFAULT_CNV_CALC)
except Exception, e:
import sys, traceback
traceback.print_exc(file=sys.stdout)
Session.rollback()
return
def command(self):
app = self.load_wsgi_app()
# enforce config.ini
if len(self.args) < 2:
raise ValueError, self.__class__.usage
plate_id = int(self.args[0])
plate = dbplate_tree(plate_id)
if not plate:
raise ValueError, "Invalid plate id: %s" % plate_id
storage = QLStorageSource(app.config)
# for now, no reprocessing.
plate_path = storage.plate_path(plate)
qlplate = get_plate(plate_path)
if not qlplate:
raise ValueError, "Could not read plate: %s" % plate_path
try:
plate_metrics = plate.metrics[0]
self.process_plate( qlplate, plate_metrics)
except Exception, e:
import sys, traceback
traceback.print_exc(file=sys.stdout)
Session.rollback()
return
def do_register_lab_reader(self):
LAB_FILEROOT = 'main'
storage = QLStorageSource(config)
path = self.form_result['path']
name = self.form_result['path'][3:] # sans DR
new_reader_path = storage.real_path(LAB_FILEROOT, self.form_result['path'])
new_reader = Box2(name=name,
code=name,
src_dir=path,
reader_type=Box2.READER_TYPE_WHOLE,
active=True,
fileroot=LAB_FILEROOT)
Session.add(new_reader)
Session.commit()
session['flash'] = 'Reader %s added.' % name
session.save()
return redirect(url(controller='admin', action='register_lab'))
def from_plate(self, id=None):
if not id:
abort(404)
plate = Session.query(Plate).get(id)
if not plate:
abort(404)
# ignore possibility of reprocess config for now
storage = QLStorageSource(config)
path = storage.plate_path(plate)
factory = ExperimentMetadataObjectFactory()
qlp = factory.parse_plate(path)
c.wells = qlp.analyzed_wells
c.original_source = plate.name
c.plate_template_id = None
c.plate_setup_id = None
c.id = id
return self.__display(rowcol_order=qlp.acquisition_order)
def process_plates(self, app, analysis_group, reprocess_config):
storage = QLStorageSource(app.config)
plates = analysis_group.plates
for plate in plates:
pms = [pm for pm in plate.metrics if pm.reprocess_config_id == reprocess_config.id]
if not pms:
print "Cannot find analysis group for plate %s" % plate.id
else:
pm = pms[0]
dbplate = dbplate_tree(plate.id)
if reprocess_config:
data_root = app.config['qlb.reprocess_root']
storage = QLPReprocessedFileSource(data_root, reprocess_config)
else:
storage = QLStorageSource(app.config)
try:
if reprocess_config:
plate_path = storage.full_path(analysis_group, dbplate)
else:
plate_path = storage.plate_path(dbplate)
except Exception, e:
print "Could not read plate: %s" % plate.id
continue
qlplate = get_plate(plate_path)
if not qlplate:
raise ValueError, "Could not read plate: %s" % plate_path
else:
print "Processing %s" % plate_path
try:
self.backfill_plate(qlplate, pm)
Session.commit()
except Exception, e:
print "Could not process plate %s" % dbplate.id
import sys, traceback
traceback.print_exc(file=sys.stdout)
Session.rollback()
continue
def _register_lab_base(self):
"""
This works in reverse of create(). Instead, require that the user
create a folder on HyperV, and then only show the new folders that
have not yet been registered.
"""
LAB_FILEROOT = 'main'
file_source = QLStorageSource(config)
dir_root = file_source.real_path(LAB_FILEROOT,'')
rel_dirs = [os.path.relpath(path, dir_root) for path in glob.glob('%s/DR*' % dir_root)]
# for now, assume that a lab reader will be at the top level
lab_drs = [record[0].split('/')[0] for record in Session.query(Box2.src_dir).all()]
new_dirs = set(rel_dirs) - set(lab_drs)
# what should remain are new directories and misnamed directories
c.reader_field = {'value': '',
'options': [(d, d[3:]) for d in new_dirs]}
return render('/admin/register_lab.html')
def command(self):
app = self.load_wsgi_app()
storage = QLStorageSource(app.config)
# enforce config.ini
if len(self.args) > 1:
plate_id = int(self.args[0])
else:
plate_id = 1 ## default start....
plate_metrics = Session.query(PlateMetric).filter(PlateMetric.plate_id >= plate_id)\
.options(joinedload(PlateMetric.plate, innerjoin=True))\
.options(joinedload(PlateMetric.reprocess_config))
# TODO come up with version that takes care of reprocessed plates as well
# (by iterating through analysis groups, most likely)
for pm in plate_metrics:
if pm.from_reprocessed:
continue
plate = dbplate_tree(pm.plate_id)
try:
plate_path = storage.plate_path(plate)
except Exception:
print "Could not read plate: %s" % pm.plate_id
continue
try:
qlplate = get_plate(plate_path)
except Exception:
print "Could not read plate: %s" % plate_path
continue
if not qlplate:
print "Could not read plate: %s" % plate_path
continue
else:
print "Processing %s: %s..." % (pm.plate_id, plate_path)
self.process_plate(qlplate, pm)
def fft(self, id=None):
from qtools.lib.mplot import cleanup, render as plt_render
# make it better?
if not deps_loaded:
abort(500)
if not id:
abort(404)
well = Session.query(QLBWell).get(id)
if not well:
abort(404)
file = well.file
if not file:
abort(404)
storage = QLStorageSource(config)
path = storage.qlbwell_path(well)
wellobj = get_well(path)
fam_samples = wellobj.samples[:,0][::FFT_DOWNSAMPLE].astype('float')
vic_samples = wellobj.samples[:,1][::FFT_DOWNSAMPLE].astype('float')
# normalize to 0->1
fam_samples /= np.max(fam_samples)
vic_samples /= np.max(vic_samples)
Sf = np.fft.fft(fam_samples)
numpy.fft.fftpack._fft_cache = {}
ff = np.fft.fftfreq(len(Sf), d=FFT_DOWNSAMPLE/wellobj.data_acquisition_params.sample_rate)
# only consider positive half of frequency domain, normalize by num of samples
Yf = abs(Sf[0:len(ff)/2])/len(fam_samples)
# make X-axis in terms of Hz
Wf = int((len(ff)/2)/max(ff)) # group by 1 Hz
Sv = np.fft.fft(vic_samples)
numpy.fft.fftpack._fft_cache = {}
fv = np.fft.fftfreq(len(Sv), d=FFT_DOWNSAMPLE/wellobj.data_acquisition_params.sample_rate)
Yv = abs(Sv[0:len(fv)/2])/len(vic_samples)
Wv = int((len(fv)/2)/max(fv)) # group by 1 Hz
# find peaks
fam_peak_indices = fft.find_peak_indices(Yf, Wf*2) # group by 2Hz
vic_peak_indices = fft.find_peak_indices(Yv, Wv*2) # group by 2Hz
ftops = ff.take(fam_peak_indices)[0:10]
vtops = fv.take(vic_peak_indices)[0:10]
fig = plt.figure()
fig.set_figwidth(8)
fig.set_figheight(6)
plt.subplot(211)
plt.title('FAM Channel FFT (0-150Hz)')
plt.plot(ff[0:len(ff)/2], Yf)
plt.axis([-10, 150, -0.002, 0.04])
plt.text(70, 0.03, "Top Peaks (2 Hz windows):", weight='bold')
for i, val in enumerate(ftops):
plt.text(70 if i < 5 else 90, 0.027-(0.003*(i % 5)), "%.2f" % val, size=10)
# todo render text func?
plt.subplot(212)
plt.title('VIC Channel FFT (0-150Hz)')
plt.plot(fv[0:len(fv)/2], Yv)
plt.axis([-10, 150, -0.002, 0.04])
plt.text(70, 0.03, "Top Peaks (2 Hz windows):", weight='bold')
for i, val in enumerate(vtops):
plt.text(70 if i < 5 else 90, 0.027-(0.003*(i % 5)), "%.2f" % val, size=10)
# fft leaves cache-- not good for threaded server
imgdata = self.__render(fig)
cleanup(fig)
del fig, wellobj, fam_samples, vic_samples, Sf, ff, Yf, Wf, Sv, fv, Yv, Wv, ftops, vtops
gc.collect()
return imgdata
def command(self):
app = self.load_wsgi_app()
# enforce config.ini
if len(self.args) < 2:
raise ValueError, self.__class__.usage
analysis_group_id = int(self.args[0])
if len(self.args) == 3:
reprocess_config = Session.query(ReprocessConfig).filter_by(code=self.args[1]).one()
reprocess_config_id = reprocess_config.id
else:
reprocess_config = None
reprocess_config_id = None
analysis_group = Session.query(AnalysisGroup).get(analysis_group_id)
if not analysis_group:
raise ValueError, "No analysis group for id %s" % analysis_group_id
plates = analysis_group.plates
# todo: add in reprocess config id
for plate in plates:
pm = [pm for pm in plate.metrics if pm.reprocess_config_id == reprocess_config_id]
# should only be of length 1, but just to be safe
for p in pm:
Session.delete(p)
# TODO: how to make this whole operation transactional
Session.commit()
# this is a little tricky in the ORM world. only get the
# ids of the analysis_group plates, so that you can load the plate
# and all the necessary children
plate_ids = [plate.id for plate in plates]
if reprocess_config_id is None:
storage = QLStorageSource(app.config)
for id in plate_ids:
dbplate = dbplate_tree(id)
plate_path = storage.plate_path(dbplate)
print "Reading/updating metrics for %s" % plate_path
qlplate = get_plate(plate_path)
if not qlplate:
print "Could not read plate: %s" % plate_path
continue
plate_metrics = get_beta_plate_metrics(dbplate, qlplate)
Session.add(plate_metrics)
del qlplate
else:
data_root = app.config['qlb.reprocess_root']
file_source = QLPReprocessedFileSource(data_root, reprocess_config)
for id in plate_ids:
dbplate = dbplate_tree(id)
# TODO: right abstraction?
plate_path = file_source.full_path(analysis_group, dbplate)
print "Reading/updating metrics for %s" % plate_path
qlplate = get_plate(plate_path)
if not qlplate:
print "Could not read plate: %s" % plate_path
continue
plate_metrics = get_beta_plate_metrics(dbplate, qlplate, reprocess_config)
Session.add(plate_metrics)
del qlplate
Session.commit()
def __upload_file_path(self, box2, path):
source = QLStorageSource(config)
return source.box2_relative_path(box2, 'qtools_attachments', path)