def batch_plate_do_upload(self, id=None):
batch_plate = self.__load_batch_plate(id)
if not batch_plate:
abort(404)
box2 = self.form_result['box2']
plate = self.form_result['plate']
plate_type = batch_plate.batch.plate_type
if plate_type.code == 'fvtitr' and len(plate.analyzed_wells) == 4:
# if four wells, it's really a MFGCC (FVTITR FAM+/VIC+ should have 2)
plate_type = Session.query(PlateType).filter_by(code='mfgcc').one()
plateobj = save_plate_from_upload_request(request.POST['plate'], plate, box2, plate_type_obj=plate_type)
# I want to put this in the form validator, but it's field dependent, so not right now
if plate_type.code in ('mfgcc', 'bcc'):
ok, message = validate_colorcomp_plate(plate)
if not ok:
response = self._batch_plate_upload_base(id)
Session.rollback()
return h.render_bootstrap_form(response, errors={'plate': message})
Session.add(plateobj)
if batch_plate.batch.plate_type.code == plate_type.code:
batch_plate.plate = plateobj
else:
batch_plate.secondary_plate = plateobj
batch_plate.qc_plate = self.form_result['qc_plate']
batch_plate.plate_notes = self.form_result['plate_notes']
Session.commit()
session['flash'] = 'Plate linked.'
session.save()
return redirect(url(controller='metrics', action='per_plate', id=plateobj.id))
def command(self):
self.load_wsgi_app()
unknown_plates = Session.query(QLBPlate).filter(QLBPlate.plate == None)
for qlbplate in unknown_plates:
try:
plate = plate_from_qlp(qlbplate)
Session.add(plate)
qlbplate.plate = plate
Session.commit()
except Exception:
Session.rollback()
def tearDown(self):
Session.rollback() # catches pending changes
Session.delete(self.plateTemplate)
Session.delete(self.shortTemplate)
Session.delete(self.qlbPlate)
Session.delete(self.plateVariant)
Session.delete(self.unknownPlate)
Session.delete(self.nameVariant)
Session.delete(self.unknownName)
Session.commit()
super(TestNamePlate, self).tearDown()
def create(self):
model = self.__form_to_model(self.form_result)
try:
Session.commit()
session['flash'] = 'Category saved.'
session.save()
return redirect(url(controller='assay_group', action='list'))
except IntegrityError, e:
Session.rollback()
session['flash'] = 'There is already a category named %s.' % model.name
session['flash_class'] = 'error'
session.save()
return redirect(url(controller='assay_group', action='new'))
def delete(self, id=None):
if id is None:
abort(404)
tag = Session.query(SequenceGroupTag).get(int(id))
if not tag:
abort(404)
try:
tag.sequence_groups = []
Session.delete(tag)
Session.commit()
except Exception, e:
Session.rollback()
session['flash'] = 'Could not delete category.'
session['flash_class'] = 'error'
return redirect(url(controller='assay_group', action='edit', id=id))
def command(self):
self.load_wsgi_app()
sequence_groups = Session.query(SequenceGroup).all()
for sg in sequence_groups:
try:
print "Backfilling channels for %s" % sg.name
channels = Session.query(QLBWellChannel).filter_by(target=sg.name)
for c in channels:
c.sequence_group_id = sg.id
Session.commit()
except Exception:
import sys, traceback
traceback.print_exc(file=sys.stdout)
Session.rollback()
continue
def add_qlp_plate_record(qlplate, qlbfile):
"""
Create a QLBPlate object based off a new QLBFile. Adds to
the current SQLAlchemy Session object, but does not commit (will
rollback, however, if there is a problem)
"""
valid_plate = True
plate = None
try:
plate = QLBPlate()
set_qlp_plate_record_attrs(plate, qlplate)
plate.file = qlbfile
Session.add(plate)
except Exception, e:
print e
Session.rollback()
valid_plate = False
def save_circuit(self):
log = Session.query(Box2Log).get(self.form_result['log_id'])
if not log:
abort(404)
try:
circ = Box2Circuit(name=self.form_result['name'],
log_template_id=self.form_result['log_id'])
Session.add(circ)
Session.commit()
log.box2_circuit_id = circ.id
Session.commit()
session['flash'] = 'Configuration for %s updated and circuit "%s" created.' % (log.box2.name, circ.name)
session.save()
redirect(url(controller='admin', action='reader_history', id=log.box2.code))
except exc.IntegrityError, e:
Session.rollback()
session['flash'] = 'There is already a circuit by that name.'
session['flash_class'] = 'error'
session.save()
redirect(url(controller='admin', action='circuit', id=log.id))
def save(self, id=None):
if not id:
abort(404)
model = Session.query(SequenceGroupTag).get(int(id))
if not model:
abort(404)
model = self.__form_to_model(self.form_result, model=model)
try:
Session.commit()
session['flash'] = 'Category saved.'
session.save()
return redirect(url(controller='assay_group', action='list'))
except IntegrityError, e:
Session.rollback()
session['flash'] = 'There is already a category named %s.' % model.name
session['flash_class'] = 'error'
session.save()
return redirect(url(controller='assay_group', action='edit', id=id))
def __update_batch_record(self, record=None):
new_record = False
if not record:
new_record = True
record = ManufacturingPlateBatch(name=self.form_result['name'],
plate_type_id=self.form_result['plate_type'],
creation_date=self.form_result['creation_date'],
default_dg_method=self.form_result['dg_method'])
record.creator_id = self.form_result['creator_id']
record.notes = self.form_result['notes']
record.fam_hi_size = self.form_result['fam_hi_size']
record.vic_hi_size = self.form_result['vic_hi_size']
record.hex_hi_size = self.form_result['hex_hi_size']
try:
Session.add(record)
Session.commit()
except IntegrityError, e:
Session.rollback()
raise e
def create(self, id):
"""
KIND OF UNSAFE. Creates/registers a reader.
"""
serial = self.form_result['serial']
reader_type = self.form_result['reader_type']
if reader_type == Box2.READER_TYPE_WHOLE:
## check name is in correct format...
if not re.search( '^771BR\d{4}$', serial ):
session['flash'] = 'New Reader name must follow "771BR####" convention'
session['flash_class'] = 'error'
session.save()
return redirect(url(controller='admin', action='register', id=id))
elif ( len( serial ) > 15 ):
session['flash'] = 'Reader name can not contain more then 15 characters'
session['flash_class'] = 'error'
session.save()
return redirect(url(controller='admin', action='register', id=id))
code = 'p%s' % serial
elif reader_type == Box2.READER_TYPE_FLUIDICS_MODULE:
serial = serial.upper()
if ( len( serial ) > 15 ):
session['flash'] = 'Fluidics name can not contain more then 15 characters'
session['flash_class'] = 'error'
session.save()
return redirect(url(controller='admin', action='register', id=id))
code = 'f%s' % serial
elif reader_type == Box2.READER_TYPE_DETECTOR_MODULE:
## check to make sure someone isn't acidently adding a DR
if re.search( '^771BR\d*', serial ):
session['flash'] = 'New Detector module names do not contain "771BR" '
session['flash_class'] = 'error'
session.save()
return redirect(url(controller='admin', action='register', id=id) )
elif ( len( serial ) > 15 ):
session['flash'] = 'Detector name can not contain more then 15 characters'
session['flash_class'] = 'error'
session.save()
return redirect(url(controller='admin', action='register', id=id))
serial = serial.upper()
code = 'd%s' % serial
# check and catch if reader already exists
box2 = Session.query(Box2).filter_by(code=code).first()
if box2:
session['flash'] = 'Unit %s is already registered.' % self.form_result['serial']
session['flash_class'] = 'error'
session.save()
return redirect(url(controller='admin', action='register', id=id))
#If not exists create one instead
if reader_type == Box2.READER_TYPE_WHOLE:
src_dir = "DR %s" % serial
reader = Box2(name=u'Prod %s' % serial, code=code, src_dir=src_dir, \
reader_type = Box2.READER_TYPE_WHOLE, \
fileroot=config['qlb.fileroot.register_fileroot'], active=True)
elif reader_type == Box2.READER_TYPE_FLUIDICS_MODULE:
src_dir="FM %s" % serial
reader = Box2(name=u'QL-FSA %s' % serial, code=code, src_dir=src_dir, \
reader_type = Box2.READER_TYPE_FLUIDICS_MODULE, \
fileroot=config['qlb.fileroot.register_fileroot'], active=True)
elif reader_type == Box2.READER_TYPE_DETECTOR_MODULE:
src_dir="DM %s" % serial
reader = Box2(name=u'DET %s' % serial, code=code, src_dir=src_dir, \
reader_type = Box2.READER_TYPE_DETECTOR_MODULE, \
fileroot=config['qlb.fileroot.register_fileroot'], active=True)
Session.add(reader)
reader.active = True
local_plate_source = QLBPlateSource(config, [reader])
dirname = local_plate_source.real_path(reader.fileroot, src_dir)
try:
os.mkdir(dirname)
Session.commit()
session['flash'] = 'Unit %s registered.' % serial
session.save()
except Exception, e:
session['flash'] = 'Could not create a directory for unit %s' % serial
session['flash_class'] = 'error'
session.save()
Session.rollback()
return redirect(url(controller='admin', action='register', id=id))
def __scan_plate(file_source, image_source, path_id, path, mtime_dict, plate_type=None, file_lists=None):
"""
The method responsible for taking a QLP file on disk and creating
thumbnails and adding/updating records in the database based off
the contents of that file.
This is a nasty abstraction, I apologize.
TODO: make this more natural along add/update line, do not require use of
mtime_dict or file_list (or return file_list as files touched)
Returns the Plate object of the added/updated plate, or None if there was
no touch/error.
:param file_source: The source of the QLP files (QLStorageSource)
:param image_source: The source/sink of thumbnails (QLStorageSource)
:param path_id: The unique identifier of the plate file. Computed by run_id()
:param path: The actual file path of the QLP.
:param mtime_dict: A mapping between plates and their last updated times. This will
indicate whether or not a plate is 'dirty' with respect to the DB.
:param plate_type: A plate type. Supplying this will indicate that the special metrics
corresponding to that plate type should be computed during the scan.
:param file_lists: A logging object used in the scan to record files that are missing,
poorly processed, etc. Side-effected by this method.
"""
if not file_lists:
file_lists = defaultdict(list)
# if the file is not being tracked, attempt to add it
if not mtime_dict.has_key(path_id):
print "Adding plate: %s" % path
qlbfile, qlplate, valid_file = add_qlp_file_record(file_source, path)
if not valid_file:
print "Invalid file: %s" % path
file_lists['invalid_plates'].append(path)
return None
elif path.endswith('HFE_Plate.qlp'):
qlbfile.read_status = -7
print "Ignoring HFE Plate: %s" % path
Session.commit()
return None
elif qlbfile.version is 'Unknown':
qlbfile.read_status = -8
print "Ignoring plate run with unknown QS version: %s" % path
Session.commit()
return None
if(qlbfile.version_tuple < (0,1,1,9)):
# we don't recognize the QLP file version, ditch
qlbfile.read_status = -2
Session.commit()
return None
qlbplate, valid_plate = add_qlp_plate_record(qlplate, qlbfile)
if not valid_plate:
# invalid plate
print "Could not read plate: %s" % path
qlbfile.read_status = -20
Session.commit()
file_lists['unreadable_plates'].append(path)
return None
for well_name, proc_qlwell in sorted(qlplate.analyzed_wells.items()):
# remove empty/blank wells generated by eng group
if (well_name is None or well_name == ''):
del qlplate.analyzed_wells[well_name]
continue
raw_qlwell = None
# TODO: abstract?
well_loc = "%s_%s_RAW.qlb" % (path[:-4], well_name)
# process QLP only
if not os.path.isfile(well_loc):
print "Could not find well file: %s" % well_loc
file_lists['missing_wells'].append(well_loc)
well_file = None
# proceed, as file may just not have been delivered
valid_file = True
else:
well_file, raw_qlwell, valid_file = add_qlb_file_record(file_source, well_loc)
if not valid_file:
print "Invalid well file: %s" % well_loc
file_lists['invalid_wells'].append(well_loc)
continue
qlbwell, valid_well = add_qlb_well_record(well_file, well_name, proc_qlwell, raw_qlwell)
if valid_well:
qlbplate.wells.append(qlbwell)
# bug 829: if there are invalid wells, do not process the plate;
# wait for the well files to complete processing, get on next run
#
#
if file_lists['invalid_wells']:
print "Skipping plate processing (invalid well): %s" % path
Session.rollback()
return None # continue plate
plate_meta = plate_from_qlp(qlbplate)
Session.add(plate_meta)
qlbplate.plate = plate_meta
validation_test = get_product_validation_plate(qlplate, plate_meta)
if not validation_test:
if not apply_setup_to_plate(qlplate, plate_meta):
apply_template_to_plate(qlplate, plate_meta)
# OK, try it now
try:
for well in qlbplate.wells:
if well.file_id != -1:
well.file.read_status = 1
qlbplate.file.read_status = 1
Session.commit()
write_images_stats_for_plate(qlbplate, qlplate, image_source, override_plate_type=plate_type)
Session.commit()
qlbplate.plate.score = Plate.compute_score(qlbplate.plate)
Session.commit()
if validation_test:
validation_test.plate_id = qlbplate.plate.id
Session.add(validation_test)
Session.commit()
file_lists['added_plates'].append(path)
return plate_meta
except Exception, e:
print e
print "Could not process new plate: %s" % path
file_lists['unwritable_plates'].append(path)
Session.rollback()
elif time_equals(mtime_dict[path_id][1], datetime.fromtimestamp(os.stat(path).st_mtime)):
return None
else:
# strategy: reprocess the plate and update.
qlbfile = Session.query(QLBFile).get(mtime_dict[path_id][0])
if not qlbfile:
print "No file for path: %s" % path
return None
elif path.endswith('HFE_Plate.qlp'):
qlbfile.mtime = datetime.fromtimestamp(os.stat(path).st_mtime)
Session.commit()
return None
qlbplates = Session.query(QLBPlate).filter_by(file_id=qlbfile.id).\
options(joinedload_all(QLBPlate.wells, QLBWell.channels)).all()
if not qlbplates:
print "No plate for read file: %s" % path
return None
qlbplate = qlbplates[0]
if not qlbplate.plate_id:
print "No plate for read file (plate deleted): %s" % path
qlbfile.mtime = datetime.fromtimestamp(os.stat(path).st_mtime)
Session.commit()
return None
print "Updating plate %s/%s: %s" % (qlbplate.plate_id, qlbplate.id, path)
qlplate = get_plate(path)
updated = update_qlp_plate_record(qlbplate, qlplate)
if not updated:
print "Could not read updated file"
Session.rollback()
qlbplate.file.read_status = -30
Session.commit()
file_lists['unreadable_plates'].append(path)
return None
# this is basically the same as on add -- abstract?
#
# TODO (GitHub Issue 30): handle case where a previously analyzed well is switched to 'Not Used'
for well_name, proc_qlwell in sorted(qlplate.analyzed_wells.items()):
raw_qlwell = None
# TODO: abstract?
well_loc = "%s_%s_RAW.qlb" % (path[:-4], well_name)
qlbwells = [well for well in qlbplate.wells if well.well_name == well_name]
if not qlbwells:
# add qlb file record
if not os.path.isfile(well_loc):
print "Could not find well file: %s" % well_loc
well_file = None
valid_file = True
file_lists['missing_wells'].append(well_loc)
else:
well_file, raw_qlwell, valid_file = add_qlb_file_record(file_source, well_loc)
if not valid_file:
print "Invalid well file: %s" % well_loc
file_lists['invalid_wells'].append(well_loc)
continue
qlbwell, valid_well = add_qlb_well_record(well_file, well_name, proc_qlwell, raw_qlwell)
if valid_well:
qlbplate.wells.append(qlbwell)
else:
file_lists['invalid_wells'].append(well_loc)
print "Could not add well %s: %s" % (well_name, well_loc)
else:
qlbwell = qlbwells[0]
if not os.path.isfile(well_loc):
print "Could not find well file to update: %s" % well_loc
file_lists['missing_wells'].append(well_loc)
update_qlb_well_record(qlbwell, well_name, proc_qlwell, None)
else:
if qlbwell.file_id == -1:
well_file, raw_qlwell, valid_file = add_qlb_file_record(file_source, well_loc)
if valid_file:
qlbwell.file = well_file
update_qlb_well_record(qlbwell, well_name, proc_qlwell, raw_qlwell)
# in lieu of updating plate meta (though it maybe should be done)
qlbplate.plate.program_version = qlbplate.host_software
try:
for well in qlbplate.wells:
if well.file_id != -1 and well.file:
well.file.read_status = 1
qlbplate.file.read_status = 1
qlbfile.mtime = datetime.fromtimestamp(os.stat(path).st_mtime)
Session.commit()
# this is where updating the dirty bits would come in handy
write_images_stats_for_plate(qlbplate, qlplate, image_source, overwrite=True, override_plate_type=plate_type)
Session.commit()
qlbplate.plate.score = Plate.compute_score(qlbplate.plate)
Session.commit()
file_lists['updated_plates'].append(path)
return qlbplate.plate
except Exception, e:
print e
print "Could not update plate %s/%s: %s" % (qlbplate.plate_id, qlbplate.id, path)
file_lists['unwritable_plates'].append(path)
Session.rollback()
file_metadata = {'run_id': run_id(path_id),
'dirname': os.path.dirname(path_id),
'basename': os.path.basename(path_id),
'read_status': -10,
'type': 'unknown',
'version': '',
'mtime': datetime.fromtimestamp(os.stat(path).st_mtime)}
qlbfile = QLBFile(**file_metadata)
valid_file = False
plate = None
try:
Session.add(qlbfile)
return (qlbfile, plate, valid_file)
except IntegrityError, e:
Session.rollback()
# arbitrary
qlbfile.run_id = duplicate_run_id(path_id)
qlbfile.read_status = -1
return (qlbfile, plate, False)
def add_qlb_file_record(source, path):
"""
Attempt to create a QLP file record. Adds to the current
SQLAlchemy Session object, but does not commit (will
rollback, however, if there is a problem)
Returns (record, valid) tuple
"""
path_id = source.path_id(path)
valid_file = True
def tearDown(self): Session.rollback()
