def batch_plate_do_upload(self, id=None): batch_plate = self.__load_batch_plate(id) if not batch_plate: abort(404) box2 = self.form_result['box2'] plate = self.form_result['plate'] plate_type = batch_plate.batch.plate_type if plate_type.code == 'fvtitr' and len(plate.analyzed_wells) == 4: # if four wells, it's really a MFGCC (FVTITR FAM+/VIC+ should have 2) plate_type = Session.query(PlateType).filter_by(code='mfgcc').one() plateobj = save_plate_from_upload_request(request.POST['plate'], plate, box2, plate_type_obj=plate_type) # I want to put this in the form validator, but it's field dependent, so not right now if plate_type.code in ('mfgcc', 'bcc'): ok, message = validate_colorcomp_plate(plate) if not ok: response = self._batch_plate_upload_base(id) Session.rollback() return h.render_bootstrap_form(response, errors={'plate': message}) Session.add(plateobj) if batch_plate.batch.plate_type.code == plate_type.code: batch_plate.plate = plateobj else: batch_plate.secondary_plate = plateobj batch_plate.qc_plate = self.form_result['qc_plate'] batch_plate.plate_notes = self.form_result['plate_notes'] Session.commit() session['flash'] = 'Plate linked.' session.save() return redirect(url(controller='metrics', action='per_plate', id=plateobj.id))
def command(self): self.load_wsgi_app() unknown_plates = Session.query(QLBPlate).filter(QLBPlate.plate == None) for qlbplate in unknown_plates: try: plate = plate_from_qlp(qlbplate) Session.add(plate) qlbplate.plate = plate Session.commit() except Exception: Session.rollback()
def tearDown(self): Session.rollback() # catches pending changes Session.delete(self.plateTemplate) Session.delete(self.shortTemplate) Session.delete(self.qlbPlate) Session.delete(self.plateVariant) Session.delete(self.unknownPlate) Session.delete(self.nameVariant) Session.delete(self.unknownName) Session.commit() super(TestNamePlate, self).tearDown()
def create(self): model = self.__form_to_model(self.form_result) try: Session.commit() session['flash'] = 'Category saved.' session.save() return redirect(url(controller='assay_group', action='list')) except IntegrityError, e: Session.rollback() session['flash'] = 'There is already a category named %s.' % model.name session['flash_class'] = 'error' session.save() return redirect(url(controller='assay_group', action='new'))
def delete(self, id=None): if id is None: abort(404) tag = Session.query(SequenceGroupTag).get(int(id)) if not tag: abort(404) try: tag.sequence_groups = [] Session.delete(tag) Session.commit() except Exception, e: Session.rollback() session['flash'] = 'Could not delete category.' session['flash_class'] = 'error' return redirect(url(controller='assay_group', action='edit', id=id))
def command(self): self.load_wsgi_app() sequence_groups = Session.query(SequenceGroup).all() for sg in sequence_groups: try: print "Backfilling channels for %s" % sg.name channels = Session.query(QLBWellChannel).filter_by(target=sg.name) for c in channels: c.sequence_group_id = sg.id Session.commit() except Exception: import sys, traceback traceback.print_exc(file=sys.stdout) Session.rollback() continue
def add_qlp_plate_record(qlplate, qlbfile): """ Create a QLBPlate object based off a new QLBFile. Adds to the current SQLAlchemy Session object, but does not commit (will rollback, however, if there is a problem) """ valid_plate = True plate = None try: plate = QLBPlate() set_qlp_plate_record_attrs(plate, qlplate) plate.file = qlbfile Session.add(plate) except Exception, e: print e Session.rollback() valid_plate = False
def save_circuit(self): log = Session.query(Box2Log).get(self.form_result['log_id']) if not log: abort(404) try: circ = Box2Circuit(name=self.form_result['name'], log_template_id=self.form_result['log_id']) Session.add(circ) Session.commit() log.box2_circuit_id = circ.id Session.commit() session['flash'] = 'Configuration for %s updated and circuit "%s" created.' % (log.box2.name, circ.name) session.save() redirect(url(controller='admin', action='reader_history', id=log.box2.code)) except exc.IntegrityError, e: Session.rollback() session['flash'] = 'There is already a circuit by that name.' session['flash_class'] = 'error' session.save() redirect(url(controller='admin', action='circuit', id=log.id))
def save(self, id=None): if not id: abort(404) model = Session.query(SequenceGroupTag).get(int(id)) if not model: abort(404) model = self.__form_to_model(self.form_result, model=model) try: Session.commit() session['flash'] = 'Category saved.' session.save() return redirect(url(controller='assay_group', action='list')) except IntegrityError, e: Session.rollback() session['flash'] = 'There is already a category named %s.' % model.name session['flash_class'] = 'error' session.save() return redirect(url(controller='assay_group', action='edit', id=id))
def __update_batch_record(self, record=None): new_record = False if not record: new_record = True record = ManufacturingPlateBatch(name=self.form_result['name'], plate_type_id=self.form_result['plate_type'], creation_date=self.form_result['creation_date'], default_dg_method=self.form_result['dg_method']) record.creator_id = self.form_result['creator_id'] record.notes = self.form_result['notes'] record.fam_hi_size = self.form_result['fam_hi_size'] record.vic_hi_size = self.form_result['vic_hi_size'] record.hex_hi_size = self.form_result['hex_hi_size'] try: Session.add(record) Session.commit() except IntegrityError, e: Session.rollback() raise e
def create(self, id): """ KIND OF UNSAFE. Creates/registers a reader. """ serial = self.form_result['serial'] reader_type = self.form_result['reader_type'] if reader_type == Box2.READER_TYPE_WHOLE: ## check name is in correct format... if not re.search( '^771BR\d{4}$', serial ): session['flash'] = 'New Reader name must follow "771BR####" convention' session['flash_class'] = 'error' session.save() return redirect(url(controller='admin', action='register', id=id)) elif ( len( serial ) > 15 ): session['flash'] = 'Reader name can not contain more then 15 characters' session['flash_class'] = 'error' session.save() return redirect(url(controller='admin', action='register', id=id)) code = 'p%s' % serial elif reader_type == Box2.READER_TYPE_FLUIDICS_MODULE: serial = serial.upper() if ( len( serial ) > 15 ): session['flash'] = 'Fluidics name can not contain more then 15 characters' session['flash_class'] = 'error' session.save() return redirect(url(controller='admin', action='register', id=id)) code = 'f%s' % serial elif reader_type == Box2.READER_TYPE_DETECTOR_MODULE: ## check to make sure someone isn't acidently adding a DR if re.search( '^771BR\d*', serial ): session['flash'] = 'New Detector module names do not contain "771BR" ' session['flash_class'] = 'error' session.save() return redirect(url(controller='admin', action='register', id=id) ) elif ( len( serial ) > 15 ): session['flash'] = 'Detector name can not contain more then 15 characters' session['flash_class'] = 'error' session.save() return redirect(url(controller='admin', action='register', id=id)) serial = serial.upper() code = 'd%s' % serial # check and catch if reader already exists box2 = Session.query(Box2).filter_by(code=code).first() if box2: session['flash'] = 'Unit %s is already registered.' % self.form_result['serial'] session['flash_class'] = 'error' session.save() return redirect(url(controller='admin', action='register', id=id)) #If not exists create one instead if reader_type == Box2.READER_TYPE_WHOLE: src_dir = "DR %s" % serial reader = Box2(name=u'Prod %s' % serial, code=code, src_dir=src_dir, \ reader_type = Box2.READER_TYPE_WHOLE, \ fileroot=config['qlb.fileroot.register_fileroot'], active=True) elif reader_type == Box2.READER_TYPE_FLUIDICS_MODULE: src_dir="FM %s" % serial reader = Box2(name=u'QL-FSA %s' % serial, code=code, src_dir=src_dir, \ reader_type = Box2.READER_TYPE_FLUIDICS_MODULE, \ fileroot=config['qlb.fileroot.register_fileroot'], active=True) elif reader_type == Box2.READER_TYPE_DETECTOR_MODULE: src_dir="DM %s" % serial reader = Box2(name=u'DET %s' % serial, code=code, src_dir=src_dir, \ reader_type = Box2.READER_TYPE_DETECTOR_MODULE, \ fileroot=config['qlb.fileroot.register_fileroot'], active=True) Session.add(reader) reader.active = True local_plate_source = QLBPlateSource(config, [reader]) dirname = local_plate_source.real_path(reader.fileroot, src_dir) try: os.mkdir(dirname) Session.commit() session['flash'] = 'Unit %s registered.' % serial session.save() except Exception, e: session['flash'] = 'Could not create a directory for unit %s' % serial session['flash_class'] = 'error' session.save() Session.rollback() return redirect(url(controller='admin', action='register', id=id))
def __scan_plate(file_source, image_source, path_id, path, mtime_dict, plate_type=None, file_lists=None): """ The method responsible for taking a QLP file on disk and creating thumbnails and adding/updating records in the database based off the contents of that file. This is a nasty abstraction, I apologize. TODO: make this more natural along add/update line, do not require use of mtime_dict or file_list (or return file_list as files touched) Returns the Plate object of the added/updated plate, or None if there was no touch/error. :param file_source: The source of the QLP files (QLStorageSource) :param image_source: The source/sink of thumbnails (QLStorageSource) :param path_id: The unique identifier of the plate file. Computed by run_id() :param path: The actual file path of the QLP. :param mtime_dict: A mapping between plates and their last updated times. This will indicate whether or not a plate is 'dirty' with respect to the DB. :param plate_type: A plate type. Supplying this will indicate that the special metrics corresponding to that plate type should be computed during the scan. :param file_lists: A logging object used in the scan to record files that are missing, poorly processed, etc. Side-effected by this method. """ if not file_lists: file_lists = defaultdict(list) # if the file is not being tracked, attempt to add it if not mtime_dict.has_key(path_id): print "Adding plate: %s" % path qlbfile, qlplate, valid_file = add_qlp_file_record(file_source, path) if not valid_file: print "Invalid file: %s" % path file_lists['invalid_plates'].append(path) return None elif path.endswith('HFE_Plate.qlp'): qlbfile.read_status = -7 print "Ignoring HFE Plate: %s" % path Session.commit() return None elif qlbfile.version is 'Unknown': qlbfile.read_status = -8 print "Ignoring plate run with unknown QS version: %s" % path Session.commit() return None if(qlbfile.version_tuple < (0,1,1,9)): # we don't recognize the QLP file version, ditch qlbfile.read_status = -2 Session.commit() return None qlbplate, valid_plate = add_qlp_plate_record(qlplate, qlbfile) if not valid_plate: # invalid plate print "Could not read plate: %s" % path qlbfile.read_status = -20 Session.commit() file_lists['unreadable_plates'].append(path) return None for well_name, proc_qlwell in sorted(qlplate.analyzed_wells.items()): # remove empty/blank wells generated by eng group if (well_name is None or well_name == ''): del qlplate.analyzed_wells[well_name] continue raw_qlwell = None # TODO: abstract? well_loc = "%s_%s_RAW.qlb" % (path[:-4], well_name) # process QLP only if not os.path.isfile(well_loc): print "Could not find well file: %s" % well_loc file_lists['missing_wells'].append(well_loc) well_file = None # proceed, as file may just not have been delivered valid_file = True else: well_file, raw_qlwell, valid_file = add_qlb_file_record(file_source, well_loc) if not valid_file: print "Invalid well file: %s" % well_loc file_lists['invalid_wells'].append(well_loc) continue qlbwell, valid_well = add_qlb_well_record(well_file, well_name, proc_qlwell, raw_qlwell) if valid_well: qlbplate.wells.append(qlbwell) # bug 829: if there are invalid wells, do not process the plate; # wait for the well files to complete processing, get on next run # # if file_lists['invalid_wells']: print "Skipping plate processing (invalid well): %s" % path Session.rollback() return None # continue plate plate_meta = plate_from_qlp(qlbplate) Session.add(plate_meta) qlbplate.plate = plate_meta validation_test = get_product_validation_plate(qlplate, plate_meta) if not validation_test: if not apply_setup_to_plate(qlplate, plate_meta): apply_template_to_plate(qlplate, plate_meta) # OK, try it now try: for well in qlbplate.wells: if well.file_id != -1: well.file.read_status = 1 qlbplate.file.read_status = 1 Session.commit() write_images_stats_for_plate(qlbplate, qlplate, image_source, override_plate_type=plate_type) Session.commit() qlbplate.plate.score = Plate.compute_score(qlbplate.plate) Session.commit() if validation_test: validation_test.plate_id = qlbplate.plate.id Session.add(validation_test) Session.commit() file_lists['added_plates'].append(path) return plate_meta except Exception, e: print e print "Could not process new plate: %s" % path file_lists['unwritable_plates'].append(path) Session.rollback() elif time_equals(mtime_dict[path_id][1], datetime.fromtimestamp(os.stat(path).st_mtime)): return None else: # strategy: reprocess the plate and update. qlbfile = Session.query(QLBFile).get(mtime_dict[path_id][0]) if not qlbfile: print "No file for path: %s" % path return None elif path.endswith('HFE_Plate.qlp'): qlbfile.mtime = datetime.fromtimestamp(os.stat(path).st_mtime) Session.commit() return None qlbplates = Session.query(QLBPlate).filter_by(file_id=qlbfile.id).\ options(joinedload_all(QLBPlate.wells, QLBWell.channels)).all() if not qlbplates: print "No plate for read file: %s" % path return None qlbplate = qlbplates[0] if not qlbplate.plate_id: print "No plate for read file (plate deleted): %s" % path qlbfile.mtime = datetime.fromtimestamp(os.stat(path).st_mtime) Session.commit() return None print "Updating plate %s/%s: %s" % (qlbplate.plate_id, qlbplate.id, path) qlplate = get_plate(path) updated = update_qlp_plate_record(qlbplate, qlplate) if not updated: print "Could not read updated file" Session.rollback() qlbplate.file.read_status = -30 Session.commit() file_lists['unreadable_plates'].append(path) return None # this is basically the same as on add -- abstract? # # TODO (GitHub Issue 30): handle case where a previously analyzed well is switched to 'Not Used' for well_name, proc_qlwell in sorted(qlplate.analyzed_wells.items()): raw_qlwell = None # TODO: abstract? well_loc = "%s_%s_RAW.qlb" % (path[:-4], well_name) qlbwells = [well for well in qlbplate.wells if well.well_name == well_name] if not qlbwells: # add qlb file record if not os.path.isfile(well_loc): print "Could not find well file: %s" % well_loc well_file = None valid_file = True file_lists['missing_wells'].append(well_loc) else: well_file, raw_qlwell, valid_file = add_qlb_file_record(file_source, well_loc) if not valid_file: print "Invalid well file: %s" % well_loc file_lists['invalid_wells'].append(well_loc) continue qlbwell, valid_well = add_qlb_well_record(well_file, well_name, proc_qlwell, raw_qlwell) if valid_well: qlbplate.wells.append(qlbwell) else: file_lists['invalid_wells'].append(well_loc) print "Could not add well %s: %s" % (well_name, well_loc) else: qlbwell = qlbwells[0] if not os.path.isfile(well_loc): print "Could not find well file to update: %s" % well_loc file_lists['missing_wells'].append(well_loc) update_qlb_well_record(qlbwell, well_name, proc_qlwell, None) else: if qlbwell.file_id == -1: well_file, raw_qlwell, valid_file = add_qlb_file_record(file_source, well_loc) if valid_file: qlbwell.file = well_file update_qlb_well_record(qlbwell, well_name, proc_qlwell, raw_qlwell) # in lieu of updating plate meta (though it maybe should be done) qlbplate.plate.program_version = qlbplate.host_software try: for well in qlbplate.wells: if well.file_id != -1 and well.file: well.file.read_status = 1 qlbplate.file.read_status = 1 qlbfile.mtime = datetime.fromtimestamp(os.stat(path).st_mtime) Session.commit() # this is where updating the dirty bits would come in handy write_images_stats_for_plate(qlbplate, qlplate, image_source, overwrite=True, override_plate_type=plate_type) Session.commit() qlbplate.plate.score = Plate.compute_score(qlbplate.plate) Session.commit() file_lists['updated_plates'].append(path) return qlbplate.plate except Exception, e: print e print "Could not update plate %s/%s: %s" % (qlbplate.plate_id, qlbplate.id, path) file_lists['unwritable_plates'].append(path) Session.rollback()
file_metadata = {'run_id': run_id(path_id), 'dirname': os.path.dirname(path_id), 'basename': os.path.basename(path_id), 'read_status': -10, 'type': 'unknown', 'version': '', 'mtime': datetime.fromtimestamp(os.stat(path).st_mtime)} qlbfile = QLBFile(**file_metadata) valid_file = False plate = None try: Session.add(qlbfile) return (qlbfile, plate, valid_file) except IntegrityError, e: Session.rollback() # arbitrary qlbfile.run_id = duplicate_run_id(path_id) qlbfile.read_status = -1 return (qlbfile, plate, False) def add_qlb_file_record(source, path): """ Attempt to create a QLP file record. Adds to the current SQLAlchemy Session object, but does not commit (will rollback, however, if there is a problem) Returns (record, valid) tuple """ path_id = source.path_id(path) valid_file = True
def tearDown(self): Session.rollback()