def __fetch_tubes(self): # Fetches tube (for location data), from the the DB. Uses the tube # barcodes from the library layouts. self.add_debug('Fetch tubes ...') tube_barcodes = [] for lib_pos in self.__library_layout.working_positions(): for barcode in lib_pos.stock_tube_barcodes: tube_barcodes.append(barcode) tube_agg = get_root_aggregate(ITube) tube_agg.filter = cntd(barcode=tube_barcodes) iterator = tube_agg.iterator() while True: try: tube = iterator.next() except StopIteration: break else: self.__tube_map[tube.barcode] = tube if not len(tube_barcodes) == len(self.__tube_map): missing_tubes = [] for tube_barcode in tube_barcodes: if not self.__tube_map.has_key(tube_barcode): missing_tubes.append(tube_barcode) msg = 'Could not find tubes for the following tube barcodes: %s.' \ % (', '.join(sorted(missing_tubes))) self.add_error(msg)
def __get_tube_candidates_for_tubes(self, tube_barcodes, tube_type): """ As far as possible we use the tube aggregate to fetch tubes from the DB. The tubes for the barcodes are converted into :class:`TubeCandidate` objects and mapped onto pools. Valid tubes must contain stock samples and be managed. """ self.__tube_agg.filter = cntd(barcode=tube_barcodes) iterator = self.__tube_agg.iterator() candidate_map = dict() no_stock_sample = [] not_managed = [] while True: try: tube = iterator.next() except StopIteration: break else: if not tube.item_status == ITEM_STATUS_NAMES.MANAGED.upper(): not_managed.append(tube.barcode) continue sample = tube.sample if not isinstance(sample, StockSample): no_stock_sample.append(tube.barcode) continue pool = sample.molecule_design_pool tc = TubeCandidate(pool_id=pool.id, rack_barcode=tube.location.rack.barcode, rack_position=tube.location.position, tube_barcode=tube.barcode, concentration=sample.concentration, volume=sample.volume) # we could store the pool itself to (instead of its ID), but # for some reason the pool entities are not recognised as equal add_list_map_element(candidate_map, pool.id, tc) self.__tube_map[tube.barcode] = tube if len(no_stock_sample) > 0: msg = 'The following %s tubes do not contain stock ' \ 'samples: %s! The referring tubes are replaced, if ' \ 'possible.' % (tube_type, self._get_joined_str(no_stock_sample)) self.add_warning(msg) if len(not_managed) > 0: msg = 'The following %s tubes are not managed: %s. They ' \ 'are replaced, if possible.' \ % (tube_type, self._get_joined_str(not_managed)) self.add_warning(msg) not_found = [] for tube_barcode in tube_barcodes: if not self.__tube_map.has_key(tube_barcode): not_found.append(tube_barcode) if len(not_found) > 0: msg = 'The following %s tubes have not been found in the DB: %s.' \ % (tube_type, self._get_joined_str(not_found)) self.add_warning(msg) return candidate_map
def opt_callback(cls, value, options): # pylint: disable=W0613 agg = get_root_aggregate(option_type) if not ',' in value: val = agg.get_by_slug(value) else: agg.filter = cntd(slug=value.split(',')) val = [ent for ent in agg] return val
def __get_tube_map_for_sector(self, layout): # We fetch all tubes for the given sector layout in one go. tube_agg = get_root_aggregate(ITube) tube_agg.filter = \ cntd(barcode=reduce(add, [pos.stock_tube_barcodes for pos in layout.get_sorted_working_positions()]) ) return (dict([(t.barcode, t) for t in iter(tube_agg)]))
def __get_molecule_design_pools(self): """ Returns the molecule design pool objects for the specified IDs. """ self.add_debug('Checks validity of the molecule design pool IDs ...') ids = self.parser.molecule_design_pool_ids agg = get_root_aggregate(IMoleculeDesignPool) agg.filter = cntd(molecule_design_set_id=ids) db_ids = set() molecule_design_pools = set() stock_concentrations = set() iterator = agg.iterator() while True: try: md_pool = iterator.next() except StopIteration: break else: if self.__molecule_type is None: self.__molecule_type = md_pool.molecule_type else: if not self.__molecule_type == md_pool.molecule_type: msg = 'There is more than one molecule type in the ' \ 'the molecule design set ("%s" and "%s")!' \ % (self.__molecule_type.name, md_pool.molecule_type.name) self.add_error(msg) return None molecule_design_pools.add(md_pool) db_ids.add(md_pool.id) stock_conc = md_pool.default_stock_concentration \ * CONCENTRATION_CONVERSION_FACTOR stock_concentrations.add(stock_conc) if not len(molecule_design_pools) == len(ids): unknown_molecule_design_pools = [] for pool_id in ids: if not pool_id in db_ids: unknown_molecule_design_pools.append(pool_id) msg = 'The following molecule design pool IDs have not be found ' \ 'in the DB: %s' % (unknown_molecule_design_pools) self.add_error(msg) if len(stock_concentrations) > 1: conc = sorted(list(stock_concentrations)) msg = 'The pools in the set have different stock concentrations: ' \ '(shown in nM): %s.' % (', '.join([str(c) for c in conc])) self.add_error(msg) else: self.__stock_concentration = list(stock_concentrations)[0] return molecule_design_pools
def __get_tube_racks(self): self.add_debug('Fetching tube racks.') tube_rack_agg = get_root_aggregate(ITubeRack) query_bcs = self.single_stock_racks if not self.pool_stock_racks is None: query_bcs.extend(self.pool_stock_racks) tube_rack_agg.filter = cntd(barcode=query_bcs) rack_map = dict([(tr.barcode, tr) for tr in tube_rack_agg]) bcs_not_found = set(rack_map.keys()).symmetric_difference(query_bcs) if len(bcs_not_found) > 0: msg = 'The following racks have not been found in the DB: %s!' \ % (', '.join(sorted(bcs_not_found))) self.add_error(msg) return rack_map
def __get_molecule_designs_and_pools(self): # Fetches the molecule designs for all molecule design IDs and pools for # all found pools. self.add_debug('Fetch molecule designs and pools for IDs ...') found_md_ids = set() found_pool_ids = set() i = 0 while True: i += 1 md_ids, pool_id = self.__get_ids_for_row(i) if md_ids is None and pool_id is None: break if pool_id is not None: found_pool_ids.add(pool_id) if md_ids is None: continue for md_id in md_ids: found_md_ids.add(md_id) if len(found_md_ids) > 0: md_filter = cntd(molecule_design_id=found_md_ids) self.__fetch_entities(IMoleculeDesign, md_filter, found_md_ids, 'molecule design', self.__md_map) if len(found_pool_ids) > 0: pool_filter = cntd(molecule_design_set_id=found_pool_ids) self.__fetch_entities(IMoleculeDesignPool, pool_filter, found_pool_ids, 'pool', self.__pool_map)
def __getitem__(self, name): if name == 'completed-iso-plates' and self.iso_type == ISO_TYPES.LAB: # These are the plates that can be used as input for experiment # job scheduling. iso_plate_bcs = [] for iso in self.isos: if iso.status == ISO_STATUS.DONE: for plt in self.__get_completed_iso_plates_for_iso(iso): iso_plate_bcs.append(plt.barcode) iso_plates = get_root_collection(IPlate) iso_plates.filter = cntd(barcode=iso_plate_bcs) result = iso_plates else: result = Member.__getitem__(self, name) return result
def __get_molecule_designs_and_pools(self): # Fetches the molecule designs for all molecule design IDs and pools for # all found pools. self.add_debug("Fetch molecule designs and pools for IDs ...") found_md_ids = set() found_pool_ids = set() i = 0 while True: i += 1 md_ids, pool_id = self.__get_ids_for_row(i) if md_ids is None and pool_id is None: break if pool_id is not None: found_pool_ids.add(pool_id) if md_ids is None: continue for md_id in md_ids: found_md_ids.add(md_id) if len(found_md_ids) > 0: md_filter = cntd(molecule_design_id=found_md_ids) self.__fetch_entities(IMoleculeDesign, md_filter, found_md_ids, "molecule design", self.__md_map) if len(found_pool_ids) > 0: pool_filter = cntd(molecule_design_set_id=found_pool_ids) self.__fetch_entities(IMoleculeDesignPool, pool_filter, found_pool_ids, "pool", self.__pool_map)
def __getitem__(self, name): if name == 'tags': tags_dict = {} design_racks = self.__get_design_racks() for rack in design_racks: for tp in rack.rack_layout.tagged_rack_position_sets: for tag in tp.tags: tags_dict[tag.get_entity().slug] = tag tag_coll = get_root_collection(ITag) tag_coll.filter = cntd(id=[tag.id for tag in tags_dict.values()]) result = tag_coll elif name == 'experiment-design-racks': result = self.__get_design_racks() else: result = Member.__getitem__(self, name) return result
def __get_molecule_design_pools(self): self.add_debug("Fetch molecule design pools for IDs ...") invalid_length = [] found_md_ids = set() for md_ids in self.parser.molecule_design_lists: if len(md_ids) != self.number_molecule_designs: invalid_length.append("-".join([str(md_id) for md_id in md_ids])) continue for md_id in md_ids: found_md_ids.add(md_id) if len(invalid_length) > 0: msg = ( "Some of the specified molecule design pools do not " "have the expected number of molecule designs (%i): %s." % (self.number_molecule_designs, ", ".join(invalid_length)) ) self.add_error(msg) return invalid_type = [] agg = get_root_aggregate(IMoleculeDesignPool) agg.filter = cntd(molecule_design_id=found_md_ids) iterator = agg.iterator() while True: try: md = iterator.next() except StopIteration: break else: if md.molecule_type != self.molecule_type: invalid_type.append(md.id) self.__md_map[md.id] = md # Search for missing molecule designs. if len(found_md_ids) != len(self.__md_map): diff = found_md_ids.symmetric_difference(self.__md_map.keys()) diff = sorted(list(diff)) msg = "The following molecule designs could not be found in " "the DB: %s." % ( ", ".join([str(md_id) for md_id in diff]) ) self.add_error(msg) if len(invalid_type) > 0: msg = ( "The molecule designs in the list have different molecule " "types. Expected: %s. Others (molecule designs): %s." % (self.molecule_type, ", ".join([str(md_id) for md_id in sorted(invalid_type)])) ) self.add_error(msg)
def __get_molecule_design_pools(self): self.add_debug('Fetch molecule design pools for IDs ...') invalid_length = [] found_md_ids = set() for md_ids in self.parser.molecule_design_lists: if len(md_ids) != self.number_molecule_designs: invalid_length.append('-'.join( [str(md_id) for md_id in md_ids])) continue for md_id in md_ids: found_md_ids.add(md_id) if len(invalid_length) > 0: msg = 'Some of the specified molecule design pools do not ' \ 'have the expected number of molecule designs (%i): %s.' \ % (self.number_molecule_designs, ', '.join(invalid_length)) self.add_error(msg) return invalid_type = [] agg = get_root_aggregate(IMoleculeDesignPool) agg.filter = cntd(molecule_design_id=found_md_ids) iterator = agg.iterator() while True: try: md = iterator.next() except StopIteration: break else: if md.molecule_type != self.molecule_type: invalid_type.append(md.id) self.__md_map[md.id] = md # Search for missing molecule designs. if len(found_md_ids) != len(self.__md_map): diff = found_md_ids.symmetric_difference(self.__md_map.keys()) diff = sorted(list(diff)) msg = 'The following molecule designs could not be found in ' \ 'the DB: %s.' % (', '.join([str(md_id) for md_id in diff])) self.add_error(msg) if len(invalid_type) > 0: msg = 'The molecule designs in the list have different molecule ' \ 'types. Expected: %s. Others (molecule designs): %s.' \ % (self.molecule_type, ', '.join([str(md_id) for md_id in sorted(invalid_type)])) self.add_error(msg)
def __check_new_molecule_designs(self): self.add_debug('Checking for new molecule designs.') md_agg = get_root_aggregate(IMoleculeDesign) # Build a map design hash -> registration item for all molecule design # registration items. mdri_map = {} for mdri in self.registration_items: struc_hash = \ MoleculeDesign.make_structure_hash(mdri.chemical_structures) mdri_map.setdefault(struc_hash, []).append(mdri) # Build "contained" specification, filter all existing designs and # build difference set of new designs. md_agg.filter = cntd(structure_hash=mdri_map.keys()) existing_md_map = dict([(md.structure_hash, md) for md in md_agg.iterator()]) # Update registration items with existing molecule designs and # remove from hash map. for struc_hash, md in existing_md_map.iteritems(): mdris = mdri_map.pop(struc_hash) for mdri in mdris: mdri.molecule_design = md self.__new_mdris = list(mdri_map.values())
def __check_tubes(self): tube_agg = get_root_aggregate(ITube) bcs = [getattr(sri, 'tube_barcode') for sri in self.registration_items] self.add_debug('Checking tubes. Barcodes: %s' % bcs) tube_agg.filter = cntd(barcode=bcs) tube_map = dict([(tube.barcode, tube) for tube in tube_agg.iterator()]) new_tubes = [] for sri in self.registration_items: tube_bc = sri.tube_barcode if not tube_bc in tube_map: tube = self.__make_new_tube(sri) tube_agg.add(tube) new_tubes.append(tube) else: # FIXME: Do we need to apply location information here? tube = tube_map[tube_bc] if not self.__validate_tube(tube): continue sri.container = tube self.return_value['tubes'] = new_tubes
def __check_racks(self): rack_agg = get_root_aggregate(ITubeRack) self.add_debug('Checking rack position information.') pos_infos = [(getattr(sri, 'rack_barcode'), getattr(sri, 'rack_position')) for sri in self.registration_items if not (getattr(sri, 'rack_barcode') is None or getattr(sri, 'rack_position') is None)] self.__has_location_info = len(pos_infos) > 0 if self.__has_location_info \ and len(pos_infos) != len(self.registration_items): msg = 'Some sample registration items contain location ' \ 'information (rack barcode and rack position), but ' \ 'not all of them do.' self.add_error(msg) else: new_racks = [] if self.__has_location_info: rack_agg.filter = cntd(barcode=[pos_info[0] for pos_info in pos_infos]) rack_map = dict([(rack.barcode, rack) for rack in rack_agg.iterator()]) for sri in self.registration_items: rack_bc = sri.rack_barcode if not rack_bc in rack_map: # Create new item and add. rack = self.__make_new_rack(sri) rack_agg.add(rack) new_racks.append(rack) rack_map[rack_bc] = rack else: rack = rack_map[rack_bc] if not self.__validate_rack(rack): continue # Update sample registration item. sri.rack = rack self.return_value['racks'] = new_racks
def test_cntd_generator(self): spec = cntd(text_attr=['attr']) self.assert_true(spec.is_satisfied_by(self.candidate))
def get_ids_from_file(cls, file_name): with open(file_name, 'rU') as input_file: ids = [int(line.strip()) for line in input_file] agg = get_root_aggregate(IMoleculeDesignPool) agg.filter = cntd(id=ids) return list(iter(agg))
def __process_molecule_design_pools(self): self.add_debug('Processing molecule design pools.') md_pool_agg = get_root_aggregate(IMoleculeDesignPool) mdpri_hash_map = {} hash_func = MoleculeDesignPool.make_member_hash new_mdpri_map = {} new_mds = self.return_value['molecule_designs'] for mdpri in self.registration_items: # By definition, any mdpri that contains one or more new designs # must be new. We must treat this as a special case because # building member hash values with the new designs does not work # reliably since they may not have been flushed yet. mds = [mdri.molecule_design for mdri in mdpri.molecule_design_registration_items] if any(md in new_mds for md in mds): # We use the *structure* as key for the new pools map here # as this is always available (unlike the design IDs, which # may not have been generated at this point). key = self.__make_new_mdpri_key(mdpri) new_mdpri_map.setdefault(key, []).append(mdpri) else: # For pools that consist only of existing designs, we build # a map with member hashes as keys so we can query with a # single DB call. hash_val = \ hash_func([mdri.molecule_design for mdri in mdpri.molecule_design_registration_items]) mdpri_hash_map.setdefault(hash_val, []).append(mdpri) if len(mdpri_hash_map) > 0: md_pool_agg.filter = cntd(member_hash=mdpri_hash_map.keys()) existing_mdp_map = dict([(mdp.member_hash, mdp) for mdp in md_pool_agg.iterator()]) # Update existing molecule design pool registration items. for hash_val, mdp in existing_mdp_map.iteritems(): mdpris = mdpri_hash_map[hash_val] for mdpri in mdpris: if not mdpri.molecule_design_pool is None \ and mdp.id != mdpri.molecule_design_pool.id: msg = 'The molecule design pool ID (%s) specified ' \ 'in the sample data does not match the ID ' \ 'of the pool that retrieved for the design ' \ 'structure information associated with it.' self.add_error(msg) continue mdpri.molecule_design_pool = mdp else: existing_mdp_map = {} # Determine non-existing molecule design pool registration items and # build up a map (this makes sure the same design is registered at # most once. new_mdp_hashes = \ set(mdpri_hash_map.keys()).difference(existing_mdp_map.keys()) for new_mdp_hash in new_mdp_hashes: mdpris = mdpri_hash_map[new_mdp_hash] for mdpri in mdpris: if not mdpri.molecule_design_pool is None: # This is a case where we supplied both design pool ID *and* # structure information in the data file and the two do not # match (i.e., the structures were not found). msg = 'The molecule design pool ID (%s) specified in the ' \ 'sample data does not match the design structure ' \ 'information associated with it.' self.add_error(msg) continue key = self.__make_new_mdpri_key(mdpri) new_mdpri_map.setdefault(key, []).append(mdpri) if len(new_mdpri_map) > 0: new_md_pools = [] for mdpris in new_mdpri_map.values(): # We use the first mdp registration item to create a # new pool and update all with the latter. md_pool = MoleculeDesignPool( set([mdri.molecule_design for mdri in mdpris[0].molecule_design_registration_items])) md_pool_agg.add(md_pool) new_md_pools.append(md_pool) for mdpri in mdpris: mdpri.molecule_design_pool = md_pool self.return_value['molecule_design_pools'] = new_md_pools
def __get_md_pool_map(self, pool_ids): agg = get_root_aggregate(IMoleculeDesignPool) agg.filter = cntd(id=pool_ids) return dict([(md.id, md) for md in agg.iterator()])
def __get_tube_map(self, tube_barcodes): tb_agg = get_root_aggregate(ITube) tb_agg.filter = cntd(barcode=tube_barcodes) return dict([(tb.barcode, tb) for tb in tb_agg.iterator()])
def __get_molecule_types(self): agg = get_root_aggregate(IMoleculeType) agg.filter = cntd(id=self.molecule_type_ids) return list(agg.iterator())