class VidMapper(object): def __init__(self, host, user, passwd, logger, study): self.logger = logger self.study = study self.kb = KB(driver="omero")(host, user, passwd) plates = self.kb.get_objects(self.kb.TiterPlate) self.logger.info("fetched %d plates" % len(plates)) self.plate_map = {} self.enroll_map = {} for p in plates: self.plate_map[p.omero_id] = p.barcode s = self.kb.get_study(self.study) enrolls = self.kb.get_enrolled(s) self.logger.info("fetched %d enrollments" % len(enrolls)) for e in enrolls: self.logger.debug('Retrieving wells for %s' % e.studyCode) wells = [ w for w in self.kb.get_vessels_by_individual( e.individual, "PlateWell") ] self.enroll_map[e.studyCode] = wells def map_vid(self, r): en_code = r["source"] pl_barcode = en_code.split("|")[1] try: wells = self.enroll_map[en_code] except KeyError: msg = "%s is not enrolled in %s" % (en_code, self.study) self.logger.error(msg) raise ValueError(msg) self.logger.info("found %d wells for %s" % (len(wells), en_code)) imm_wells = [ w for w in wells if self.plate_map[w.container.omero_id] == pl_barcode ] if len(imm_wells) > 1: msg = ("more than 1 (%d) immuno wells for plate %s" % (len(imm_wells), pl_barcode)) self.logger.error(msg) raise ValueError(msg) elif len(imm_wells) == 0: msg = "no immuno well for plate %s" % pl_barcode self.logger.warn(msg) raise ValueError(msg) else: r["source"] = imm_wells[0].id
class App(object): def __init__(self, host, user, passwd): self.kb = KB(driver='omero')(host, user, passwd) #FIXME we need to do this to sync with the DB idea of the enums. self.kb.Gender.map_enums_values(self.kb) self.logger = logging.getLogger() def get_ehr_iterator(self): inds = self.kb.get_objects(self.kb.Individual) inds_by_vid = dict([(i.id, i) for i in inds]) for e in self.kb.get_ehr_iterator(): if not e[0] in inds_by_vid: #FIXME we need to do this for potential stray records left by testing continue yield (inds_by_vid[e[0]], e[1]) def count_males(self, inds): return sum([i.gender == self.kb.Gender.MALE for i in inds]) def do_enrollment(self): controls = [] affected = [] # FIXME in a future version, we will be able to issue actual AQL # queries. for i, ehr in self.get_ehr_iterator(): if ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1): affected.append(i) elif ehr.matches(EXCLUSION): # FIXME we assume that this is enough controls.append(i) # A more sophisticated example # will keep only records where age of onset is between 10y and 15y # for i, ehr in self.get_ehr_iterator(): # if (ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1) # and # ehr.matches(DIAGNOSIS, DIAGNOSIS_AGE_OF_ONSET, ("10y", "15y")): # affected.append(i) print ('there are %d controls [%d male]' % (len(controls), self.count_males(controls))) print ('there are %d affected [%d male] with E10' % (len(affected), self.count_males(affected)))
class VidMapper(object): def __init__(self, host, user, passwd, logger, study): self.logger = logger self.study = study self.kb = KB(driver="omero")(host, user, passwd) plates = self.kb.get_objects(self.kb.TiterPlate) self.logger.info("fetched %d plates" % len(plates)) self.plate_map = {} self.enroll_map = {} for p in plates: self.plate_map[p.omero_id] = p.barcode s = self.kb.get_study(self.study) enrolls = self.kb.get_enrolled(s) self.logger.info("fetched %d enrollments" % len(enrolls)) for e in enrolls: self.logger.debug("Retrieving wells for %s" % e.studyCode) wells = [w for w in self.kb.get_vessels_by_individual(e.individual, "PlateWell")] self.enroll_map[e.studyCode] = wells def map_vid(self, r): en_code = r["source"] pl_barcode = en_code.split("|")[1] try: wells = self.enroll_map[en_code] except KeyError: msg = "%s is not enrolled in %s" % (en_code, self.study) self.logger.error(msg) raise ValueError(msg) self.logger.info("found %d wells for %s" % (len(wells), en_code)) imm_wells = [w for w in wells if self.plate_map[w.container.omero_id] == pl_barcode] if len(imm_wells) > 1: msg = "more than 1 (%d) immuno wells for plate %s" % (len(imm_wells), pl_barcode) self.logger.error(msg) raise ValueError(msg) elif len(imm_wells) == 0: msg = "no immuno well for plate %s" % pl_barcode self.logger.warn(msg) raise ValueError(msg) else: r["source"] = imm_wells[0].id
class App(object): def __init__(self, host, user, passwd): self.kb = KB(driver="omero")(host, user, passwd) # FIXME we need to do this to sync with the DB idea of the enums. self.kb.Gender.map_enums_values(self.kb) self.logger = logging.getLogger() def get_ehr_iterator(self): inds = self.kb.get_objects(self.kb.Individual) inds_by_vid = dict([(i.id, i) for i in inds]) for e in self.kb.get_ehr_iterator(): if not e[0] in inds_by_vid: # FIXME we need to do this for potential stray records left by testing continue yield (inds_by_vid[e[0]], e[1]) def count_males(self, inds): return sum([i.gender == self.kb.Gender.MALE for i in inds]) def do_enrollment(self): controls = [] affected = [] # FIXME in a future version, we will be able to issue actual AQL # queries. for i, ehr in self.get_ehr_iterator(): if ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1): affected.append(i) elif ehr.matches(EXCLUSION): # FIXME we assume that this is enough controls.append(i) # A more sophisticated example # will keep only records where age of onset is between 10y and 15y # for i, ehr in self.get_ehr_iterator(): # if (ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1) # and # ehr.matches(DIAGNOSIS, DIAGNOSIS_AGE_OF_ONSET, ("10y", "15y")): # affected.append(i) print("there are %d controls [%d male]" % (len(controls), self.count_males(controls))) print("there are %d affected [%d male] with E10" % (len(affected), self.count_males(affected)))
def main(): parser = make_parser() args = parser.parse_args() kb = KB(driver='omero')(args.host, args.user, args.passwd) by_label = dict(((x.label, x) for x in kb.get_objects(kb.GenotypeDataSample))) msets = {} itsv = csv.DictReader(args.data_samples, delimiter='\t') otsv = csv.DictWriter(open(args., mode='w'), fieldnames=['path', 'data_sample_label', 'mimetype', 'size', 'sha1'], delimiter='\t') otsv.writeheader() for r in itsv: ds_label = r['label'] logger.info('Gathering info on %s' % ds_label) if ds_label not in by_label: logger.critical('There is no GenotypeDataSample with label %s' % label) sys.exit(1) ds = by_label[ds_label] # FIXME if ds.spnMarkersSet.omero_id not in msets: ms = ds.snpMarkersSet ms.load_markers() msets[ds.snpMarkersSet.omero_id] = ms ms = msets[ds.snpMarkersSet.omero_id] fname = ds_label + '_do.ssc' make_data_object(ds.id, fname, ms) size = os.stat(fname).st_size sha1 = compute_sha1(fname) otsv.writerow({ 'path' : 'file://' + os.path.realpath(fname), 'data_sample' : ds.id, 'mimetype' : mimetypes.SSC_FILE, 'size' : size, 'sha1' : sha1, })
def main(argv): parser = make_parser() args = parser.parse_args(argv) log_level = getattr(logging, args.loglevel) kwargs = {'format' : LOG_FORMAT, 'datefmt' : LOG_DATEFMT, 'level' : log_level} if args.logfile: kwargs['filename'] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() kb = KB(driver='omero')(args.host, args.user, args.passwd) logger.info('Loading GenotypeDataSample objects') dsamples = kb.get_objects(kb.GenotypeDataSample) logger.info('Loaded %d objects' % len(dsamples)) logger.info('Loading SNPMarkersSet') query = 'SELECT snpm FROM SNPMarkersSet snpm WHERE snpm.label = :mset_label' mset = kb.find_all_by_query(query, {'mset_label' : args.marker_set})[0] if not mset: logger.error('Unable to load SNPMarkersSet with label %s' % args.marker_set) sys.exit(2) else: logger.info('Object loaded') gdo_iterator = kb.get_gdo_iterator(mset, dsamples[:args.fetch_size]) gdos = [] logger.info('Loading GDOs') for gdo in gdo_iterator: logger.info(gdo['vid']) gdos.append(gdo) logger.debug('%d/%d GDOs loaded' % (len(gdos), args.fetch_size)) logger.info('Loaded %d GDOs' % len(gdos))
def main(argv): parser = make_parser() args = parser.parse_args(argv) log_level = getattr(logging, args.loglevel) kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level} if args.logfile: kwargs['filename'] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() kb = KB(driver='omero')(args.host, args.user, args.passwd) logger.info('Loading GenotypeDataSample objects') dsamples = kb.get_objects(kb.GenotypeDataSample) logger.info('Loaded %d objects' % len(dsamples)) logger.info('Loading SNPMarkersSet') query = 'SELECT snpm FROM SNPMarkersSet snpm WHERE snpm.label = :mset_label' mset = kb.find_all_by_query(query, {'mset_label': args.marker_set})[0] if not mset: logger.error('Unable to load SNPMarkersSet with label %s' % args.marker_set) sys.exit(2) else: logger.info('Object loaded') gdo_iterator = kb.get_gdo_iterator(mset, dsamples[:args.fetch_size]) gdos = [] logger.info('Loading GDOs') for gdo in gdo_iterator: logger.info(gdo['vid']) gdos.append(gdo) logger.debug('%d/%d GDOs loaded' % (len(gdos), args.fetch_size)) logger.info('Loaded %d GDOs' % len(gdos))
except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logger.info('Retrieving diagnosis records') ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' % DIAGNOSIS_ARCHETYPE) ehr_records.extend( kb.get_ehr_records('(archetype == "%s") & (valid == True)' % EXCL_DIAG_ARCHETYPE)) logger.info('%d records retrieved' % len(ehr_records)) logger.info('Loading actions') actions = kb.get_objects(kb.Action) act_map = {} for act in actions: act_map[act.id] = act.context.label logger.info('%d actions loaded' % len(act_map)) with open(args.ofile, 'w') as out_file: writer = csv.DictWriter( out_file, ['study', 'individual', 'timestamp', 'diagnosis'], delimiter='\t') writer.writeheader() for rec in ehr_records: writer.writerow(extract_csv_row(rec, act_map)) logger.info('Job completed')
passwd = args.passwd or vlu.ome_passwd() except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logger.info('Retrieving diagnosis records') ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' % DIAGNOSIS_ARCHETYPE) ehr_records.extend(kb.get_ehr_records('(archetype == "%s") & (valid == True)' % EXCL_DIAG_ARCHETYPE)) logger.info('%d records retrieved' % len(ehr_records)) logger.info('Loading actions') actions = kb.get_objects(kb.Action) act_map = {} for act in actions: act_map[act.id] = act.context.label logger.info('%d actions loaded' % len(act_map)) with open(args.ofile, 'w') as out_file: writer = csv.DictWriter(out_file, ['study', 'individual', 'timestamp', 'diagnosis'], delimiter='\t') writer.writeheader() for rec in ehr_records: writer.writerow(extract_csv_row(rec, act_map)) logger.info('Job completed')
logger = logging.getLogger() try: host = args.host or vlu.ome_host() user = args.user or vlu.ome_user() passwd = args.passwd or vlu.ome_passwd() except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) enrolls_map = get_enrollments_lookup(kb, logger) logger.info('Loading Individuals') inds = kb.get_objects(kb.Individual) logger.info('Loaded %d Individuals' % len(inds)) with open(args.in_file) as in_file, open(args.out_file, 'w') as out_file: ts = long(time.time()) reader = csv.DictReader(in_file, delimiter='\t') writer = csv.DictWriter(out_file, ['study', 'individual', 'timestamp', 'birth_date', 'birth_place', 'birth_place_district'], delimiter='\t') writer.writeheader() for row in reader: try: writer.writerow({'study' : row['individual'].split(':')[0], 'individual' : enrolls_map[row['individual']].individual.id, 'timestamp' : ts,
class UniverseApplication(OrigUniverseApplication): def __init__(self, **kwargs): omero_host = vlu.ome_host() omero_user = vlu.ome_user() omero_passwd = vlu.ome_passwd() self.kb = KB(driver='omero')(omero_host, omero_user, omero_passwd) super(UniverseApplication, self).__init__(**kwargs) self.config.omero_default_host = kwargs.get('omero_default_host') self.config.omero_default_user = kwargs.get('omero_default_user') self.config.omero_default_passwd = kwargs.get('omero_default_passwd') self.config.vl_loglevel = kwargs.get('vl_loglevel', 'INFO') self.config.vl_import_enabled_users = kwargs.get('vl_import_enabled_users') @property def known_studies(self): studies = self.kb.get_objects(self.kb.Study) if studies: return [(s.label, s.description or '') for s in studies] else: return [] @property def known_scanners(self): scanners = self.kb.get_objects(self.kb.Scanner) if scanners: return [(s.id, s.label, s.physicalLocation) for s in scanners] else: return [] @property def known_map_vid_source_types(self): from bl.vl.app.kb_query.map_vid import MapVIDApp sources = MapVIDApp.SUPPORTED_SOURCE_TYPES if sources: return sources else: return [] @property def known_marker_sets(self): msets = self.kb.get_objects(self.kb.SNPMarkersSet) if msets: return [(ms.id, ms.label) for ms in msets] else: return [] @property def known_data_collections(self): dcolls = self.kb.get_objects(self.kb.DataCollection) if dcolls: return [(dc.id, dc.label) for dc in dcolls] else: return [] @property def known_vessels_collections(self): vcolls = self.kb.get_objects(self.kb.VesselsCollection) if vcolls: return [(vc.id, vc.label) for vc in vcolls] else: return [] @property def known_titer_plates(self): plates = self.kb.get_objects(self.kb.TiterPlate) if plates: return [(pl.barcode, pl.label) for pl in plates if pl.barcode] else: return [] @property def known_vessel_status(self): vstatus = self.kb.get_objects(self.kb.VesselStatus) if vstatus: return [(v.omero_id, v.enum_label()) for v in vstatus] else: return [] @property def known_data_sample_status(self): dsstatus = self.kb.get_objects(self.kb.DataSampleStatus) if dsstatus: return [(ds.omero_id, ds.enum_label()) for ds in dsstatus] else: return [] @property def known_hardware_devices(self): hdev = self.kb.get_objects(self.kb.HardwareDevice) if hdev: return [(h.id, h.label) for h in hdev] else: return [] @property def known_software_devices(self): sdev = self.kb.get_objects(self.kb.SoftwareDevice) if sdev: return [(s.id, s.label) for s in sdev] else: return [] @property def known_devices(self): dev = self.kb.get_objects(self.kb.Device) if dev: return [(d.id, d.label) for d in dev] else: return []
logger = logging.getLogger() try: host = args.host or vlu.ome_host() user = args.user or vlu.ome_user() passwd = args.passwd or vlu.ome_passwd() except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) enrolls_map = get_enrollments_lookup(kb, logger) logger.info('Loading Individuals') inds = kb.get_objects(kb.Individual) logger.info('Loaded %d Individuals' % len(inds)) with open(args.in_file) as in_file, open(args.out_file, 'w') as out_file: ts = long(time.time()) reader = csv.DictReader(in_file, delimiter='\t') writer = csv.DictWriter(out_file, [ 'study', 'individual', 'timestamp', 'birth_date', 'birth_place', 'birth_place_district' ], delimiter='\t') writer.writeheader() for row in reader: try: writer.writerow({ 'study':
class Core(object): def __init__(self, host=None, user=None, passwd=None, group=None, keep_tokens=1, study_label=None, logger=None): self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens) self.logger = logger if logger else logging.getLogger() self.record_counter = 0 self.default_study = None if study_label: s = self.kb.get_study(study_label) if not s: raise ValueError('No known study with label %s' % study_label) self.logger.info('Selecting %s[%d,%s] as default study' % (s.label, s.omero_id, s.id)) self.default_study = s @classmethod def find_action_setup_conf(klass, args): action_setup_conf = {} for x in dir(args): if not (x.startswith('_') or x.startswith('func')): action_setup_conf[x] = getattr(args, x) # HACKS action_setup_conf['ifile'] = action_setup_conf['ifile'].name action_setup_conf['ofile'] = action_setup_conf['ofile'].name action_setup_conf['report_file'] = action_setup_conf[ 'report_file'].name return action_setup_conf @classmethod def get_action_setup_options(klass, record, action_setup_conf=None, object_history=None): options = {} if 'options' in record and record['options']: kvs = record['options'].split(',') for kv in kvs: k, v = kv.split('=') options[k] = v if action_setup_conf: options['importer_setup'] = action_setup_conf if object_history: options['object_history'] = object_history return json.dumps(options) def get_device(self, label, maker, model, release): device = self.kb.get_device(label) if not device: self.logger.debug('creating a device') device = self.kb.create_device(label, maker, model, release) return device def get_action_setup(self, label, conf): """ Return the ActionSetup corresponding to label if there is one, else create a new one using conf. """ asetup = self.kb.get_action_setup(label) if not asetup: kb_conf = { 'label': label, 'conf': json.dumps(conf), } asetup = self.kb.factory.create(self.kb.ActionSetup, kb_conf).save() return asetup def get_study(self, label): if self.default_study: return self.default_study study = self.kb.get_study(label) if not study: study = self.kb.factory.create(self.kb.Study, { 'label': label }).save() return study def find_study(self, records): study_label = records[0]['study'] for r in records: if r['study'] != study_label: m = 'all records should have the same study label' self.logger.critical(m) raise ValueError(m) return self.get_study(study_label) def find_klass(self, col_name, records): o_type = records[0][col_name] for r in records: if r[col_name] != o_type: m = 'all records should have the same %s' % col_name self.logger.critical(m) raise ValueError(m) return getattr(self.kb, o_type) def __preload_items__(self, key_field, klass, preloaded): objs = self.kb.get_objects(klass) for o in objs: assert not getattr(o, key_field) in preloaded preloaded[getattr(o, key_field)] = o def preload_by_type(self, name, klass, preloaded): self.logger.info('start preloading %s' % name) self.__preload_items__('id', klass, preloaded) self.logger.info('done preloading %s' % name) def preload_studies(self, preloaded): self.logger.info('start preloading studies') self.__preload_items__('label', self.kb.Study, preloaded) self.logger.info('done preloading studies') def missing_fields(self, fields, r): for f in fields: if f not in r: return f return False
kwargs['filename'] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() try: host = args.host or vlu.ome_host() user = args.user or vlu.ome_user() passwd = args.passwd or vlu.ome_passwd() except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logger.info('Loading Demographic data') demogs = kb.get_objects(kb.Demographic) logger.info('Loaded %d items' % len(demogs)) logger.info('Loading Individuals') inds = kb.get_objects(kb.Individual) logger.info('Loaded %d individuals' % len(inds)) demogs_lookup = {} for d in demogs: demogs_lookup[d.individual] = d enrolls_map = get_enrollments_lookup(kb, logger) in_file = open(args.in_file) reader = csv.DictReader(in_file, delimiter='\t') bd_matches_file = open(args.match_files, 'w')
logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logger.info('Start preloading dependency tree') kb.update_dependency_tree() logger.info('Dependency tree loaded') with open(args.out_file, 'w') as ofile: writer = csv.DictWriter( ofile, ['target', 'new_source_type', 'new_source', 'old_source'], delimiter='\t') writer.writeheader() logger.info('Loading wells data') for w in kb.get_objects(kb.PlateWell): if type(w.action.target) == kb.Individual: tubes = list(kb.dt.get_connected(w, kb.Tube)) if len(tubes) == 1: record = { 'target': '%s:%s' % (w.container.label, w.label), 'new_source_type': tubes[0].__class__.__name__, 'new_source': tubes[0].label, 'old_source': '%s:%s' % (w.action.target.__class__.__name__, w.action.target.id) } writer.writerow(record)
logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logger.info('Start preloading dependency tree') kb.update_dependency_tree() logger.info('Dependency tree loaded') with open(args.out_file, 'w') as ofile: writer = csv.DictWriter(ofile, ['target', 'new_source_type', 'new_source', 'old_source'], delimiter='\t') writer.writeheader() logger.info('Loading wells data') for w in kb.get_objects(kb.PlateWell): if type(w.action.target) == kb.Individual: tubes = list(kb.dt.get_connected(w, kb.Tube)) if len(tubes) == 1: record = {'target' : '%s:%s' % (w.container.label, w.label), 'new_source_type' : tubes[0].__class__.__name__, 'new_source' : tubes[0].label, 'old_source' : '%s:%s' % (w.action.target.__class__.__name__, w.action.target.id)} writer.writerow(record) else: logger.debug('Well %s:%s has %d tubes connected' % (w.container.label, w.label, len(tubes))) else: logger.debug('Well %s:%s source is %s' % (w.container.label,
class Core(object): def __init__(self, host=None, user=None, passwd=None, group=None, keep_tokens=1, study_label=None, logger=None): self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens) self.logger = logger if logger else logging.getLogger() self.record_counter = 0 self.default_study = None if study_label: s = self.kb.get_study(study_label) if not s: raise ValueError('No known study with label %s' % study_label) self.logger.info('Selecting %s[%d,%s] as default study' % (s.label, s.omero_id, s.id)) self.default_study = s @classmethod def find_action_setup_conf(klass, args): action_setup_conf = {} for x in dir(args): if not (x.startswith('_') or x.startswith('func')): action_setup_conf[x] = getattr(args, x) # HACKS action_setup_conf['ifile'] = action_setup_conf['ifile'].name action_setup_conf['ofile'] = action_setup_conf['ofile'].name action_setup_conf['report_file'] = action_setup_conf['report_file'].name return action_setup_conf @classmethod def get_action_setup_options(klass, record, action_setup_conf = None, object_history = None): options = {} if 'options' in record and record['options']: kvs = record['options'].split(',') for kv in kvs: k, v = kv.split('=') options[k] = v if action_setup_conf: options['importer_setup'] = action_setup_conf if object_history: options['object_history'] = object_history return json.dumps(options) def get_device(self, label, maker, model, release): device = self.kb.get_device(label) if not device: self.logger.debug('creating a device') device = self.kb.create_device(label, maker, model, release) return device def get_action_setup(self, label, conf): """ Return the ActionSetup corresponding to label if there is one, else create a new one using conf. """ asetup = self.kb.get_action_setup(label) if not asetup: kb_conf = { 'label': label, 'conf': json.dumps(conf), } asetup = self.kb.factory.create(self.kb.ActionSetup, kb_conf).save() return asetup def get_study(self, label): if self.default_study: return self.default_study study = self.kb.get_study(label) if not study: study = self.kb.factory.create(self.kb.Study, {'label': label}).save() return study def find_study(self, records): study_label = records[0]['study'] for r in records: if r['study'] != study_label: m = 'all records should have the same study label' self.logger.critical(m) raise ValueError(m) return self.get_study(study_label) def find_klass(self, col_name, records): o_type = records[0][col_name] for r in records: if r[col_name] != o_type: m = 'all records should have the same %s' % col_name self.logger.critical(m) raise ValueError(m) return getattr(self.kb, o_type) def __preload_items__(self, key_field, klass, preloaded): objs = self.kb.get_objects(klass) for o in objs: assert not getattr(o, key_field) in preloaded preloaded[getattr(o, key_field)] = o def preload_by_type(self, name, klass, preloaded): self.logger.info('start preloading %s' % name) self.__preload_items__('id', klass, preloaded) self.logger.info('done preloading %s' % name) def preload_studies(self, preloaded): self.logger.info('start preloading studies') self.__preload_items__('label', self.kb.Study, preloaded) self.logger.info('done preloading studies') def missing_fields(self, fields, r): for f in fields: if f not in r: return f return False