class TestKB(KBObjectCreator): def __init__(self, name): super(TestKB, self).__init__(name) self.kill_list = [] def setUp(self): logger.info('start setup') self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS) logger.info('done with setup') def tearDown(self): logger.info('start tear-down') self.kill_list.reverse() for x in self.kill_list: self.kb.delete(x) self.kill_list = [] logger.info('done with tear-down') def create_archetype_record(self): terminology = 'terminology://apps.who.int/classifications/apps/' term = terminology + 'gE10.htm#E10' archetype = 'openEHR-EHR-EVALUATION.problem-diagnosis.v1' field = 'at0002.1' rec = {field : term} selector = ('(archetype=="%s")&(field=="%s")&(svalue=="%s")' % (archetype, field, term)) return archetype, rec, selector def test_ehr_record(self): logger.info('creating ActionOnIndividual') conf, action = self.create_action_on_individual() self.kill_list.append(action.save()) logger.info('done creating ActionOnIndividual') archetype, fields, selector = self.create_archetype_record() # FIXME there should be a function for this... timestamp = int(time.time() * 1000) self.kb.add_ehr_record(action, timestamp, archetype, fields) rs = self.kb.get_ehr_records(selector) individual = action.target individual.reload() self.assertTrue(len(rs) > 0) not_found = True for r in rs: for k in ['a_id', 'i_id', 'timestamp', 'archetype', 'fields']: self.assertTrue(k in r) if r['a_id'] == action.id: not_found = False self.assertEqual(r['i_id'], individual.id) self.assertEqual(r['timestamp'], timestamp) self.assertEqual(r['archetype'], archetype) nfields = r['fields'] self.assertTrue(len(nfields) >= len(fields)) for k in fields: self.assertTrue(k in nfields) self.assertEqual(nfields[k], fields[k]) break else: self.assertTrue(not_found)
class TestKB(KBObjectCreator): def __init__(self, name): super(TestKB, self).__init__(name) self.kill_list = [] def setUp(self): logger.info('start setup') self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS) logger.info('done with setup') def tearDown(self): logger.info('start tear-down') self.kill_list.reverse() for x in self.kill_list: self.kb.delete(x) self.kill_list = [] logger.info('done with tear-down') def create_archetype_record(self): terminology = 'terminology://apps.who.int/classifications/apps/' term = terminology + 'gE10.htm#E10' archetype = 'openEHR-EHR-EVALUATION.problem-diagnosis.v1' field = 'at0002.1' rec = {field: term} selector = ('(archetype=="%s")&(field=="%s")&(svalue=="%s")' % (archetype, field, term)) return archetype, rec, selector def test_ehr_record(self): logger.info('creating ActionOnIndividual') conf, action = self.create_action_on_individual() self.kill_list.append(action.save()) logger.info('done creating ActionOnIndividual') archetype, fields, selector = self.create_archetype_record() # FIXME there should be a function for this... timestamp = int(time.time() * 1000) self.kb.add_ehr_record(action, timestamp, archetype, fields) rs = self.kb.get_ehr_records(selector) individual = action.target individual.reload() self.assertTrue(len(rs) > 0) not_found = True for r in rs: for k in ['a_id', 'i_id', 'timestamp', 'archetype', 'fields']: self.assertTrue(k in r) if r['a_id'] == action.id: not_found = False self.assertEqual(r['i_id'], individual.id) self.assertEqual(r['timestamp'], timestamp) self.assertEqual(r['archetype'], archetype) nfields = r['fields'] self.assertTrue(len(nfields) >= len(fields)) for k in fields: self.assertTrue(k in nfields) self.assertEqual(nfields[k], fields[k]) break else: self.assertTrue(not_found)
def main(argv): parser = make_parser() args = parser.parse_args(argv) log_level = getattr(logging, args.loglevel) kwargs = {"format": LOG_FORMAT, "datefmt": LOG_DATEFMT, "level": log_level} if args.logfile: kwargs["filename"] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() kb = KB(driver="omero")(args.host, args.user, args.passwd) # Load enrollments and individual (needed to build sample label and for gender field) enrolls = [] for sl in STUDY_LABELS: logger.debug("Loading enrollments for study %s" % sl) enrolls.extend(kb.get_enrolled(kb.get_study(sl))) logger.debug("Fetched %d individuals" % len(enrolls)) wells_lookup = get_wells_enrolls_lookup(enrolls, kb) logger.debug("Loading EHR records") ehr_records = kb.get_ehr_records("(valid == True)") ehr_records_map = {} for r in ehr_records: ehr_records_map.setdefault(r["i_id"], []).append(r) # Read plate barcodes with open(args.plates_list) as pl_list: barcodes = [row.strip() for row in pl_list] # Load plate for plate_barcode in barcodes: logger.info("Creating datasheet for plate %s" % plate_barcode) pl = load_plate(plate_barcode, kb) if not pl: logger.error("No plate with barcode %s exists, skipping it." % (plate_barcode)) continue # Load wells for selected plate pl_wells = get_wells_by_plate(plate_barcode, kb) with open(os.path.join(args.out_dir, "%s_datasheet.csv" % plate_barcode), "w") as of: writer = csv.DictWriter(of, CSV_FIELDS, delimiter="\t") writer.writeheader() last_slot = 0 for slot, well in sorted(pl_wells.iteritems()): cl_record = ehr_records_map[wells_lookup[well][0].individual.id] t1d, ms = get_affections(cl_record) # Fill empty slots while last_slot != slot - 1: last_slot += 1 writer.writerow( { "Sample_ID": "X", "PLATE_barcode": pl.barcode, "PLATE_name": pl.label, "WELL_label": get_well_label(last_slot), "INDIVIDUAL_gender": "X", "INDIVIDUAL_vid": "X", "T1D_affected": "X", "MS_affected": "X", } ) writer.writerow( { "Sample_ID": get_ichip_sample_code(wells_lookup[well], pl.barcode), "PLATE_barcode": pl.barcode, "PLATE_name": pl.label, "WELL_label": well.label, "INDIVIDUAL_gender": map_gender(wells_lookup[well][0].individual), "INDIVIDUAL_vid": wells_lookup[well][0].individual.id, "T1D_affected": t1d, "MS_affected": ms, } ) last_slot = slot
kwargs['filename'] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() try: host = args.host or vlu.ome_host() user = args.user or vlu.ome_user() passwd = args.passwd or vlu.ome_passwd() except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logger.info('Retrieving diagnosis records') ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' % DIAGNOSIS_ARCHETYPE) ehr_records.extend( kb.get_ehr_records('(archetype == "%s") & (valid == True)' % EXCL_DIAG_ARCHETYPE)) logger.info('%d records retrieved' % len(ehr_records)) logger.info('Loading actions') actions = kb.get_objects(kb.Action) act_map = {} for act in actions: act_map[act.id] = act.context.label logger.info('%d actions loaded' % len(act_map)) with open(args.ofile, 'w') as out_file: writer = csv.DictWriter( out_file, ['study', 'individual', 'timestamp', 'diagnosis'],
kwargs['filename'] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() try: host = args.host or vlu.ome_host() user = args.user or vlu.ome_user() passwd = args.passwd or vlu.ome_passwd() except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logger.info('Retrieving diagnosis records') ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' % DIAGNOSIS_ARCHETYPE) ehr_records.extend(kb.get_ehr_records('(archetype == "%s") & (valid == True)' % EXCL_DIAG_ARCHETYPE)) logger.info('%d records retrieved' % len(ehr_records)) logger.info('Loading actions') actions = kb.get_objects(kb.Action) act_map = {} for act in actions: act_map[act.id] = act.context.label logger.info('%d actions loaded' % len(act_map)) with open(args.ofile, 'w') as out_file: writer = csv.DictWriter(out_file, ['study', 'individual', 'timestamp', 'diagnosis'], delimiter='\t')
kwargs['filename'] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() try: host = args.host or vlu.ome_host() user = args.user or vlu.ome_user() passwd = args.passwd or vlu.ome_passwd() except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logging.info('Loading EHR records') ehr_records = kb.get_ehr_records() logging.info('Loaded %d EHR records' % len(ehr_records)) ehr_lookup = {} for rec in ehr_records: key = get_ehr_key(rec) ehr_lookup.setdefault(rec['i_id'], {}).setdefault(key, []).append(rec) logger.debug('Loaded EHR lookup table, %d individuals involved' % len(ehr_lookup.keys())) for ind, records in ehr_lookup.iteritems(): for k, vals in records.iteritems(): if len(vals) > 1: logger.info( '##### Individual %s has %d duplicated EHR records #####' % (ind, len(vals)))
def main(argv): parser = make_parser() args = parser.parse_args(argv) log_level = getattr(logging, args.loglevel) kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level} if args.logfile: kwargs['filename'] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() kb = KB(driver='omero')(args.host, args.user, args.passwd) # Load enrollments and individual (needed to build sample label and for gender field) enrolls = [] for sl in STUDY_LABELS: logger.debug('Loading enrollments for study %s' % sl) enrolls.extend(kb.get_enrolled(kb.get_study(sl))) logger.debug('Fetched %d individuals' % len(enrolls)) wells_lookup = get_wells_enrolls_lookup(enrolls, kb) logger.debug('Loading EHR records') ehr_records = kb.get_ehr_records('(valid == True)') ehr_records_map = {} for r in ehr_records: ehr_records_map.setdefault(r['i_id'], []).append(r) # Read plate barcodes with open(args.plates_list) as pl_list: barcodes = [row.strip() for row in pl_list] # Load plate for plate_barcode in barcodes: logger.info('Creating datasheet for plate %s' % plate_barcode) pl = load_plate(plate_barcode, kb) if not pl: logger.error('No plate with barcode %s exists, skipping it.' % (plate_barcode)) continue # Load wells for selected plate pl_wells = get_wells_by_plate(plate_barcode, kb) with open( os.path.join(args.out_dir, '%s_datasheet.csv' % plate_barcode), 'w') as of: writer = csv.DictWriter(of, CSV_FIELDS, delimiter='\t') writer.writeheader() last_slot = 0 for slot, well in sorted(pl_wells.iteritems()): cl_record = ehr_records_map[wells_lookup[well] [0].individual.id] t1d, ms = get_affections(cl_record) # Fill empty slots while (last_slot != slot - 1): last_slot += 1 writer.writerow({ 'Sample_ID': 'X', 'PLATE_barcode': pl.barcode, 'PLATE_name': pl.label, 'WELL_label': get_well_label(last_slot), 'INDIVIDUAL_gender': 'X', 'INDIVIDUAL_vid': 'X', 'T1D_affected': 'X', 'MS_affected': 'X' }) writer.writerow({ 'Sample_ID': get_ichip_sample_code(wells_lookup[well], pl.barcode), 'PLATE_barcode': pl.barcode, 'PLATE_name': pl.label, 'WELL_label': well.label, 'INDIVIDUAL_gender': map_gender(wells_lookup[well][0].individual), 'INDIVIDUAL_vid': wells_lookup[well][0].individual.id, 'T1D_affected': t1d, 'MS_affected': ms }) last_slot = slot
kwargs['filename'] = args.logfile logging.basicConfig(**kwargs) logger = logging.getLogger() try: host = args.host or vlu.ome_host() user = args.user or vlu.ome_user() passwd = args.passwd or vlu.ome_passwd() except ValueError, ve: logger.critical(ve) sys.exit(ve) kb = KB(driver='omero')(host, user, passwd) logging.info('Loading EHR records') ehr_records = kb.get_ehr_records() logging.info('Loaded %d EHR records' % len(ehr_records)) ehr_lookup = {} for rec in ehr_records: key = get_ehr_key(rec) ehr_lookup.setdefault(rec['i_id'], {}).setdefault(key,[]).append(rec) logger.debug('Loaded EHR lookup table, %d individuals involved' % len(ehr_lookup.keys())) for ind, records in ehr_lookup.iteritems(): for k, vals in records.iteritems(): if len(vals) > 1: logger.info('##### Individual %s has %d duplicated EHR records #####' % (ind, len(vals))) logger.debug(vals)