class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
logger.info('start setup')
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
logger.info('done with setup')
def tearDown(self):
logger.info('start tear-down')
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
logger.info('done with tear-down')
def create_archetype_record(self):
terminology = 'terminology://apps.who.int/classifications/apps/'
term = terminology + 'gE10.htm#E10'
archetype = 'openEHR-EHR-EVALUATION.problem-diagnosis.v1'
field = 'at0002.1'
rec = {field : term}
selector = ('(archetype=="%s")&(field=="%s")&(svalue=="%s")'
% (archetype, field, term))
return archetype, rec, selector
def test_ehr_record(self):
logger.info('creating ActionOnIndividual')
conf, action = self.create_action_on_individual()
self.kill_list.append(action.save())
logger.info('done creating ActionOnIndividual')
archetype, fields, selector = self.create_archetype_record()
# FIXME there should be a function for this...
timestamp = int(time.time() * 1000)
self.kb.add_ehr_record(action, timestamp, archetype, fields)
rs = self.kb.get_ehr_records(selector)
individual = action.target
individual.reload()
self.assertTrue(len(rs) > 0)
not_found = True
for r in rs:
for k in ['a_id', 'i_id', 'timestamp', 'archetype', 'fields']:
self.assertTrue(k in r)
if r['a_id'] == action.id:
not_found = False
self.assertEqual(r['i_id'], individual.id)
self.assertEqual(r['timestamp'], timestamp)
self.assertEqual(r['archetype'], archetype)
nfields = r['fields']
self.assertTrue(len(nfields) >= len(fields))
for k in fields:
self.assertTrue(k in nfields)
self.assertEqual(nfields[k], fields[k])
break
else:
self.assertTrue(not_found)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
logger.info('start setup')
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
logger.info('done with setup')
def tearDown(self):
logger.info('start tear-down')
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
logger.info('done with tear-down')
def create_archetype_record(self):
terminology = 'terminology://apps.who.int/classifications/apps/'
term = terminology + 'gE10.htm#E10'
archetype = 'openEHR-EHR-EVALUATION.problem-diagnosis.v1'
field = 'at0002.1'
rec = {field: term}
selector = ('(archetype=="%s")&(field=="%s")&(svalue=="%s")' %
(archetype, field, term))
return archetype, rec, selector
def test_ehr_record(self):
logger.info('creating ActionOnIndividual')
conf, action = self.create_action_on_individual()
self.kill_list.append(action.save())
logger.info('done creating ActionOnIndividual')
archetype, fields, selector = self.create_archetype_record()
# FIXME there should be a function for this...
timestamp = int(time.time() * 1000)
self.kb.add_ehr_record(action, timestamp, archetype, fields)
rs = self.kb.get_ehr_records(selector)
individual = action.target
individual.reload()
self.assertTrue(len(rs) > 0)
not_found = True
for r in rs:
for k in ['a_id', 'i_id', 'timestamp', 'archetype', 'fields']:
self.assertTrue(k in r)
if r['a_id'] == action.id:
not_found = False
self.assertEqual(r['i_id'], individual.id)
self.assertEqual(r['timestamp'], timestamp)
self.assertEqual(r['archetype'], archetype)
nfields = r['fields']
self.assertTrue(len(nfields) >= len(fields))
for k in fields:
self.assertTrue(k in nfields)
self.assertEqual(nfields[k], fields[k])
break
else:
self.assertTrue(not_found)
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
log_level = getattr(logging, args.loglevel)
kwargs = {"format": LOG_FORMAT, "datefmt": LOG_DATEFMT, "level": log_level}
if args.logfile:
kwargs["filename"] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
kb = KB(driver="omero")(args.host, args.user, args.passwd)
# Load enrollments and individual (needed to build sample label and for gender field)
enrolls = []
for sl in STUDY_LABELS:
logger.debug("Loading enrollments for study %s" % sl)
enrolls.extend(kb.get_enrolled(kb.get_study(sl)))
logger.debug("Fetched %d individuals" % len(enrolls))
wells_lookup = get_wells_enrolls_lookup(enrolls, kb)
logger.debug("Loading EHR records")
ehr_records = kb.get_ehr_records("(valid == True)")
ehr_records_map = {}
for r in ehr_records:
ehr_records_map.setdefault(r["i_id"], []).append(r)
# Read plate barcodes
with open(args.plates_list) as pl_list:
barcodes = [row.strip() for row in pl_list]
# Load plate
for plate_barcode in barcodes:
logger.info("Creating datasheet for plate %s" % plate_barcode)
pl = load_plate(plate_barcode, kb)
if not pl:
logger.error("No plate with barcode %s exists, skipping it." % (plate_barcode))
continue
# Load wells for selected plate
pl_wells = get_wells_by_plate(plate_barcode, kb)
with open(os.path.join(args.out_dir, "%s_datasheet.csv" % plate_barcode), "w") as of:
writer = csv.DictWriter(of, CSV_FIELDS, delimiter="\t")
writer.writeheader()
last_slot = 0
for slot, well in sorted(pl_wells.iteritems()):
cl_record = ehr_records_map[wells_lookup[well][0].individual.id]
t1d, ms = get_affections(cl_record)
# Fill empty slots
while last_slot != slot - 1:
last_slot += 1
writer.writerow(
{
"Sample_ID": "X",
"PLATE_barcode": pl.barcode,
"PLATE_name": pl.label,
"WELL_label": get_well_label(last_slot),
"INDIVIDUAL_gender": "X",
"INDIVIDUAL_vid": "X",
"T1D_affected": "X",
"MS_affected": "X",
}
)
writer.writerow(
{
"Sample_ID": get_ichip_sample_code(wells_lookup[well], pl.barcode),
"PLATE_barcode": pl.barcode,
"PLATE_name": pl.label,
"WELL_label": well.label,
"INDIVIDUAL_gender": map_gender(wells_lookup[well][0].individual),
"INDIVIDUAL_vid": wells_lookup[well][0].individual.id,
"T1D_affected": t1d,
"MS_affected": ms,
}
)
last_slot = slot
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Retrieving diagnosis records')
ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
DIAGNOSIS_ARCHETYPE)
ehr_records.extend(
kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
EXCL_DIAG_ARCHETYPE))
logger.info('%d records retrieved' % len(ehr_records))
logger.info('Loading actions')
actions = kb.get_objects(kb.Action)
act_map = {}
for act in actions:
act_map[act.id] = act.context.label
logger.info('%d actions loaded' % len(act_map))
with open(args.ofile, 'w') as out_file:
writer = csv.DictWriter(
out_file, ['study', 'individual', 'timestamp', 'diagnosis'],
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Retrieving diagnosis records')
ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
DIAGNOSIS_ARCHETYPE)
ehr_records.extend(kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
EXCL_DIAG_ARCHETYPE))
logger.info('%d records retrieved' % len(ehr_records))
logger.info('Loading actions')
actions = kb.get_objects(kb.Action)
act_map = {}
for act in actions:
act_map[act.id] = act.context.label
logger.info('%d actions loaded' % len(act_map))
with open(args.ofile, 'w') as out_file:
writer = csv.DictWriter(out_file, ['study', 'individual',
'timestamp', 'diagnosis'],
delimiter='\t')
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logging.info('Loading EHR records')
ehr_records = kb.get_ehr_records()
logging.info('Loaded %d EHR records' % len(ehr_records))
ehr_lookup = {}
for rec in ehr_records:
key = get_ehr_key(rec)
ehr_lookup.setdefault(rec['i_id'], {}).setdefault(key, []).append(rec)
logger.debug('Loaded EHR lookup table, %d individuals involved' %
len(ehr_lookup.keys()))
for ind, records in ehr_lookup.iteritems():
for k, vals in records.iteritems():
if len(vals) > 1:
logger.info(
'##### Individual %s has %d duplicated EHR records #####' %
(ind, len(vals)))
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
kb = KB(driver='omero')(args.host, args.user, args.passwd)
# Load enrollments and individual (needed to build sample label and for gender field)
enrolls = []
for sl in STUDY_LABELS:
logger.debug('Loading enrollments for study %s' % sl)
enrolls.extend(kb.get_enrolled(kb.get_study(sl)))
logger.debug('Fetched %d individuals' % len(enrolls))
wells_lookup = get_wells_enrolls_lookup(enrolls, kb)
logger.debug('Loading EHR records')
ehr_records = kb.get_ehr_records('(valid == True)')
ehr_records_map = {}
for r in ehr_records:
ehr_records_map.setdefault(r['i_id'], []).append(r)
# Read plate barcodes
with open(args.plates_list) as pl_list:
barcodes = [row.strip() for row in pl_list]
# Load plate
for plate_barcode in barcodes:
logger.info('Creating datasheet for plate %s' % plate_barcode)
pl = load_plate(plate_barcode, kb)
if not pl:
logger.error('No plate with barcode %s exists, skipping it.' %
(plate_barcode))
continue
# Load wells for selected plate
pl_wells = get_wells_by_plate(plate_barcode, kb)
with open(
os.path.join(args.out_dir, '%s_datasheet.csv' % plate_barcode),
'w') as of:
writer = csv.DictWriter(of, CSV_FIELDS, delimiter='\t')
writer.writeheader()
last_slot = 0
for slot, well in sorted(pl_wells.iteritems()):
cl_record = ehr_records_map[wells_lookup[well]
[0].individual.id]
t1d, ms = get_affections(cl_record)
# Fill empty slots
while (last_slot != slot - 1):
last_slot += 1
writer.writerow({
'Sample_ID': 'X',
'PLATE_barcode': pl.barcode,
'PLATE_name': pl.label,
'WELL_label': get_well_label(last_slot),
'INDIVIDUAL_gender': 'X',
'INDIVIDUAL_vid': 'X',
'T1D_affected': 'X',
'MS_affected': 'X'
})
writer.writerow({
'Sample_ID':
get_ichip_sample_code(wells_lookup[well], pl.barcode),
'PLATE_barcode':
pl.barcode,
'PLATE_name':
pl.label,
'WELL_label':
well.label,
'INDIVIDUAL_gender':
map_gender(wells_lookup[well][0].individual),
'INDIVIDUAL_vid':
wells_lookup[well][0].individual.id,
'T1D_affected':
t1d,
'MS_affected':
ms
})
last_slot = slot
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logging.info('Loading EHR records')
ehr_records = kb.get_ehr_records()
logging.info('Loaded %d EHR records' % len(ehr_records))
ehr_lookup = {}
for rec in ehr_records:
key = get_ehr_key(rec)
ehr_lookup.setdefault(rec['i_id'], {}).setdefault(key,[]).append(rec)
logger.debug('Loaded EHR lookup table, %d individuals involved' %
len(ehr_lookup.keys()))
for ind, records in ehr_lookup.iteritems():
for k, vals in records.iteritems():
if len(vals) > 1:
logger.info('##### Individual %s has %d duplicated EHR records #####' % (ind,
len(vals)))
logger.debug(vals)
