class ped_writer(unittest.TestCase):
def setUp(self):
self.kb = KB(driver='omero')('localhost', 'root', 'romeo')
def tearDown(self):
pass
def test_base(self):
def extract_data_sample(group, mset, dsample_name):
by_individual = {}
for i in self.kb.get_individuals(group):
gds = filter(lambda x: x.snpMarkersSet == mset,
self.kb.get_data_samples(i, dsample_name))
assert (len(gds) == 1)
by_individual[i.id] = gds[0]
return by_individual
study = self.kb.get_study('TEST01')
family = self.kb.get_individuals(study)
mset = self.kb.get_snp_markers_set(label='FakeTaqSet01')
gds_by_individual = extract_data_sample(study, mset,
'GenotypeDataSample')
pw = PedWriter(mset, base_path="./foo")
pw.write_map()
pw.write_family(study.id, family, gds_by_individual)
pw.close()
class ped_writer(unittest.TestCase):
def setUp(self):
self.kb = KB(driver="omero")("localhost", "root", "romeo")
def tearDown(self):
pass
def test_base(self):
def extract_data_sample(group, mset, dsample_name):
by_individual = {}
for i in self.kb.get_individuals(group):
gds = filter(lambda x: x.snpMarkersSet == mset, self.kb.get_data_samples(i, dsample_name))
assert len(gds) == 1
by_individual[i.id] = gds[0]
return by_individual
study = self.kb.get_study("TEST01")
family = self.kb.get_individuals(study)
mset = self.kb.get_snp_markers_set(label="FakeTaqSet01")
gds_by_individual = extract_data_sample(study, mset, "GenotypeDataSample")
pw = PedWriter(mset, base_path="./foo")
pw.write_map()
pw.write_family(study.id, family, gds_by_individual)
pw.close()
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
logger.info('start setup')
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
logger.info('done with setup')
def tearDown(self):
logger.info('start tear-down')
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
logger.info('done with tear-down')
def create_archetype_record(self):
terminology = 'terminology://apps.who.int/classifications/apps/'
term = terminology + 'gE10.htm#E10'
archetype = 'openEHR-EHR-EVALUATION.problem-diagnosis.v1'
field = 'at0002.1'
rec = {field : term}
selector = ('(archetype=="%s")&(field=="%s")&(svalue=="%s")'
% (archetype, field, term))
return archetype, rec, selector
def test_ehr_record(self):
logger.info('creating ActionOnIndividual')
conf, action = self.create_action_on_individual()
self.kill_list.append(action.save())
logger.info('done creating ActionOnIndividual')
archetype, fields, selector = self.create_archetype_record()
# FIXME there should be a function for this...
timestamp = int(time.time() * 1000)
self.kb.add_ehr_record(action, timestamp, archetype, fields)
rs = self.kb.get_ehr_records(selector)
individual = action.target
individual.reload()
self.assertTrue(len(rs) > 0)
not_found = True
for r in rs:
for k in ['a_id', 'i_id', 'timestamp', 'archetype', 'fields']:
self.assertTrue(k in r)
if r['a_id'] == action.id:
not_found = False
self.assertEqual(r['i_id'], individual.id)
self.assertEqual(r['timestamp'], timestamp)
self.assertEqual(r['archetype'], archetype)
nfields = r['fields']
self.assertTrue(len(nfields) >= len(fields))
for k in fields:
self.assertTrue(k in nfields)
self.assertEqual(nfields[k], fields[k])
break
else:
self.assertTrue(not_found)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
logger.info('start setup')
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
logger.info('done with setup')
def tearDown(self):
logger.info('start tear-down')
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
logger.info('done with tear-down')
def create_archetype_record(self):
terminology = 'terminology://apps.who.int/classifications/apps/'
term = terminology + 'gE10.htm#E10'
archetype = 'openEHR-EHR-EVALUATION.problem-diagnosis.v1'
field = 'at0002.1'
rec = {field: term}
selector = ('(archetype=="%s")&(field=="%s")&(svalue=="%s")' %
(archetype, field, term))
return archetype, rec, selector
def test_ehr_record(self):
logger.info('creating ActionOnIndividual')
conf, action = self.create_action_on_individual()
self.kill_list.append(action.save())
logger.info('done creating ActionOnIndividual')
archetype, fields, selector = self.create_archetype_record()
# FIXME there should be a function for this...
timestamp = int(time.time() * 1000)
self.kb.add_ehr_record(action, timestamp, archetype, fields)
rs = self.kb.get_ehr_records(selector)
individual = action.target
individual.reload()
self.assertTrue(len(rs) > 0)
not_found = True
for r in rs:
for k in ['a_id', 'i_id', 'timestamp', 'archetype', 'fields']:
self.assertTrue(k in r)
if r['a_id'] == action.id:
not_found = False
self.assertEqual(r['i_id'], individual.id)
self.assertEqual(r['timestamp'], timestamp)
self.assertEqual(r['archetype'], archetype)
nfields = r['fields']
self.assertTrue(len(nfields) >= len(fields))
for k in fields:
self.assertTrue(k in nfields)
self.assertEqual(nfields[k], fields[k])
break
else:
self.assertTrue(not_found)
def main(argv):
global logger
parser = make_parser()
args = parser.parse_args(argv)
logformat = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
loglevel = getattr(logging, args.loglevel)
if args.logfile:
logging.basicConfig(filename=args.logfile, format=logformat, level=loglevel)
else:
logging.basicConfig(format=logformat, level=loglevel)
logger = logging.getLogger()
abi_service = ABISnpService()
found_markers = {}
data = {}
min_datetime = datetime.max #datetime(2999, 12, 31, 23, 59, 59)
max_datetime = datetime.min #datetime(1000, 1, 1, 0, 0, 1)
for f in (_.strip() for _ in args.ifile if _.strip()):
logger.info('processing %s' % f)
sds = SDSReader(open(f), swap_sample_well_columns=True)
min_datetime = min(min_datetime, sds.datetime)
max_datetime = max(max_datetime, sds.datetime)
get_markers_definition(found_markers, sds, abi_service)
for r in sds:
data.setdefault(r['Sample Name'], []).append(r)
kb = KB(driver='omero')(args.host, args.user, args.passwd)
logger.info('qui - main')
missing_kb_markers = add_kb_marker_objects(kb, found_markers)
if missing_kb_markers:
logger.info('there are missing markers. Cannot proceed further.')
fname = '%smarker-defs.tsv' % args.prefix
write_import_markers_file(fname, found_markers, missing_kb_markers)
logger.info('the list of the missing marker is in %s.' % fname)
sys.exit(0)
fname = '%smarkers-set-def.tsv' % args.prefix
write_markers_set_def_file(fname, found_markers)
ssc_data_set = {}
device = kb.get_device(args.device_label)
for sample_id, d in data.iteritems():
fname = '%s%s-%s.ssc' % (args.prefix, device.id, sample_id)
write_ssc_data_set_file(fname, found_markers,
device.id, sample_id,
min_datetime, max_datetime, d)
ssc_data_set[sample_id] = ('taqman-%s-%s' % (args.run_id, sample_id),
sample_id, device.id, fname)
fname = '%simport.ssc' % args.prefix
write_ssc_data_samples_import_file(fname, ssc_data_set.values())
write_ssc_data_objects_import_file(fname, ssc_data_set.values())
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_container(self):
conf, c = self.create_container()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.Container)
def test_slotted_container(self):
conf, c = self.create_slotted_container()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.SlottedContainer)
def test_titer_plate(self):
conf, c = self.create_titer_plate()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.TiterPlate)
def test_data_collection(self):
conf, c = self.create_data_collection()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.DataCollection)
def test_data_collection_item(self):
conf, c = self.create_data_collection_item()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.DataCollectionItem)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_data_sample(self):
conf, ds = self.create_data_sample()
self.kill_list.append(ds.save())
self.check_object(ds, conf, self.kb.DataSample)
def test_affymetrix_cel(self):
conf, ds = self.create_affymetrix_cel()
self.kill_list.append(ds.save())
self.check_object(ds, conf, self.kb.AffymetrixCel)
def test_snp_markers_set(self):
conf, sms = self.create_snp_markers_set()
self.kill_list.append(sms.save())
self.check_object(sms, conf, self.kb.SNPMarkersSet)
def test_genotype_data_sample(self):
conf, gds = self.create_genotype_data_sample()
self.kill_list.append(gds.save())
self.check_object(gds, conf, self.kb.GenotypeDataSample)
def test_data_object(self):
conf, do = self.create_data_object()
self.kill_list.append(do.save())
self.check_object(do, conf, self.kb.DataObject)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_container(self):
conf, c = self.create_container()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.Container)
def test_slotted_container(self):
conf, c = self.create_slotted_container()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.SlottedContainer)
def test_titer_plate(self):
conf, c = self.create_titer_plate()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.TiterPlate)
def test_data_collection(self):
conf, c = self.create_data_collection()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.DataCollection)
def test_data_collection_item(self):
conf, c = self.create_data_collection_item()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.DataCollectionItem)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver="omero")(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, "ome_obj"):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, "_id"):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_data_sample(self):
conf, ds = self.create_data_sample()
self.kill_list.append(ds.save())
self.check_object(ds, conf, self.kb.DataSample)
def test_affymetrix_cel(self):
conf, ds = self.create_affymetrix_cel()
self.kill_list.append(ds.save())
self.check_object(ds, conf, self.kb.AffymetrixCel)
def test_snp_markers_set(self):
conf, sms = self.create_snp_markers_set()
self.kill_list.append(sms.save())
self.check_object(sms, conf, self.kb.SNPMarkersSet)
def test_genotype_data_sample(self):
conf, gds = self.create_genotype_data_sample()
self.kill_list.append(gds.save())
self.check_object(gds, conf, self.kb.GenotypeDataSample)
def test_data_object(self):
conf, do = self.create_data_object()
self.kill_list.append(do.save())
self.check_object(do, conf, self.kb.DataObject)
def __init__(self, host, user, passwd, study_label, maker, model, release):
self.kb = KB(driver='omero')(host, user, passwd)
self.mset = self.kb.get_snp_markers_set(maker, model, release)
self.logger = logging.getLogger()
if not self.mset:
raise ValueError('SNPMarkersSet[%s,%s,%s] has not been defined.' %
(maker, model, release))
#--
alabel = 'load_genotypes-setup-%s' % time.time()
self.asetup = self.kb.factory.create(self.kb.ActionSetup, {
'label': alabel,
'conf': ''
}).save()
#--
dmaker, dmodel, drelease = 'CRS4', 'load_genotypes', '0.1'
dlabel = '%s-%s-%s' % (dmaker, dmodel, drelease)
device = self.kb.get_device(dlabel)
if not device:
device = self.kb.factory.create(
self.kb.Device, {
'label': dlabel,
'maker': dmaker,
'model': dmodel,
'release': drelease
}).save()
self.device = device
#-- FIXME this will break if study is not defined.
self.study = self.kb.get_study(study_label)
def __init__(self, host, user, passwd,
study_label,
maker, model, release):
self.kb = KB(driver='omero')(host, user, passwd)
self.mset = self.kb.get_snp_markers_set(maker, model, release)
self.logger = logging.getLogger()
if not self.mset:
raise ValueError('SNPMarkersSet[%s,%s,%s] has not been defined.'
% (maker, model, release))
#--
alabel = 'load_genotypes-setup-%s' % time.time()
self.asetup = self.kb.factory.create(self.kb.ActionSetup,
{'label' : alabel,
'conf' : ''}).save()
#--
dmaker, dmodel, drelease = 'CRS4', 'load_genotypes', '0.1'
dlabel = '%s-%s-%s' % (dmaker, dmodel, drelease)
device = self.kb.get_device(dlabel)
if not device:
device = self.kb.factory.create(self.kb.Device,
{'label' : dlabel,
'maker' : dmaker,
'model' : dmodel,
'release' : drelease}).save()
self.device = device
#-- FIXME this will break if study is not defined.
self.study = self.kb.get_study(study_label)
class VidMapper(object):
def __init__(self, host, user, passwd, logger, study):
self.logger = logger
self.study = study
self.kb = KB(driver="omero")(host, user, passwd)
plates = self.kb.get_objects(self.kb.TiterPlate)
self.logger.info("fetched %d plates" % len(plates))
self.plate_map = {}
self.enroll_map = {}
for p in plates:
self.plate_map[p.omero_id] = p.barcode
s = self.kb.get_study(self.study)
enrolls = self.kb.get_enrolled(s)
self.logger.info("fetched %d enrollments" % len(enrolls))
for e in enrolls:
self.logger.debug('Retrieving wells for %s' % e.studyCode)
wells = [
w for w in self.kb.get_vessels_by_individual(
e.individual, "PlateWell")
]
self.enroll_map[e.studyCode] = wells
def map_vid(self, r):
en_code = r["source"]
pl_barcode = en_code.split("|")[1]
try:
wells = self.enroll_map[en_code]
except KeyError:
msg = "%s is not enrolled in %s" % (en_code, self.study)
self.logger.error(msg)
raise ValueError(msg)
self.logger.info("found %d wells for %s" % (len(wells), en_code))
imm_wells = [
w for w in wells
if self.plate_map[w.container.omero_id] == pl_barcode
]
if len(imm_wells) > 1:
msg = ("more than 1 (%d) immuno wells for plate %s" %
(len(imm_wells), pl_barcode))
self.logger.error(msg)
raise ValueError(msg)
elif len(imm_wells) == 0:
msg = "no immuno well for plate %s" % pl_barcode
self.logger.warn(msg)
raise ValueError(msg)
else:
r["source"] = imm_wells[0].id
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_parallel_save(self):
aconf, action = self.create_action()
self.kill_list.append(action.save())
N = 1000
people = []
for i in range(N):
conf, i = self.create_individual(action=action,
gender=self.kb.Gender.MALE)
self.kill_list.append(i)
people.append(i)
start = time.time()
self.kb.save_array(people)
print ' \n\ttime needed to save %s object: %s' % (N,
time.time() - start)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_parallel_save(self):
aconf, action = self.create_action()
self.kill_list.append(action.save())
N = 1000
people = []
for i in range(N):
conf, i = self.create_individual(action=action,
gender=self.kb.Gender.MALE)
self.kill_list.append(i)
people.append(i)
start = time.time()
self.kb.save_array(people)
print' \n\ttime needed to save %s object: %s' % (N, time.time() - start)
class App(object):
def __init__(self, host, user, passwd):
self.kb = KB(driver='omero')(host, user, passwd)
#FIXME we need to do this to sync with the DB idea of the enums.
self.kb.Gender.map_enums_values(self.kb)
self.logger = logging.getLogger()
def get_ehr_iterator(self):
inds = self.kb.get_objects(self.kb.Individual)
inds_by_vid = dict([(i.id, i) for i in inds])
for e in self.kb.get_ehr_iterator():
if not e[0] in inds_by_vid:
#FIXME we need to do this for potential stray records left by testing
continue
yield (inds_by_vid[e[0]], e[1])
def count_males(self, inds):
return sum([i.gender == self.kb.Gender.MALE for i in inds])
def do_enrollment(self):
controls = []
affected = []
# FIXME in a future version, we will be able to issue actual AQL
# queries.
for i, ehr in self.get_ehr_iterator():
if ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1):
affected.append(i)
elif ehr.matches(EXCLUSION): # FIXME we assume that this is enough
controls.append(i)
# A more sophisticated example
# will keep only records where age of onset is between 10y and 15y
# for i, ehr in self.get_ehr_iterator():
# if (ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1)
# and
# ehr.matches(DIAGNOSIS, DIAGNOSIS_AGE_OF_ONSET, ("10y", "15y")):
# affected.append(i)
print ('there are %d controls [%d male]'
% (len(controls), self.count_males(controls)))
print ('there are %d affected [%d male] with E10'
% (len(affected), self.count_males(affected)))
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASSWD)
conf = {
'label': 'TEST-%f' % time.time(),
'description': 'unit test garbage',
}
self.study = self.kb.factory.create(self.kb.Study, conf).save()
self.kill_list.append(self.study)
self.action = self.kb.create_an_action(self.study)
self.kill_list.append(self.action)
def __init__(self, **kwargs):
omero_host = vlu.ome_host()
omero_user = vlu.ome_user()
omero_passwd = vlu.ome_passwd()
self.kb = KB(driver='omero')(omero_host, omero_user, omero_passwd)
super(UniverseApplication, self).__init__(**kwargs)
self.config.omero_default_host = kwargs.get('omero_default_host')
self.config.omero_default_user = kwargs.get('omero_default_user')
self.config.omero_default_passwd = kwargs.get('omero_default_passwd')
self.config.vl_loglevel = kwargs.get('vl_loglevel', 'INFO')
self.config.vl_import_enabled_users = kwargs.get('vl_import_enabled_users')
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_vessel(self):
conf, v = self.create_vessel()
self.kill_list.append(v.save())
self.check_object(v, conf, self.kb.Vessel)
def test_tube(self):
conf, v = self.create_tube()
self.kill_list.append(v.save())
self.check_object(v, conf, self.kb.Tube)
class App(object):
def __init__(self, host, user, passwd):
self.kb = KB(driver="omero")(host, user, passwd)
# FIXME we need to do this to sync with the DB idea of the enums.
self.kb.Gender.map_enums_values(self.kb)
self.logger = logging.getLogger()
def get_ehr_iterator(self):
inds = self.kb.get_objects(self.kb.Individual)
inds_by_vid = dict([(i.id, i) for i in inds])
for e in self.kb.get_ehr_iterator():
if not e[0] in inds_by_vid:
# FIXME we need to do this for potential stray records left by testing
continue
yield (inds_by_vid[e[0]], e[1])
def count_males(self, inds):
return sum([i.gender == self.kb.Gender.MALE for i in inds])
def do_enrollment(self):
controls = []
affected = []
# FIXME in a future version, we will be able to issue actual AQL
# queries.
for i, ehr in self.get_ehr_iterator():
if ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1):
affected.append(i)
elif ehr.matches(EXCLUSION): # FIXME we assume that this is enough
controls.append(i)
# A more sophisticated example
# will keep only records where age of onset is between 10y and 15y
# for i, ehr in self.get_ehr_iterator():
# if (ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1)
# and
# ehr.matches(DIAGNOSIS, DIAGNOSIS_AGE_OF_ONSET, ("10y", "15y")):
# affected.append(i)
print("there are %d controls [%d male]" % (len(controls), self.count_males(controls)))
print("there are %d affected [%d male] with E10" % (len(affected), self.count_males(affected)))
class VidMapper(object):
def __init__(self, host, user, passwd, logger, study):
self.logger = logger
self.study = study
self.kb = KB(driver="omero")(host, user, passwd)
plates = self.kb.get_objects(self.kb.TiterPlate)
self.logger.info("fetched %d plates" % len(plates))
self.plate_map = {}
self.enroll_map = {}
for p in plates:
self.plate_map[p.omero_id] = p.barcode
s = self.kb.get_study(self.study)
enrolls = self.kb.get_enrolled(s)
self.logger.info("fetched %d enrollments" % len(enrolls))
for e in enrolls:
self.logger.debug("Retrieving wells for %s" % e.studyCode)
wells = [w for w in self.kb.get_vessels_by_individual(e.individual, "PlateWell")]
self.enroll_map[e.studyCode] = wells
def map_vid(self, r):
en_code = r["source"]
pl_barcode = en_code.split("|")[1]
try:
wells = self.enroll_map[en_code]
except KeyError:
msg = "%s is not enrolled in %s" % (en_code, self.study)
self.logger.error(msg)
raise ValueError(msg)
self.logger.info("found %d wells for %s" % (len(wells), en_code))
imm_wells = [w for w in wells if self.plate_map[w.container.omero_id] == pl_barcode]
if len(imm_wells) > 1:
msg = "more than 1 (%d) immuno wells for plate %s" % (len(imm_wells), pl_barcode)
self.logger.error(msg)
raise ValueError(msg)
elif len(imm_wells) == 0:
msg = "no immuno well for plate %s" % pl_barcode
self.logger.warn(msg)
raise ValueError(msg)
else:
r["source"] = imm_wells[0].id
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_vessel(self):
conf, v = self.create_vessel()
self.kill_list.append(v.save())
self.check_object(v, conf, self.kb.Vessel)
def test_tube(self):
conf, v = self.create_tube()
self.kill_list.append(v.save())
self.check_object(v, conf, self.kb.Tube)
def __init__(self, host=None, user=None, passwd=None, group=None,
keep_tokens=1, study_label=None, logger=None):
self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens)
self.logger = logger if logger else logging.getLogger()
self.record_counter = 0
self.default_study = None
if study_label:
s = self.kb.get_study(study_label)
if not s:
raise ValueError('No known study with label %s' % study_label)
self.logger.info('Selecting %s[%d,%s] as default study' %
(s.label, s.omero_id, s.id))
self.default_study = s
def main():
parser = make_parser()
args = parser.parse_args()
kb = KB(driver='omero')(args.host, args.user, args.passwd)
by_label = dict(((x.label, x) for x in kb.get_objects(kb.GenotypeDataSample)))
msets = {}
itsv = csv.DictReader(args.data_samples, delimiter='\t')
otsv = csv.DictWriter(open(args., mode='w'),
fieldnames=['path', 'data_sample_label', 'mimetype',
'size', 'sha1'],
delimiter='\t')
otsv.writeheader()
for r in itsv:
ds_label = r['label']
logger.info('Gathering info on %s' % ds_label)
if ds_label not in by_label:
logger.critical('There is no GenotypeDataSample with label %s' % label)
sys.exit(1)
ds = by_label[ds_label]
# FIXME
if ds.spnMarkersSet.omero_id not in msets:
ms = ds.snpMarkersSet
ms.load_markers()
msets[ds.snpMarkersSet.omero_id] = ms
ms = msets[ds.snpMarkersSet.omero_id]
fname = ds_label + '_do.ssc'
make_data_object(ds.id, fname, ms)
size = os.stat(fname).st_size
sha1 = compute_sha1(fname)
otsv.writerow({
'path' : 'file://' + os.path.realpath(fname),
'data_sample' : ds.id,
'mimetype' : mimetypes.SSC_FILE,
'size' : size,
'sha1' : sha1,
})
class App(object):
def __init__(self, host, user, passwd):
self.kb = KB(driver="omero")(host, user, passwd)
self.logger = logging.getLogger()
def compute(self, maker, model, release):
mset = self.kb.get_snp_markers_set(maker, model, release)
if not mset:
raise ValueError("SNPMarkersSet[%s,%s,%s] has not been defined." % (maker, model, release))
# projector = (np.arange(0, 100), np.array([101, 109]), np.arange(110,N))
# selector = kb.build_selector(
# s = self.kb.get_gdo_iterator(mset, selector, projector)
s = self.kb.get_gdo_iterator(mset)
# --
start = time.clock()
counts = algo.count_homozygotes(s)
print "counts on %d:" % counts[0], time.clock() - start
start = time.clock()
mafs = algo.maf(None, counts)
print "mafs on %d:" % counts[0], time.clock() - start
start = time.clock()
hwe = algo.hwe(None, counts)
print "hwe on %d:" % counts[0], time.clock() - start
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
log_level = getattr(logging, args.loglevel)
kwargs = {'format' : LOG_FORMAT,
'datefmt' : LOG_DATEFMT,
'level' : log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
kb = KB(driver='omero')(args.host, args.user, args.passwd)
logger.info('Loading GenotypeDataSample objects')
dsamples = kb.get_objects(kb.GenotypeDataSample)
logger.info('Loaded %d objects' % len(dsamples))
logger.info('Loading SNPMarkersSet')
query = 'SELECT snpm FROM SNPMarkersSet snpm WHERE snpm.label = :mset_label'
mset = kb.find_all_by_query(query, {'mset_label' : args.marker_set})[0]
if not mset:
logger.error('Unable to load SNPMarkersSet with label %s' % args.marker_set)
sys.exit(2)
else:
logger.info('Object loaded')
gdo_iterator = kb.get_gdo_iterator(mset, dsamples[:args.fetch_size])
gdos = []
logger.info('Loading GDOs')
for gdo in gdo_iterator:
logger.info(gdo['vid'])
gdos.append(gdo)
logger.debug('%d/%d GDOs loaded' % (len(gdos), args.fetch_size))
logger.info('Loaded %d GDOs' % len(gdos))
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
kb = KB(driver='omero')(args.host, args.user, args.passwd)
logger.info('Loading GenotypeDataSample objects')
dsamples = kb.get_objects(kb.GenotypeDataSample)
logger.info('Loaded %d objects' % len(dsamples))
logger.info('Loading SNPMarkersSet')
query = 'SELECT snpm FROM SNPMarkersSet snpm WHERE snpm.label = :mset_label'
mset = kb.find_all_by_query(query, {'mset_label': args.marker_set})[0]
if not mset:
logger.error('Unable to load SNPMarkersSet with label %s' %
args.marker_set)
sys.exit(2)
else:
logger.info('Object loaded')
gdo_iterator = kb.get_gdo_iterator(mset, dsamples[:args.fetch_size])
gdos = []
logger.info('Loading GDOs')
for gdo in gdo_iterator:
logger.info(gdo['vid'])
gdos.append(gdo)
logger.debug('%d/%d GDOs loaded' % (len(gdos), args.fetch_size))
logger.info('Loaded %d GDOs' % len(gdos))
class App(object):
def __init__(self, host, user, passwd):
self.kb = KB(driver='omero')(host, user, passwd)
self.logger = logging.getLogger()
def compute(self, maker, model, release):
mset = self.kb.get_snp_markers_set(maker, model, release)
if not mset:
raise ValueError('SNPMarkersSet[%s,%s,%s] has not been defined.' %
(maker, model, release))
# projector = (np.arange(0, 100), np.array([101, 109]), np.arange(110,N))
# selector = kb.build_selector(
# s = self.kb.get_gdo_iterator(mset, selector, projector)
s = self.kb.get_gdo_iterator(mset)
#--
start = time.clock()
counts = algo.count_homozygotes(s)
print 'counts on %d:' % counts[0], time.clock() - start
start = time.clock()
mafs = algo.maf(None, counts)
print 'mafs on %d:' % counts[0], time.clock() - start
start = time.clock()
hwe = algo.hwe(None, counts)
print 'hwe on %d:' % counts[0], time.clock() - start
def __init__(self, host, user, passwd, logger, study):
self.logger = logger
self.study = study
self.kb = KB(driver="omero")(host, user, passwd)
plates = self.kb.get_objects(self.kb.TiterPlate)
self.logger.info("fetched %d plates" % len(plates))
self.plate_map = {}
self.enroll_map = {}
for p in plates:
self.plate_map[p.omero_id] = p.barcode
s = self.kb.get_study(self.study)
enrolls = self.kb.get_enrolled(s)
self.logger.info("fetched %d enrollments" % len(enrolls))
for e in enrolls:
self.logger.debug("Retrieving wells for %s" % e.studyCode)
wells = [w for w in self.kb.get_vessels_by_individual(e.individual, "PlateWell")]
self.enroll_map[e.studyCode] = wells
def __init__(self,
host=None,
user=None,
passwd=None,
group=None,
keep_tokens=1,
study_label=None,
logger=None):
self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens)
self.logger = logger if logger else logging.getLogger()
self.record_counter = 0
self.default_study = None
if study_label:
s = self.kb.get_study(study_label)
if not s:
raise ValueError('No known study with label %s' % study_label)
self.logger.info('Selecting %s[%d,%s] as default study' %
(s.label, s.omero_id, s.id))
self.default_study = s
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_individual(self):
conf, i = self.create_individual()
self.kill_list.append(i.save())
self.check_object(i, conf, self.kb.Individual)
def test_enrollment(self):
conf, e = self.create_enrollment()
self.kill_list.append(e.save())
self.check_object(e, conf, self.kb.Enrollment)
def test_enrollment_ops(self):
conf, e = self.create_enrollment()
e.save()
study = e.study
xe = self.kb.get_enrollment(study, conf['studyCode'])
self.assertTrue(not xe is None)
self.assertEqual(xe.id, e.id)
self.kb.delete(e)
self.assertEqual(self.kb.get_enrollment(study, conf['studyCode']), None)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_individual(self):
conf, i = self.create_individual()
self.kill_list.append(i.save())
self.check_object(i, conf, self.kb.Individual)
def test_enrollment(self):
conf, e = self.create_enrollment()
self.kill_list.append(e.save())
self.check_object(e, conf, self.kb.Enrollment)
def test_enrollment_ops(self):
conf, e = self.create_enrollment()
e.save()
study = e.study
xe = self.kb.get_enrollment(study, conf['studyCode'])
self.assertTrue(not xe is None)
self.assertEqual(xe.id, e.id)
self.kb.delete(e)
self.assertEqual(self.kb.get_enrollment(study, conf['studyCode']),
None)
def __init__(self, host, user, passwd, logger, study):
self.logger = logger
self.study = study
self.kb = KB(driver="omero")(host, user, passwd)
plates = self.kb.get_objects(self.kb.TiterPlate)
self.logger.info("fetched %d plates" % len(plates))
self.plate_map = {}
self.enroll_map = {}
for p in plates:
self.plate_map[p.omero_id] = p.barcode
s = self.kb.get_study(self.study)
enrolls = self.kb.get_enrolled(s)
self.logger.info("fetched %d enrollments" % len(enrolls))
for e in enrolls:
self.logger.debug('Retrieving wells for %s' % e.studyCode)
wells = [
w for w in self.kb.get_vessels_by_individual(
e.individual, "PlateWell")
]
self.enroll_map[e.studyCode] = wells
fetch all markers corresponding to the VIDs and get their masks
split mask and compare alleles
"""
import sys, os, csv
import itertools as it
from bl.vl.kb import KnowledgeBase as KB
from bl.core.seq.utils import reverse_complement as rc
import bl.vl.utils.snp as snp
OME_HOST = os.getenv('OME_HOST', 'localhost')
OME_USER = os.getenv('OME_USER', 'test')
OME_PASSWD = os.getenv('OME_PASSWD', 'test')
kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASSWD)
fn = sys.argv[1] # "affy_na32_reannot_vids.tsv"
outfn = sys.argv[2] # "affy_na32_markers_set_def.tsv"
with open(fn) as f:
reader = csv.DictReader(f, delimiter="\t")
records = [r for r in reader]
vids = [r['source'] for r in records]
markers = kb.get_snp_markers(vids=vids, col_names=['vid', 'mask'])
with open(outfn, 'w') as outf:
fieldnames = ['marker_vid', 'marker_indx', 'allele_flip']
writer = csv.DictWriter(outf, delimiter="\t", lineterminator=os.linesep,
fieldnames=fieldnames)
'datefmt' : LOG_DATEFMT,
'level' : log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Start preloading dependency tree')
kb.update_dependency_tree()
logger.info('Dependency tree loaded')
with open(args.out_file, 'w') as ofile:
writer = csv.DictWriter(ofile, ['target', 'new_source_type',
'new_source', 'old_source'],
delimiter='\t')
writer.writeheader()
logger.info('Loading wells data')
for w in kb.get_objects(kb.PlateWell):
if type(w.action.target) == kb.Individual:
tubes = list(kb.dt.get_connected(w, kb.Tube))
if len(tubes) == 1:
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Retrieving diagnosis records')
ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
DIAGNOSIS_ARCHETYPE)
ehr_records.extend(
kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
EXCL_DIAG_ARCHETYPE))
logger.info('%d records retrieved' % len(ehr_records))
logger.info('Loading actions')
actions = kb.get_objects(kb.Action)
act_map = {}
for act in actions:
act_map[act.id] = act.context.label
logger.info('%d actions loaded' % len(act_map))
* Explanations concerning how to open a connection to a KnowledgeBase
and what it is are left to other, antecedent, parts of the
documentation.
"""
import os
import itertools as it
from bl.vl.kb import KnowledgeBase as KB
import bl.vl.genotype.algo as algo
OME_HOST = os.getenv('OME_HOST', 'localhost')
OME_USER = os.getenv('OME_USER', 'test')
OME_PASSWD = os.getenv('OME_PASSWD', 'test')
kb = KB(driver="omero")(OME_HOST, OME_USER, OME_PASSWD)
""" ..
The first thing we will do is to select a markers set. See FIXME:XXX
for its definition. We will first obtain an handle to it, and then
invoke a '.load_markers()' that will bring in memory the actual definition
data.
"""
mset_name = 'FakeTaqSet01'
mset0 = kb.get_snp_markers_set(label=mset_name)
mset0.load_markers()
""" ..
For the time being, we can think the SNPMarkerSet mset0 as analogous to an array
open a tsv with at least the 'allele_a', 'allele_b' and marker_vid columns
fetch all markers corresponding to the VIDs and get their masks
split mask and compare alleles
"""
import sys, os, csv
import itertools as it
from bl.vl.kb import KnowledgeBase as KB
from bl.core.seq.utils import reverse_complement as rc
import bl.vl.utils.snp as snp
OME_HOST = os.getenv('OME_HOST', 'localhost')
OME_USER = os.getenv('OME_USER', 'test')
OME_PASSWD = os.getenv('OME_PASSWD', 'test')
kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASSWD)
fn = sys.argv[1] # "affy_na32_reannot_vids.tsv"
outfn = sys.argv[2] # "affy_na32_markers_set_def.tsv"
with open(fn) as f:
reader = csv.DictReader(f, delimiter="\t")
records = [r for r in reader]
vids = [r['source'] for r in records]
markers = kb.get_snp_markers(vids=vids, col_names=['vid', 'mask'])
with open(outfn, 'w') as outf:
fieldnames = ['marker_vid', 'marker_indx', 'allele_flip']
writer = csv.DictWriter(outf,
delimiter="\t",
class App(object):
def __init__(self, host, user, passwd,
study_label,
maker, model, release):
self.kb = KB(driver='omero')(host, user, passwd)
self.mset = self.kb.get_snp_markers_set(maker, model, release)
self.logger = logging.getLogger()
if not self.mset:
raise ValueError('SNPMarkersSet[%s,%s,%s] has not been defined.'
% (maker, model, release))
#--
alabel = 'load_genotypes-setup-%s' % time.time()
self.asetup = self.kb.factory.create(self.kb.ActionSetup,
{'label' : alabel,
'conf' : ''}).save()
#--
dmaker, dmodel, drelease = 'CRS4', 'load_genotypes', '0.1'
dlabel = '%s-%s-%s' % (dmaker, dmodel, drelease)
device = self.kb.get_device(dlabel)
if not device:
device = self.kb.factory.create(self.kb.Device,
{'label' : dlabel,
'maker' : dmaker,
'model' : dmodel,
'release' : drelease}).save()
self.device = device
#-- FIXME this will break if study is not defined.
self.study = self.kb.get_study(study_label)
def check_snp_markers_set(self, marker_types, marker_names):
self.logger.info('start checking snp_markers_set')
mdefs, msetc = self.kb.get_snp_markers_set_content(self.mset)
rs_labels = mdefs['rs_label']
for t, n in it.izip(marker_types, marker_names):
if t == 'M':
if not n in rs_labels:
msg = 'marker %s is not in the specified SNPMarkersSet' % n
self.logger.critical(msg)
raise ValueError(msg)
self.logger.info('done checking snp_markers_set')
def create_action(self, target):
conf = {'setup' : self.asetup,
'device' : self.device,
'actionCategory' : self.kb.ActionCategory.MEASUREMENT,
'operator' : 'Alfred E. Neumann',
'context' : self.study,
'target' : target,
}
action = self.kb.factory.create(self.kb.ActionOnVessel, conf).save()
return action
def create_data_sample(self, action, label):
conf = {'snpMarkersSet' : self.mset,
'label' : label,
'status' : self.kb.DataSampleStatus.USABLE,
'action' : action}
return self.kb.factory.create(self.kb.GenotypeDataSample, conf).save()
def load(self, pedfile, datfile, conf_value=1.0):
pr = PedReader(pedfile, datfile, conf_value)
self.check_snp_markers_set(pr.marker_types, pr.marker_names)
#--
self.logger.info('start loading from pedfile %s' % pedfile.name)
for x in pr:
sample = self.kb.get_vessel(x['sample_label'])
if not sample:
self.logger.error('No sample with label %s in VL' % x['sample_label'])
continue
action = self.create_action(sample)
avid = action.id
action.unload()
data_sample = self.create_data_sample(action, x['label'])
data_object = self.kb.add_gdo_data_object(avid, data_sample,
x['probs'], x['confs'])
self.logger.info('-- loaded %s' % x['label'])
self.logger.info('done loading from pedfile %s' % pedfile.name)
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logging.info('Loading EHR records')
ehr_records = kb.get_ehr_records()
logging.info('Loaded %d EHR records' % len(ehr_records))
ehr_lookup = {}
for rec in ehr_records:
key = get_ehr_key(rec)
ehr_lookup.setdefault(rec['i_id'], {}).setdefault(key, []).append(rec)
logger.debug('Loaded EHR lookup table, %d individuals involved' %
len(ehr_lookup.keys()))
for ind, records in ehr_lookup.iteritems():
for k, vals in records.iteritems():
if len(vals) > 1:
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
enrolls_map = get_enrollments_lookup(kb, logger)
logger.info('Loading Individuals')
inds = kb.get_objects(kb.Individual)
logger.info('Loaded %d Individuals' % len(inds))
with open(args.in_file) as in_file, open(args.out_file, 'w') as out_file:
ts = long(time.time())
reader = csv.DictReader(in_file, delimiter='\t')
writer = csv.DictWriter(out_file, [
'study', 'individual', 'timestamp', 'birth_date', 'birth_place',
'birth_place_district'
],
delimiter='\t')
'datefmt' : LOG_DATEFMT,
'level' : log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Retrieving diagnosis records')
ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
DIAGNOSIS_ARCHETYPE)
ehr_records.extend(kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
EXCL_DIAG_ARCHETYPE))
logger.info('%d records retrieved' % len(ehr_records))
logger.info('Loading actions')
actions = kb.get_objects(kb.Action)
act_map = {}
for act in actions:
act_map[act.id] = act.context.label
logger.info('%d actions loaded' % len(act_map))
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_study(self):
conf, s = self.create_study()
self.kill_list.append(s.save())
self.check_object(s, conf, self.kb.Study)
def test_study_ops(self):
conf, s = self.create_study()
s.save()
xs = self.kb.get_study(conf['label'])
self.assertTrue(not xs is None)
self.assertEqual(xs.id, s.id)
self.assertEqual(xs.label, s.label)
self.kb.delete(s)
self.assertEqual(self.kb.get_study(conf['label']), None)
def test_device(self):
conf, d = self.create_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.Device)
def test_hardware_device(self):
conf, d = self.create_hardware_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.HardwareDevice)
def test_device_ops(self):
conf, d = self.create_device()
d.save()
xs = self.kb.get_device(conf['label'])
self.assertTrue(not xs is None)
self.check_object(xs, conf, self.kb.Device)
self.kb.delete(d)
self.assertEqual(self.kb.get_device(conf['label']), None)
def test_action_setup(self):
conf, a = self.create_action_setup()
self.kill_list.append(a.save())
self.check_object(a, conf, self.kb.ActionSetup)
def test_action(self):
conf, action = self.create_action()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.Action)
def test_action_on_vessel(self):
conf, action = self.create_action_on_vessel()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnVessel)
def test_action_on_data_sample(self):
conf, action = self.create_action_on_data_sample()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
def test_action_on_data_collection_item(self):
conf, action = self.create_action_on_data_collection_item()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
'datefmt' : LOG_DATEFMT,
'level' : log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
enrolls_map = get_enrollments_lookup(kb, logger)
logger.info('Loading Individuals')
inds = kb.get_objects(kb.Individual)
logger.info('Loaded %d Individuals' % len(inds))
with open(args.in_file) as in_file, open(args.out_file, 'w') as out_file:
ts = long(time.time())
reader = csv.DictReader(in_file, delimiter='\t')
writer = csv.DictWriter(out_file, ['study', 'individual', 'timestamp',
'birth_date', 'birth_place',
'birth_place_district'],
delimiter='\t')
writer.writeheader()
'datefmt' : LOG_DATEFMT,
'level' : log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
plates = get_omniexpress_plates(kb)
logger.info('Retrieved %d OMNIEXPRESS related plates' % len(plates))
logger.info('Calculating dependency tree')
kb.update_dependency_tree()
logger.info('Dependency tree loaded')
with open(args.ofile, 'w') as ofile:
writer = csv.DictWriter(ofile, ['source', 'study', 'label'],
delimiter = '\t')
writer.writeheader()
inds = []
* Explanations concerning how to open a connection to a KnowledgeBase
and what it is are left to other, antecedent, parts of the
documentation.
"""
import os
import itertools as it
from bl.vl.kb import KnowledgeBase as KB
import bl.vl.genotype.algo as algo
OME_HOST = os.getenv('OME_HOST', 'localhost')
OME_USER = os.getenv('OME_USER', 'test')
OME_PASSWD = os.getenv('OME_PASSWD', 'test')
kb = KB(driver="omero")(OME_HOST, OME_USER, OME_PASSWD)
""" ..
The first thing we will do is to select a markers set. See FIXME:XXX
for its definition. We will first obtain an handle to it, and then
invoke a '.load_markers()' that will bring in memory the actual definition
data.
"""
mset_name = 'FakeTaqSet01'
mset0 = kb.get_snp_markers_set(label=mset_name)
mset0.load_markers()
""" ..
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_study(self):
conf, s = self.create_study()
self.kill_list.append(s.save())
self.check_object(s, conf, self.kb.Study)
def test_study_ops(self):
conf, s = self.create_study()
s.save()
xs = self.kb.get_study(conf['label'])
self.assertTrue(not xs is None)
self.assertEqual(xs.id, s.id)
self.assertEqual(xs.label, s.label)
self.kb.delete(s)
self.assertEqual(self.kb.get_study(conf['label']), None)
def test_device(self):
conf, d = self.create_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.Device)
def test_hardware_device(self):
conf, d = self.create_hardware_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.HardwareDevice)
def test_device_ops(self):
conf, d = self.create_device()
d.save()
xs = self.kb.get_device(conf['label'])
self.assertTrue(not xs is None)
self.check_object(xs, conf, self.kb.Device)
self.kb.delete(d)
self.assertEqual(self.kb.get_device(conf['label']), None)
def test_action_setup(self):
conf, a = self.create_action_setup()
self.kill_list.append(a.save())
self.check_object(a, conf, self.kb.ActionSetup)
def test_action(self):
conf, action = self.create_action()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.Action)
def test_action_on_vessel(self):
conf, action = self.create_action_on_vessel()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnVessel)
def test_action_on_data_sample(self):
conf, action = self.create_action_on_data_sample()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
def test_action_on_data_collection_item(self):
conf, action = self.create_action_on_data_collection_item()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
log_level = getattr(logging, args.loglevel)
kwargs = {"format": LOG_FORMAT, "datefmt": LOG_DATEFMT, "level": log_level}
if args.logfile:
kwargs["filename"] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
kb = KB(driver="omero")(args.host, args.user, args.passwd)
# Load enrollments and individual (needed to build sample label and for gender field)
enrolls = []
for sl in STUDY_LABELS:
logger.debug("Loading enrollments for study %s" % sl)
enrolls.extend(kb.get_enrolled(kb.get_study(sl)))
logger.debug("Fetched %d individuals" % len(enrolls))
wells_lookup = get_wells_enrolls_lookup(enrolls, kb)
logger.debug("Loading EHR records")
ehr_records = kb.get_ehr_records("(valid == True)")
ehr_records_map = {}
for r in ehr_records:
ehr_records_map.setdefault(r["i_id"], []).append(r)
# Read plate barcodes
with open(args.plates_list) as pl_list:
barcodes = [row.strip() for row in pl_list]
# Load plate
for plate_barcode in barcodes:
logger.info("Creating datasheet for plate %s" % plate_barcode)
pl = load_plate(plate_barcode, kb)
if not pl:
logger.error("No plate with barcode %s exists, skipping it." % (plate_barcode))
continue
# Load wells for selected plate
pl_wells = get_wells_by_plate(plate_barcode, kb)
with open(os.path.join(args.out_dir, "%s_datasheet.csv" % plate_barcode), "w") as of:
writer = csv.DictWriter(of, CSV_FIELDS, delimiter="\t")
writer.writeheader()
last_slot = 0
for slot, well in sorted(pl_wells.iteritems()):
cl_record = ehr_records_map[wells_lookup[well][0].individual.id]
t1d, ms = get_affections(cl_record)
# Fill empty slots
while last_slot != slot - 1:
last_slot += 1
writer.writerow(
{
"Sample_ID": "X",
"PLATE_barcode": pl.barcode,
"PLATE_name": pl.label,
"WELL_label": get_well_label(last_slot),
"INDIVIDUAL_gender": "X",
"INDIVIDUAL_vid": "X",
"T1D_affected": "X",
"MS_affected": "X",
}
)
writer.writerow(
{
"Sample_ID": get_ichip_sample_code(wells_lookup[well], pl.barcode),
"PLATE_barcode": pl.barcode,
"PLATE_name": pl.label,
"WELL_label": well.label,
"INDIVIDUAL_gender": map_gender(wells_lookup[well][0].individual),
"INDIVIDUAL_vid": wells_lookup[well][0].individual.id,
"T1D_affected": t1d,
"MS_affected": ms,
}
)
last_slot = slot
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.debug('Reading codes from source list')
with open(args.source_list) as f:
codes = [row.strip() for row in f.readlines()]
logger.debug('Found %d codes to discard' % len(codes))
logger.debug('Retrieving enrollments for study %s' % args.source_study)
source_enrolls = kb.get_enrolled(kb.get_study(args.source_study))
logger.debug('Retrieved %d enrollments' % len(source_enrolls))
src_st_lookup = {}
for sen in source_enrolls:
src_st_lookup[sen.studyCode] = sen
to_be_discarded = []
discard_st = kb.get_study(args.discard_study)
def setUp(self): self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
with open(args.couples_list) as f:
reader = csv.reader(f, delimiter='\t')
couples = []
for row in reader:
couples.append((row[0], row[1]))
logger.info('%d couples are going to be swapped' % len(couples))
logger.debug('Retrieving enrollments for study %s' % args.study)
enrolls = kb.get_enrolled(kb.get_study(args.study))
logger.debug('Retrieved %d enrollments' % len(enrolls))
en_lookup = {}
for en in enrolls:
en_lookup[en.studyCode] = en
importer tools for more efficient solutions.
First, as usual, we open the connection to the KnowledgeBase
"""
import os
import numpy as np
from bl.vl.kb import KnowledgeBase as KB
OME_HOST = os.getenv('OME_HOST', 'localhost')
OME_USER = os.getenv('OME_USER', 'test')
OME_PASSWD = os.getenv('OME_PASSWD', 'test')
kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASSWD)
""" ..
The following is the collection of (fake) markers used in the TaqMan assays.
"""
taq_man_markers = [
('A001', 'xrs122652', 'TCACTTCTTCAAAGCT[A/G]AGCTACAAGCATTATT'),
('A002', 'xrs741592', 'GGAAGGAAGAAATAAA[C/G]CAGCACTATGTCTGGC'),
('A003', 'xrs807079', 'CCGACCTAGTAGGCAA[A/G]TAGACACTGAGGCTGA'),
('A004', 'xrs567736', 'AGGTCTATGTTAATAC[A/G]GAATCAGTTTCTCACC'),
('A005', 'xrs4693427', 'AGATTACCATGCAGGA[A/T]CTGTTCTGAGATTAGC'),
('A006', 'xrs4757019', 'TCTACCTCTGTGACTA[C/G]AAGTGTTCTTTTATTT'),
('A007', 'xrs7958813', 'AAGGCAATACTGTTCA[C/T]ATTGTATGGAAAGAAG')
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
plates = get_omniexpress_plates(kb)
logger.info('Retrieved %d OMNIEXPRESS related plates' % len(plates))
logger.info('Calculating dependency tree')
kb.update_dependency_tree()
logger.info('Dependency tree loaded')
with open(args.ofile, 'w') as ofile:
writer = csv.DictWriter(ofile, ['source', 'study', 'label'],
delimiter='\t')
writer.writeheader()
inds = []
def setUp(self):
self.kb = KB(driver='omero')('localhost', 'root', 'romeo')
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
kb = KB(driver='omero')(args.host, args.user, args.passwd)
# Load enrollments and individual (needed to build sample label and for gender field)
enrolls = []
for sl in STUDY_LABELS:
logger.debug('Loading enrollments for study %s' % sl)
enrolls.extend(kb.get_enrolled(kb.get_study(sl)))
logger.debug('Fetched %d individuals' % len(enrolls))
wells_lookup = get_wells_enrolls_lookup(enrolls, kb)
logger.debug('Loading EHR records')
ehr_records = kb.get_ehr_records('(valid == True)')
ehr_records_map = {}
for r in ehr_records:
ehr_records_map.setdefault(r['i_id'], []).append(r)
# Read plate barcodes
with open(args.plates_list) as pl_list:
barcodes = [row.strip() for row in pl_list]
# Load plate
for plate_barcode in barcodes:
logger.info('Creating datasheet for plate %s' % plate_barcode)
pl = load_plate(plate_barcode, kb)
if not pl:
logger.error('No plate with barcode %s exists, skipping it.' %
(plate_barcode))
continue
# Load wells for selected plate
pl_wells = get_wells_by_plate(plate_barcode, kb)
with open(
os.path.join(args.out_dir, '%s_datasheet.csv' % plate_barcode),
'w') as of:
writer = csv.DictWriter(of, CSV_FIELDS, delimiter='\t')
writer.writeheader()
last_slot = 0
for slot, well in sorted(pl_wells.iteritems()):
cl_record = ehr_records_map[wells_lookup[well]
[0].individual.id]
t1d, ms = get_affections(cl_record)
# Fill empty slots
while (last_slot != slot - 1):
last_slot += 1
writer.writerow({
'Sample_ID': 'X',
'PLATE_barcode': pl.barcode,
'PLATE_name': pl.label,
'WELL_label': get_well_label(last_slot),
'INDIVIDUAL_gender': 'X',
'INDIVIDUAL_vid': 'X',
'T1D_affected': 'X',
'MS_affected': 'X'
})
writer.writerow({
'Sample_ID':
get_ichip_sample_code(wells_lookup[well], pl.barcode),
'PLATE_barcode':
pl.barcode,
'PLATE_name':
pl.label,
'WELL_label':
well.label,
'INDIVIDUAL_gender':
map_gender(wells_lookup[well][0].individual),
'INDIVIDUAL_vid':
wells_lookup[well][0].individual.id,
'T1D_affected':
t1d,
'MS_affected':
ms
})
last_slot = slot
def setUp(self):
self.kb = KB(driver="omero")("localhost", "root", "romeo")
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
