class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
logger.info('start setup')
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
logger.info('done with setup')
def tearDown(self):
logger.info('start tear-down')
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
logger.info('done with tear-down')
def create_archetype_record(self):
terminology = 'terminology://apps.who.int/classifications/apps/'
term = terminology + 'gE10.htm#E10'
archetype = 'openEHR-EHR-EVALUATION.problem-diagnosis.v1'
field = 'at0002.1'
rec = {field : term}
selector = ('(archetype=="%s")&(field=="%s")&(svalue=="%s")'
% (archetype, field, term))
return archetype, rec, selector
def test_ehr_record(self):
logger.info('creating ActionOnIndividual')
conf, action = self.create_action_on_individual()
self.kill_list.append(action.save())
logger.info('done creating ActionOnIndividual')
archetype, fields, selector = self.create_archetype_record()
# FIXME there should be a function for this...
timestamp = int(time.time() * 1000)
self.kb.add_ehr_record(action, timestamp, archetype, fields)
rs = self.kb.get_ehr_records(selector)
individual = action.target
individual.reload()
self.assertTrue(len(rs) > 0)
not_found = True
for r in rs:
for k in ['a_id', 'i_id', 'timestamp', 'archetype', 'fields']:
self.assertTrue(k in r)
if r['a_id'] == action.id:
not_found = False
self.assertEqual(r['i_id'], individual.id)
self.assertEqual(r['timestamp'], timestamp)
self.assertEqual(r['archetype'], archetype)
nfields = r['fields']
self.assertTrue(len(nfields) >= len(fields))
for k in fields:
self.assertTrue(k in nfields)
self.assertEqual(nfields[k], fields[k])
break
else:
self.assertTrue(not_found)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
logger.info('start setup')
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
logger.info('done with setup')
def tearDown(self):
logger.info('start tear-down')
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
logger.info('done with tear-down')
def create_archetype_record(self):
terminology = 'terminology://apps.who.int/classifications/apps/'
term = terminology + 'gE10.htm#E10'
archetype = 'openEHR-EHR-EVALUATION.problem-diagnosis.v1'
field = 'at0002.1'
rec = {field: term}
selector = ('(archetype=="%s")&(field=="%s")&(svalue=="%s")' %
(archetype, field, term))
return archetype, rec, selector
def test_ehr_record(self):
logger.info('creating ActionOnIndividual')
conf, action = self.create_action_on_individual()
self.kill_list.append(action.save())
logger.info('done creating ActionOnIndividual')
archetype, fields, selector = self.create_archetype_record()
# FIXME there should be a function for this...
timestamp = int(time.time() * 1000)
self.kb.add_ehr_record(action, timestamp, archetype, fields)
rs = self.kb.get_ehr_records(selector)
individual = action.target
individual.reload()
self.assertTrue(len(rs) > 0)
not_found = True
for r in rs:
for k in ['a_id', 'i_id', 'timestamp', 'archetype', 'fields']:
self.assertTrue(k in r)
if r['a_id'] == action.id:
not_found = False
self.assertEqual(r['i_id'], individual.id)
self.assertEqual(r['timestamp'], timestamp)
self.assertEqual(r['archetype'], archetype)
nfields = r['fields']
self.assertTrue(len(nfields) >= len(fields))
for k in fields:
self.assertTrue(k in nfields)
self.assertEqual(nfields[k], fields[k])
break
else:
self.assertTrue(not_found)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_container(self):
conf, c = self.create_container()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.Container)
def test_slotted_container(self):
conf, c = self.create_slotted_container()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.SlottedContainer)
def test_titer_plate(self):
conf, c = self.create_titer_plate()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.TiterPlate)
def test_data_collection(self):
conf, c = self.create_data_collection()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.DataCollection)
def test_data_collection_item(self):
conf, c = self.create_data_collection_item()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.DataCollectionItem)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver="omero")(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, "ome_obj"):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, "_id"):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_data_sample(self):
conf, ds = self.create_data_sample()
self.kill_list.append(ds.save())
self.check_object(ds, conf, self.kb.DataSample)
def test_affymetrix_cel(self):
conf, ds = self.create_affymetrix_cel()
self.kill_list.append(ds.save())
self.check_object(ds, conf, self.kb.AffymetrixCel)
def test_snp_markers_set(self):
conf, sms = self.create_snp_markers_set()
self.kill_list.append(sms.save())
self.check_object(sms, conf, self.kb.SNPMarkersSet)
def test_genotype_data_sample(self):
conf, gds = self.create_genotype_data_sample()
self.kill_list.append(gds.save())
self.check_object(gds, conf, self.kb.GenotypeDataSample)
def test_data_object(self):
conf, do = self.create_data_object()
self.kill_list.append(do.save())
self.check_object(do, conf, self.kb.DataObject)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_container(self):
conf, c = self.create_container()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.Container)
def test_slotted_container(self):
conf, c = self.create_slotted_container()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.SlottedContainer)
def test_titer_plate(self):
conf, c = self.create_titer_plate()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.TiterPlate)
def test_data_collection(self):
conf, c = self.create_data_collection()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.DataCollection)
def test_data_collection_item(self):
conf, c = self.create_data_collection_item()
self.kill_list.append(c.save())
self.check_object(c, conf, self.kb.DataCollectionItem)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_data_sample(self):
conf, ds = self.create_data_sample()
self.kill_list.append(ds.save())
self.check_object(ds, conf, self.kb.DataSample)
def test_affymetrix_cel(self):
conf, ds = self.create_affymetrix_cel()
self.kill_list.append(ds.save())
self.check_object(ds, conf, self.kb.AffymetrixCel)
def test_snp_markers_set(self):
conf, sms = self.create_snp_markers_set()
self.kill_list.append(sms.save())
self.check_object(sms, conf, self.kb.SNPMarkersSet)
def test_genotype_data_sample(self):
conf, gds = self.create_genotype_data_sample()
self.kill_list.append(gds.save())
self.check_object(gds, conf, self.kb.GenotypeDataSample)
def test_data_object(self):
conf, do = self.create_data_object()
self.kill_list.append(do.save())
self.check_object(do, conf, self.kb.DataObject)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_individual(self):
conf, i = self.create_individual()
self.kill_list.append(i.save())
self.check_object(i, conf, self.kb.Individual)
def test_enrollment(self):
conf, e = self.create_enrollment()
self.kill_list.append(e.save())
self.check_object(e, conf, self.kb.Enrollment)
def test_enrollment_ops(self):
conf, e = self.create_enrollment()
e.save()
study = e.study
xe = self.kb.get_enrollment(study, conf['studyCode'])
self.assertTrue(not xe is None)
self.assertEqual(xe.id, e.id)
self.kb.delete(e)
self.assertEqual(self.kb.get_enrollment(study, conf['studyCode']), None)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_individual(self):
conf, i = self.create_individual()
self.kill_list.append(i.save())
self.check_object(i, conf, self.kb.Individual)
def test_enrollment(self):
conf, e = self.create_enrollment()
self.kill_list.append(e.save())
self.check_object(e, conf, self.kb.Enrollment)
def test_enrollment_ops(self):
conf, e = self.create_enrollment()
e.save()
study = e.study
xe = self.kb.get_enrollment(study, conf['studyCode'])
self.assertTrue(not xe is None)
self.assertEqual(xe.id, e.id)
self.kb.delete(e)
self.assertEqual(self.kb.get_enrollment(study, conf['studyCode']),
None)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_parallel_save(self):
aconf, action = self.create_action()
self.kill_list.append(action.save())
N = 1000
people = []
for i in range(N):
conf, i = self.create_individual(action=action,
gender=self.kb.Gender.MALE)
self.kill_list.append(i)
people.append(i)
start = time.time()
self.kb.save_array(people)
print ' \n\ttime needed to save %s object: %s' % (N,
time.time() - start)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_parallel_save(self):
aconf, action = self.create_action()
self.kill_list.append(action.save())
N = 1000
people = []
for i in range(N):
conf, i = self.create_individual(action=action,
gender=self.kb.Gender.MALE)
self.kill_list.append(i)
people.append(i)
start = time.time()
self.kb.save_array(people)
print' \n\ttime needed to save %s object: %s' % (N, time.time() - start)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_vessel(self):
conf, v = self.create_vessel()
self.kill_list.append(v.save())
self.check_object(v, conf, self.kb.Vessel)
def test_tube(self):
conf, v = self.create_tube()
self.kill_list.append(v.save())
self.check_object(v, conf, self.kb.Tube)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_vessel(self):
conf, v = self.create_vessel()
self.kill_list.append(v.save())
self.check_object(v, conf, self.kb.Vessel)
def test_tube(self):
conf, v = self.create_tube()
self.kill_list.append(v.save())
self.check_object(v, conf, self.kb.Tube)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_study(self):
conf, s = self.create_study()
self.kill_list.append(s.save())
self.check_object(s, conf, self.kb.Study)
def test_study_ops(self):
conf, s = self.create_study()
s.save()
xs = self.kb.get_study(conf['label'])
self.assertTrue(not xs is None)
self.assertEqual(xs.id, s.id)
self.assertEqual(xs.label, s.label)
self.kb.delete(s)
self.assertEqual(self.kb.get_study(conf['label']), None)
def test_device(self):
conf, d = self.create_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.Device)
def test_hardware_device(self):
conf, d = self.create_hardware_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.HardwareDevice)
def test_device_ops(self):
conf, d = self.create_device()
d.save()
xs = self.kb.get_device(conf['label'])
self.assertTrue(not xs is None)
self.check_object(xs, conf, self.kb.Device)
self.kb.delete(d)
self.assertEqual(self.kb.get_device(conf['label']), None)
def test_action_setup(self):
conf, a = self.create_action_setup()
self.kill_list.append(a.save())
self.check_object(a, conf, self.kb.ActionSetup)
def test_action(self):
conf, action = self.create_action()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.Action)
def test_action_on_vessel(self):
conf, action = self.create_action_on_vessel()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnVessel)
def test_action_on_data_sample(self):
conf, action = self.create_action_on_data_sample()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
def test_action_on_data_collection_item(self):
conf, action = self.create_action_on_data_collection_item()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_study(self):
conf, s = self.create_study()
self.kill_list.append(s.save())
self.check_object(s, conf, self.kb.Study)
def test_study_ops(self):
conf, s = self.create_study()
s.save()
xs = self.kb.get_study(conf['label'])
self.assertTrue(not xs is None)
self.assertEqual(xs.id, s.id)
self.assertEqual(xs.label, s.label)
self.kb.delete(s)
self.assertEqual(self.kb.get_study(conf['label']), None)
def test_device(self):
conf, d = self.create_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.Device)
def test_hardware_device(self):
conf, d = self.create_hardware_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.HardwareDevice)
def test_device_ops(self):
conf, d = self.create_device()
d.save()
xs = self.kb.get_device(conf['label'])
self.assertTrue(not xs is None)
self.check_object(xs, conf, self.kb.Device)
self.kb.delete(d)
self.assertEqual(self.kb.get_device(conf['label']), None)
def test_action_setup(self):
conf, a = self.create_action_setup()
self.kill_list.append(a.save())
self.check_object(a, conf, self.kb.ActionSetup)
def test_action(self):
conf, action = self.create_action()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.Action)
def test_action_on_vessel(self):
conf, action = self.create_action_on_vessel()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnVessel)
def test_action_on_data_sample(self):
conf, action = self.create_action_on_data_sample()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
def test_action_on_data_collection_item(self):
conf, action = self.create_action_on_data_collection_item()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
class markers_set(unittest.TestCase):
def __init__(self, name):
super(markers_set, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASSWD)
conf = {
'label': 'TEST-%f' % time.time(),
'description': 'unit test garbage',
}
self.study = self.kb.factory.create(self.kb.Study, conf).save()
self.kill_list.append(self.study)
self.action = self.kb.create_an_action(self.study)
self.kill_list.append(self.action)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def create_markers(self, N):
def marker_generator():
for i in range(N):
label = 'A%f-%d' % (time.time(), i)
yield (label, label, 'ACCA[A/B]TACCA')
source, context, release = 'unit_testing', 'markers_set', '%f' % time.time()
ref_rs_genome, dbsnp_build = 'foo-rs-genome', 123000
lvs = self.kb.create_markers(source, context, release,
ref_rs_genome, dbsnp_build,
marker_generator(), self.action)
return lvs
def create_snp_markers_set(self, lvs):
label = 'ams-%f' % time.time()
maker, model, release = 'FOO', 'FOO1', '%f' % time.time()
markers_selection = [(v[1], i, False) for i, v in enumerate(lvs)]
mset = self.kb.create_snp_markers_set(label, maker, model, release,
len(lvs),
markers_selection, self.action)
self.kill_list.append(mset)
return mset
def create_alignments(self, mset, ref_genome, n_duplicates):
mset.load_markers()
self.assertTrue(len(mset) > 0)
n_aligns = len(mset.markers) + n_duplicates
pos = []
def insert_duplicates(markers):
count = 0
for i, m in enumerate(markers):
n_copies = 1
if count < n_duplicates:
count += 1
n_copies = 2
r = (m[0], random.randint(1,26), 22 + i*1000, True,
'A' if (i%2)== 0 else 'B', n_copies)
yield r
r = (m[0], random.randint(1,26), 1 + i*1000, True,
'A' if (i%2)== 0 else 'B', n_copies)
pos.append((0,0) if n_copies > 1 else (r[1], r[2]))
yield r
aligns = [x for x in insert_duplicates(mset.markers)]
random.shuffle(aligns)
self.kb.align_snp_markers_set(mset, ref_genome, aligns, self.action)
return pos
def create_data_sample(self, mset, label):
conf = {
'label': label,
'status': self.kb.DataSampleStatus.USABLE,
'action': self.action,
'snpMarkersSet': mset,
}
data_sample = self.kb.factory.create(self.kb.GenotypeDataSample,
conf).save()
self.kill_list.append(data_sample)
return data_sample
def create_data_object(self, data_sample, add_nan=False):
probs, confs = make_fake_data(data_sample.snpMarkersSet, add_nan)
do = self.kb.add_gdo_data_object(self.action, data_sample, probs, confs)
self.kill_list.append(do)
return probs, confs
def create_aligned_mset(self, N, N_dups, ref_genome):
lvs = self.create_markers(N)
mset = self.create_snp_markers_set(lvs)
pos = self.create_alignments(mset, ref_genome, N_dups)
return mset, pos
def test_creation_destruction(self):
N = 32
lvs = self.create_markers(N)
mset = self.create_snp_markers_set(lvs)
mset.load_markers()
self.assertEqual(len(mset), N)
for lv, m in it.izip(lvs, mset.markers):
self.assertEqual(lv[1], m[0])
def test_align(self):
N = 16
N_dups = 4
ref_genome = 'g' + ('%f' % time.time())[-14:]
mset, pos = self.create_aligned_mset(N, N_dups, ref_genome)
mset.load_alignments(ref_genome)
for p, m in it.izip(pos, mset.get_markers_iterator()):
self.assertEqual(p, m.position)
def test_read_ssc(self):
N = 16
N_dups = 4
ref_genome = 'g' + ('%f' % time.time())[-14:]
mset, pos = self.create_aligned_mset(N, N_dups, ref_genome)
mset.load_markers(additional_fields=['label'])
probs, confs = make_fake_data(mset)
sample_id = 'ffoo-%f' % time.time()
fn = tempfile.NamedTemporaryFile().name
make_fake_ssc(mset, sample_id, probs, confs, fn)
probs_1, confs_1 = gio.read_ssc(fn, mset)
self.assertAlmostEqual(np.sum(np.abs(probs - probs_1)), 0.0)
self.assertAlmostEqual(np.sum(np.abs(confs - confs_1)), 0.0)
def test_gdo(self):
N = 32
lvs = self.create_markers(N)
mset = self.create_snp_markers_set(lvs)
mset.load_markers()
data_sample = self.create_data_sample(mset, 'foo-data')
probs, confs = self.create_data_object(data_sample)
probs1, confs1 = data_sample.resolve_to_data()
self.assertTrue((probs == probs1).all())
self.assertTrue((confs == confs1).all())
s = self.kb.get_gdo_iterator(mset, data_samples=[data_sample])
for i, x in enumerate(s):
self.assertTrue((probs == x['probs']).all())
self.assertTrue((confs == x['confidence']).all())
self.assertEqual(i, 0)
indices = slice(N/4, N/2)
s = self.kb.get_gdo_iterator(mset, data_samples=[data_sample],
indices=indices)
for i, x in enumerate(s):
self.assertTrue((probs[:,indices] == x['probs']).all())
self.assertTrue((confs[indices] == x['confidence']).all())
self.assertEqual(i, 0)
def test_define_range_selector(self):
N, N_dups = 16, 0
ref_genome = 'g' + ('%f' % time.time())[-14:]
mset, pos = self.create_aligned_mset(N, N_dups, ref_genome)
mset.load_alignments(ref_genome)
pos.sort()
if len(pos) > 2:
low_pos, high_pos = pos[1], pos[-2]
gc_range = (low_pos, high_pos)
range_sel = self.kb.SNPMarkersSet.define_range_selector(mset, gc_range)
i = 0
for (i, m) in enumerate(mset.get_markers_iterator()):
if i in range_sel:
self.assertTrue(low_pos <= m.position <= high_pos)
else:
self.assertTrue(low_pos > m.position or high_pos < m.position)
def test_intersect(self):
ref_genome = 'g' + ('%f' % time.time())[-14:]
N1 = 16
M1 = 2
N2 = N1/2
M2 = 1
lvs = self.create_markers(N1)
mset1 = self.create_snp_markers_set(lvs)
mset1.load_markers()
aligns = [(m[0], random.randint(1,26), 1 + i*2000, True, 'A', 1)
for i, m in enumerate(mset1.markers)]
for i in range(M1):
aligns[i] = (aligns[i][0], 0, 0, True, 'A', 0)
self.kb.align_snp_markers_set(mset1, ref_genome, aligns, self.action)
lvs = self.create_markers(N2)
mset2 = self.create_snp_markers_set(lvs)
mset2.load_markers()
aligns = [(m[0], a[1], a[2], a[3], a[4], a[5])
for m, a in it.izip(mset2.markers, aligns[:len(mset2)])]
for i in range(M2):
aligns[i] = (aligns[i][0], 0, 0, True, 'A', 0)
self.kb.align_snp_markers_set(mset2, ref_genome, aligns, self.action)
mset1.load_alignments(ref_genome)
mset2.load_alignments(ref_genome)
idx1, idx2 = self.kb.SNPMarkersSet.intersect(mset1, mset1)
self.assertTrue(np.array_equal(idx1, idx2))
self.assertEqual(len(idx1), len(mset1))
self.assertEqual(len(idx1), N1)
idx1, idx2 = self.kb.SNPMarkersSet.intersect(mset1, mset2)
self.assertEqual(len(idx1), len(idx2))
self.assertEqual(len(idx1), N2 - max(M1, M2))
for i,j in it.izip(idx1, idx2):
m1, m2 = mset1[i], mset2[j]
self.assertEqual(m1.position, m2.position)
self.assertTrue(m1.position > (0,0))
def test_speed(self):
ref_genome = 'g' + ('%f' % time.time())[-14:]
N1 = 1024*1024
N2 = N1/2
beg = time.time()
lvs = self.create_markers(N1)
print ''
print 'creating %d markers took %f' % (N1, time.time() - beg)
beg = time.time()
mset1 = self.create_snp_markers_set(lvs)
print 'creating a markers set with %d markers took %f' % (N1,
time.time() - beg)
beg = time.time()
mset1.load_markers()
print 'loading markers took %f' % (time.time() - beg)
beg = time.time()
aligns = [(m[0], random.randint(1,26), 1 + i*2000, True, 'A', 1)
for i, m in enumerate(mset1.markers)]
print 'creating %d aligns took %f' % (N1, time.time() - beg)
beg = time.time()
self.kb.align_snp_markers_set(mset1, ref_genome, aligns, self.action)
print 'saving %d aligns took %f' % (N1, time.time() - beg)
beg = time.time()
mset1.load_alignments(ref_genome)
print 'loading %d aligns took %f' % (N1, time.time() - beg)
beg = time.time()
idx1, idx2 = self.kb.SNPMarkersSet.intersect(mset1, mset1)
print 'intersecting %d aligns took %f' % (N1, time.time() - beg)
def test_speed_gdo(self):
N = 934968
beg = time.time()
lvs = self.create_markers(N)
print ''
print 'creating %d markers took %f' % (N, time.time() - beg)
mset = self.create_snp_markers_set(lvs)
beg = time.time()
mset.load_markers()
print 'loading %d markers took %f' % (N, time.time() - beg)
beg = time.time()
data_sample = self.create_data_sample(mset, 'foo-data')
print 'creating a data sample took %f' % (time.time() - beg)
beg = time.time()
probs, confs = self.create_data_object(data_sample)
print 'creating a data object took %f' % (time.time() - beg)
beg = time.time()
probs1, confs1 = data_sample.resolve_to_data()
print 'resolving to data took %f' % (time.time() - beg)
self.assertTrue((probs == probs1).all())
self.assertTrue((confs == confs1).all())
beg = time.time()
s = self.kb.get_gdo_iterator(mset, data_samples=[data_sample])
for i, x in enumerate(s):
self.assertTrue((probs == x['probs']).all())
self.assertTrue((confs == x['confidence']).all())
self.assertEqual(i, 0)
print 'iterating took %f' % (time.time() - beg)
indices = slice(N/4, N/2)
beg = time.time()
s = self.kb.get_gdo_iterator(mset, data_samples=[data_sample],
indices=indices)
for i, x in enumerate(s):
self.assertTrue((probs[:,indices] == x['probs']).all())
self.assertTrue((confs[indices] == x['confidence']).all())
self.assertEqual(i, 0)
print 'iterating with indices took %f' % (time.time() - beg)
