def main(argv):
global logger
parser = make_parser()
args = parser.parse_args(argv)
logformat = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
loglevel = getattr(logging, args.loglevel)
if args.logfile:
logging.basicConfig(filename=args.logfile, format=logformat, level=loglevel)
else:
logging.basicConfig(format=logformat, level=loglevel)
logger = logging.getLogger()
abi_service = ABISnpService()
found_markers = {}
data = {}
min_datetime = datetime.max #datetime(2999, 12, 31, 23, 59, 59)
max_datetime = datetime.min #datetime(1000, 1, 1, 0, 0, 1)
for f in (_.strip() for _ in args.ifile if _.strip()):
logger.info('processing %s' % f)
sds = SDSReader(open(f), swap_sample_well_columns=True)
min_datetime = min(min_datetime, sds.datetime)
max_datetime = max(max_datetime, sds.datetime)
get_markers_definition(found_markers, sds, abi_service)
for r in sds:
data.setdefault(r['Sample Name'], []).append(r)
kb = KB(driver='omero')(args.host, args.user, args.passwd)
logger.info('qui - main')
missing_kb_markers = add_kb_marker_objects(kb, found_markers)
if missing_kb_markers:
logger.info('there are missing markers. Cannot proceed further.')
fname = '%smarker-defs.tsv' % args.prefix
write_import_markers_file(fname, found_markers, missing_kb_markers)
logger.info('the list of the missing marker is in %s.' % fname)
sys.exit(0)
fname = '%smarkers-set-def.tsv' % args.prefix
write_markers_set_def_file(fname, found_markers)
ssc_data_set = {}
device = kb.get_device(args.device_label)
for sample_id, d in data.iteritems():
fname = '%s%s-%s.ssc' % (args.prefix, device.id, sample_id)
write_ssc_data_set_file(fname, found_markers,
device.id, sample_id,
min_datetime, max_datetime, d)
ssc_data_set[sample_id] = ('taqman-%s-%s' % (args.run_id, sample_id),
sample_id, device.id, fname)
fname = '%simport.ssc' % args.prefix
write_ssc_data_samples_import_file(fname, ssc_data_set.values())
write_ssc_data_objects_import_file(fname, ssc_data_set.values())
class App(object):
def __init__(self, host, user, passwd,
study_label,
maker, model, release):
self.kb = KB(driver='omero')(host, user, passwd)
self.mset = self.kb.get_snp_markers_set(maker, model, release)
self.logger = logging.getLogger()
if not self.mset:
raise ValueError('SNPMarkersSet[%s,%s,%s] has not been defined.'
% (maker, model, release))
#--
alabel = 'load_genotypes-setup-%s' % time.time()
self.asetup = self.kb.factory.create(self.kb.ActionSetup,
{'label' : alabel,
'conf' : ''}).save()
#--
dmaker, dmodel, drelease = 'CRS4', 'load_genotypes', '0.1'
dlabel = '%s-%s-%s' % (dmaker, dmodel, drelease)
device = self.kb.get_device(dlabel)
if not device:
device = self.kb.factory.create(self.kb.Device,
{'label' : dlabel,
'maker' : dmaker,
'model' : dmodel,
'release' : drelease}).save()
self.device = device
#-- FIXME this will break if study is not defined.
self.study = self.kb.get_study(study_label)
def check_snp_markers_set(self, marker_types, marker_names):
self.logger.info('start checking snp_markers_set')
mdefs, msetc = self.kb.get_snp_markers_set_content(self.mset)
rs_labels = mdefs['rs_label']
for t, n in it.izip(marker_types, marker_names):
if t == 'M':
if not n in rs_labels:
msg = 'marker %s is not in the specified SNPMarkersSet' % n
self.logger.critical(msg)
raise ValueError(msg)
self.logger.info('done checking snp_markers_set')
def create_action(self, target):
conf = {'setup' : self.asetup,
'device' : self.device,
'actionCategory' : self.kb.ActionCategory.MEASUREMENT,
'operator' : 'Alfred E. Neumann',
'context' : self.study,
'target' : target,
}
action = self.kb.factory.create(self.kb.ActionOnVessel, conf).save()
return action
def create_data_sample(self, action, label):
conf = {'snpMarkersSet' : self.mset,
'label' : label,
'status' : self.kb.DataSampleStatus.USABLE,
'action' : action}
return self.kb.factory.create(self.kb.GenotypeDataSample, conf).save()
def load(self, pedfile, datfile, conf_value=1.0):
pr = PedReader(pedfile, datfile, conf_value)
self.check_snp_markers_set(pr.marker_types, pr.marker_names)
#--
self.logger.info('start loading from pedfile %s' % pedfile.name)
for x in pr:
sample = self.kb.get_vessel(x['sample_label'])
if not sample:
self.logger.error('No sample with label %s in VL' % x['sample_label'])
continue
action = self.create_action(sample)
avid = action.id
action.unload()
data_sample = self.create_data_sample(action, x['label'])
data_object = self.kb.add_gdo_data_object(avid, data_sample,
x['probs'], x['confs'])
self.logger.info('-- loaded %s' % x['label'])
self.logger.info('done loading from pedfile %s' % pedfile.name)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_study(self):
conf, s = self.create_study()
self.kill_list.append(s.save())
self.check_object(s, conf, self.kb.Study)
def test_study_ops(self):
conf, s = self.create_study()
s.save()
xs = self.kb.get_study(conf['label'])
self.assertTrue(not xs is None)
self.assertEqual(xs.id, s.id)
self.assertEqual(xs.label, s.label)
self.kb.delete(s)
self.assertEqual(self.kb.get_study(conf['label']), None)
def test_device(self):
conf, d = self.create_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.Device)
def test_hardware_device(self):
conf, d = self.create_hardware_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.HardwareDevice)
def test_device_ops(self):
conf, d = self.create_device()
d.save()
xs = self.kb.get_device(conf['label'])
self.assertTrue(not xs is None)
self.check_object(xs, conf, self.kb.Device)
self.kb.delete(d)
self.assertEqual(self.kb.get_device(conf['label']), None)
def test_action_setup(self):
conf, a = self.create_action_setup()
self.kill_list.append(a.save())
self.check_object(a, conf, self.kb.ActionSetup)
def test_action(self):
conf, action = self.create_action()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.Action)
def test_action_on_vessel(self):
conf, action = self.create_action_on_vessel()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnVessel)
def test_action_on_data_sample(self):
conf, action = self.create_action_on_data_sample()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
def test_action_on_data_collection_item(self):
conf, action = self.create_action_on_data_collection_item()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
class TestKB(KBObjectCreator):
def __init__(self, name):
super(TestKB, self).__init__(name)
self.kill_list = []
def setUp(self):
self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS)
def tearDown(self):
self.kill_list.reverse()
for x in self.kill_list:
self.kb.delete(x)
self.kill_list = []
def check_object(self, o, conf, otype):
try:
self.assertTrue(isinstance(o, otype))
for k in conf.keys():
v = conf[k]
# FIXME this is omero specific...
if hasattr(v, 'ome_obj'):
self.assertEqual(getattr(o, k).id, v.id)
self.assertEqual(type(getattr(o, k)), type(v))
elif hasattr(v, '_id'):
self.assertEqual(getattr(o, k)._id, v._id)
else:
self.assertEqual(getattr(o, k), v)
except:
pass
def test_study(self):
conf, s = self.create_study()
self.kill_list.append(s.save())
self.check_object(s, conf, self.kb.Study)
def test_study_ops(self):
conf, s = self.create_study()
s.save()
xs = self.kb.get_study(conf['label'])
self.assertTrue(not xs is None)
self.assertEqual(xs.id, s.id)
self.assertEqual(xs.label, s.label)
self.kb.delete(s)
self.assertEqual(self.kb.get_study(conf['label']), None)
def test_device(self):
conf, d = self.create_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.Device)
def test_hardware_device(self):
conf, d = self.create_hardware_device()
self.kill_list.append(d.save())
self.check_object(d, conf, self.kb.HardwareDevice)
def test_device_ops(self):
conf, d = self.create_device()
d.save()
xs = self.kb.get_device(conf['label'])
self.assertTrue(not xs is None)
self.check_object(xs, conf, self.kb.Device)
self.kb.delete(d)
self.assertEqual(self.kb.get_device(conf['label']), None)
def test_action_setup(self):
conf, a = self.create_action_setup()
self.kill_list.append(a.save())
self.check_object(a, conf, self.kb.ActionSetup)
def test_action(self):
conf, action = self.create_action()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.Action)
def test_action_on_vessel(self):
conf, action = self.create_action_on_vessel()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnVessel)
def test_action_on_data_sample(self):
conf, action = self.create_action_on_data_sample()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
def test_action_on_data_collection_item(self):
conf, action = self.create_action_on_data_collection_item()
self.kill_list.append(action.save())
self.check_object(action, conf, self.kb.ActionOnDataSample)
class Core(object):
def __init__(self, host=None, user=None, passwd=None, group=None,
keep_tokens=1, study_label=None, logger=None):
self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens)
self.logger = logger if logger else logging.getLogger()
self.record_counter = 0
self.default_study = None
if study_label:
s = self.kb.get_study(study_label)
if not s:
raise ValueError('No known study with label %s' % study_label)
self.logger.info('Selecting %s[%d,%s] as default study' %
(s.label, s.omero_id, s.id))
self.default_study = s
@classmethod
def find_action_setup_conf(klass, args):
action_setup_conf = {}
for x in dir(args):
if not (x.startswith('_') or x.startswith('func')):
action_setup_conf[x] = getattr(args, x)
# HACKS
action_setup_conf['ifile'] = action_setup_conf['ifile'].name
action_setup_conf['ofile'] = action_setup_conf['ofile'].name
action_setup_conf['report_file'] = action_setup_conf['report_file'].name
return action_setup_conf
@classmethod
def get_action_setup_options(klass, record, action_setup_conf = None,
object_history = None):
options = {}
if 'options' in record and record['options']:
kvs = record['options'].split(',')
for kv in kvs:
k, v = kv.split('=')
options[k] = v
if action_setup_conf:
options['importer_setup'] = action_setup_conf
if object_history:
options['object_history'] = object_history
return json.dumps(options)
def get_device(self, label, maker, model, release):
device = self.kb.get_device(label)
if not device:
self.logger.debug('creating a device')
device = self.kb.create_device(label, maker, model, release)
return device
def get_action_setup(self, label, conf):
"""
Return the ActionSetup corresponding to label if there is one,
else create a new one using conf.
"""
asetup = self.kb.get_action_setup(label)
if not asetup:
kb_conf = {
'label': label,
'conf': json.dumps(conf),
}
asetup = self.kb.factory.create(self.kb.ActionSetup, kb_conf).save()
return asetup
def get_study(self, label):
if self.default_study:
return self.default_study
study = self.kb.get_study(label)
if not study:
study = self.kb.factory.create(self.kb.Study, {'label': label}).save()
return study
def find_study(self, records):
study_label = records[0]['study']
for r in records:
if r['study'] != study_label:
m = 'all records should have the same study label'
self.logger.critical(m)
raise ValueError(m)
return self.get_study(study_label)
def find_klass(self, col_name, records):
o_type = records[0][col_name]
for r in records:
if r[col_name] != o_type:
m = 'all records should have the same %s' % col_name
self.logger.critical(m)
raise ValueError(m)
return getattr(self.kb, o_type)
def __preload_items__(self, key_field, klass, preloaded):
objs = self.kb.get_objects(klass)
for o in objs:
assert not getattr(o, key_field) in preloaded
preloaded[getattr(o, key_field)] = o
def preload_by_type(self, name, klass, preloaded):
self.logger.info('start preloading %s' % name)
self.__preload_items__('id', klass, preloaded)
self.logger.info('done preloading %s' % name)
def preload_studies(self, preloaded):
self.logger.info('start preloading studies')
self.__preload_items__('label', self.kb.Study, preloaded)
self.logger.info('done preloading studies')
def missing_fields(self, fields, r):
for f in fields:
if f not in r:
return f
return False
class Core(object):
def __init__(self,
host=None,
user=None,
passwd=None,
group=None,
keep_tokens=1,
study_label=None,
logger=None):
self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens)
self.logger = logger if logger else logging.getLogger()
self.record_counter = 0
self.default_study = None
if study_label:
s = self.kb.get_study(study_label)
if not s:
raise ValueError('No known study with label %s' % study_label)
self.logger.info('Selecting %s[%d,%s] as default study' %
(s.label, s.omero_id, s.id))
self.default_study = s
@classmethod
def find_action_setup_conf(klass, args):
action_setup_conf = {}
for x in dir(args):
if not (x.startswith('_') or x.startswith('func')):
action_setup_conf[x] = getattr(args, x)
# HACKS
action_setup_conf['ifile'] = action_setup_conf['ifile'].name
action_setup_conf['ofile'] = action_setup_conf['ofile'].name
action_setup_conf['report_file'] = action_setup_conf[
'report_file'].name
return action_setup_conf
@classmethod
def get_action_setup_options(klass,
record,
action_setup_conf=None,
object_history=None):
options = {}
if 'options' in record and record['options']:
kvs = record['options'].split(',')
for kv in kvs:
k, v = kv.split('=')
options[k] = v
if action_setup_conf:
options['importer_setup'] = action_setup_conf
if object_history:
options['object_history'] = object_history
return json.dumps(options)
def get_device(self, label, maker, model, release):
device = self.kb.get_device(label)
if not device:
self.logger.debug('creating a device')
device = self.kb.create_device(label, maker, model, release)
return device
def get_action_setup(self, label, conf):
"""
Return the ActionSetup corresponding to label if there is one,
else create a new one using conf.
"""
asetup = self.kb.get_action_setup(label)
if not asetup:
kb_conf = {
'label': label,
'conf': json.dumps(conf),
}
asetup = self.kb.factory.create(self.kb.ActionSetup,
kb_conf).save()
return asetup
def get_study(self, label):
if self.default_study:
return self.default_study
study = self.kb.get_study(label)
if not study:
study = self.kb.factory.create(self.kb.Study, {
'label': label
}).save()
return study
def find_study(self, records):
study_label = records[0]['study']
for r in records:
if r['study'] != study_label:
m = 'all records should have the same study label'
self.logger.critical(m)
raise ValueError(m)
return self.get_study(study_label)
def find_klass(self, col_name, records):
o_type = records[0][col_name]
for r in records:
if r[col_name] != o_type:
m = 'all records should have the same %s' % col_name
self.logger.critical(m)
raise ValueError(m)
return getattr(self.kb, o_type)
def __preload_items__(self, key_field, klass, preloaded):
objs = self.kb.get_objects(klass)
for o in objs:
assert not getattr(o, key_field) in preloaded
preloaded[getattr(o, key_field)] = o
def preload_by_type(self, name, klass, preloaded):
self.logger.info('start preloading %s' % name)
self.__preload_items__('id', klass, preloaded)
self.logger.info('done preloading %s' % name)
def preload_studies(self, preloaded):
self.logger.info('start preloading studies')
self.__preload_items__('label', self.kb.Study, preloaded)
self.logger.info('done preloading studies')
def missing_fields(self, fields, r):
for f in fields:
if f not in r:
return f
return False
class App(object):
def __init__(self, host, user, passwd, study_label, maker, model, release):
self.kb = KB(driver='omero')(host, user, passwd)
self.mset = self.kb.get_snp_markers_set(maker, model, release)
self.logger = logging.getLogger()
if not self.mset:
raise ValueError('SNPMarkersSet[%s,%s,%s] has not been defined.' %
(maker, model, release))
#--
alabel = 'load_genotypes-setup-%s' % time.time()
self.asetup = self.kb.factory.create(self.kb.ActionSetup, {
'label': alabel,
'conf': ''
}).save()
#--
dmaker, dmodel, drelease = 'CRS4', 'load_genotypes', '0.1'
dlabel = '%s-%s-%s' % (dmaker, dmodel, drelease)
device = self.kb.get_device(dlabel)
if not device:
device = self.kb.factory.create(
self.kb.Device, {
'label': dlabel,
'maker': dmaker,
'model': dmodel,
'release': drelease
}).save()
self.device = device
#-- FIXME this will break if study is not defined.
self.study = self.kb.get_study(study_label)
def check_snp_markers_set(self, marker_types, marker_names):
self.logger.info('start checking snp_markers_set')
mdefs, msetc = self.kb.get_snp_markers_set_content(self.mset)
rs_labels = mdefs['rs_label']
for t, n in it.izip(marker_types, marker_names):
if t == 'M':
if not n in rs_labels:
msg = 'marker %s is not in the specified SNPMarkersSet' % n
self.logger.critical(msg)
raise ValueError(msg)
self.logger.info('done checking snp_markers_set')
def create_action(self, target):
conf = {
'setup': self.asetup,
'device': self.device,
'actionCategory': self.kb.ActionCategory.MEASUREMENT,
'operator': 'Alfred E. Neumann',
'context': self.study,
'target': target,
}
action = self.kb.factory.create(self.kb.ActionOnVessel, conf).save()
return action
def create_data_sample(self, action, label):
conf = {
'snpMarkersSet': self.mset,
'label': label,
'status': self.kb.DataSampleStatus.USABLE,
'action': action
}
return self.kb.factory.create(self.kb.GenotypeDataSample, conf).save()
def load(self, pedfile, datfile, conf_value=1.0):
pr = PedReader(pedfile, datfile, conf_value)
self.check_snp_markers_set(pr.marker_types, pr.marker_names)
#--
self.logger.info('start loading from pedfile %s' % pedfile.name)
for x in pr:
sample = self.kb.get_vessel(x['sample_label'])
if not sample:
self.logger.error('No sample with label %s in VL' %
x['sample_label'])
continue
action = self.create_action(sample)
avid = action.id
action.unload()
data_sample = self.create_data_sample(action, x['label'])
data_object = self.kb.add_gdo_data_object(avid, data_sample,
x['probs'], x['confs'])
self.logger.info('-- loaded %s' % x['label'])
self.logger.info('done loading from pedfile %s' % pedfile.name)
