def main(argv): global logger parser = make_parser() args = parser.parse_args(argv) logformat = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" loglevel = getattr(logging, args.loglevel) if args.logfile: logging.basicConfig(filename=args.logfile, format=logformat, level=loglevel) else: logging.basicConfig(format=logformat, level=loglevel) logger = logging.getLogger() abi_service = ABISnpService() found_markers = {} data = {} min_datetime = datetime.max #datetime(2999, 12, 31, 23, 59, 59) max_datetime = datetime.min #datetime(1000, 1, 1, 0, 0, 1) for f in (_.strip() for _ in args.ifile if _.strip()): logger.info('processing %s' % f) sds = SDSReader(open(f), swap_sample_well_columns=True) min_datetime = min(min_datetime, sds.datetime) max_datetime = max(max_datetime, sds.datetime) get_markers_definition(found_markers, sds, abi_service) for r in sds: data.setdefault(r['Sample Name'], []).append(r) kb = KB(driver='omero')(args.host, args.user, args.passwd) logger.info('qui - main') missing_kb_markers = add_kb_marker_objects(kb, found_markers) if missing_kb_markers: logger.info('there are missing markers. Cannot proceed further.') fname = '%smarker-defs.tsv' % args.prefix write_import_markers_file(fname, found_markers, missing_kb_markers) logger.info('the list of the missing marker is in %s.' % fname) sys.exit(0) fname = '%smarkers-set-def.tsv' % args.prefix write_markers_set_def_file(fname, found_markers) ssc_data_set = {} device = kb.get_device(args.device_label) for sample_id, d in data.iteritems(): fname = '%s%s-%s.ssc' % (args.prefix, device.id, sample_id) write_ssc_data_set_file(fname, found_markers, device.id, sample_id, min_datetime, max_datetime, d) ssc_data_set[sample_id] = ('taqman-%s-%s' % (args.run_id, sample_id), sample_id, device.id, fname) fname = '%simport.ssc' % args.prefix write_ssc_data_samples_import_file(fname, ssc_data_set.values()) write_ssc_data_objects_import_file(fname, ssc_data_set.values())
class App(object): def __init__(self, host, user, passwd, study_label, maker, model, release): self.kb = KB(driver='omero')(host, user, passwd) self.mset = self.kb.get_snp_markers_set(maker, model, release) self.logger = logging.getLogger() if not self.mset: raise ValueError('SNPMarkersSet[%s,%s,%s] has not been defined.' % (maker, model, release)) #-- alabel = 'load_genotypes-setup-%s' % time.time() self.asetup = self.kb.factory.create(self.kb.ActionSetup, {'label' : alabel, 'conf' : ''}).save() #-- dmaker, dmodel, drelease = 'CRS4', 'load_genotypes', '0.1' dlabel = '%s-%s-%s' % (dmaker, dmodel, drelease) device = self.kb.get_device(dlabel) if not device: device = self.kb.factory.create(self.kb.Device, {'label' : dlabel, 'maker' : dmaker, 'model' : dmodel, 'release' : drelease}).save() self.device = device #-- FIXME this will break if study is not defined. self.study = self.kb.get_study(study_label) def check_snp_markers_set(self, marker_types, marker_names): self.logger.info('start checking snp_markers_set') mdefs, msetc = self.kb.get_snp_markers_set_content(self.mset) rs_labels = mdefs['rs_label'] for t, n in it.izip(marker_types, marker_names): if t == 'M': if not n in rs_labels: msg = 'marker %s is not in the specified SNPMarkersSet' % n self.logger.critical(msg) raise ValueError(msg) self.logger.info('done checking snp_markers_set') def create_action(self, target): conf = {'setup' : self.asetup, 'device' : self.device, 'actionCategory' : self.kb.ActionCategory.MEASUREMENT, 'operator' : 'Alfred E. Neumann', 'context' : self.study, 'target' : target, } action = self.kb.factory.create(self.kb.ActionOnVessel, conf).save() return action def create_data_sample(self, action, label): conf = {'snpMarkersSet' : self.mset, 'label' : label, 'status' : self.kb.DataSampleStatus.USABLE, 'action' : action} return self.kb.factory.create(self.kb.GenotypeDataSample, conf).save() def load(self, pedfile, datfile, conf_value=1.0): pr = PedReader(pedfile, datfile, conf_value) self.check_snp_markers_set(pr.marker_types, pr.marker_names) #-- self.logger.info('start loading from pedfile %s' % pedfile.name) for x in pr: sample = self.kb.get_vessel(x['sample_label']) if not sample: self.logger.error('No sample with label %s in VL' % x['sample_label']) continue action = self.create_action(sample) avid = action.id action.unload() data_sample = self.create_data_sample(action, x['label']) data_object = self.kb.add_gdo_data_object(avid, data_sample, x['probs'], x['confs']) self.logger.info('-- loaded %s' % x['label']) self.logger.info('done loading from pedfile %s' % pedfile.name)
class TestKB(KBObjectCreator): def __init__(self, name): super(TestKB, self).__init__(name) self.kill_list = [] def setUp(self): self.kb = KB(driver='omero')(OME_HOST, OME_USER, OME_PASS) def tearDown(self): self.kill_list.reverse() for x in self.kill_list: self.kb.delete(x) self.kill_list = [] def check_object(self, o, conf, otype): try: self.assertTrue(isinstance(o, otype)) for k in conf.keys(): v = conf[k] # FIXME this is omero specific... if hasattr(v, 'ome_obj'): self.assertEqual(getattr(o, k).id, v.id) self.assertEqual(type(getattr(o, k)), type(v)) elif hasattr(v, '_id'): self.assertEqual(getattr(o, k)._id, v._id) else: self.assertEqual(getattr(o, k), v) except: pass def test_study(self): conf, s = self.create_study() self.kill_list.append(s.save()) self.check_object(s, conf, self.kb.Study) def test_study_ops(self): conf, s = self.create_study() s.save() xs = self.kb.get_study(conf['label']) self.assertTrue(not xs is None) self.assertEqual(xs.id, s.id) self.assertEqual(xs.label, s.label) self.kb.delete(s) self.assertEqual(self.kb.get_study(conf['label']), None) def test_device(self): conf, d = self.create_device() self.kill_list.append(d.save()) self.check_object(d, conf, self.kb.Device) def test_hardware_device(self): conf, d = self.create_hardware_device() self.kill_list.append(d.save()) self.check_object(d, conf, self.kb.HardwareDevice) def test_device_ops(self): conf, d = self.create_device() d.save() xs = self.kb.get_device(conf['label']) self.assertTrue(not xs is None) self.check_object(xs, conf, self.kb.Device) self.kb.delete(d) self.assertEqual(self.kb.get_device(conf['label']), None) def test_action_setup(self): conf, a = self.create_action_setup() self.kill_list.append(a.save()) self.check_object(a, conf, self.kb.ActionSetup) def test_action(self): conf, action = self.create_action() self.kill_list.append(action.save()) self.check_object(action, conf, self.kb.Action) def test_action_on_vessel(self): conf, action = self.create_action_on_vessel() self.kill_list.append(action.save()) self.check_object(action, conf, self.kb.ActionOnVessel) def test_action_on_data_sample(self): conf, action = self.create_action_on_data_sample() self.kill_list.append(action.save()) self.check_object(action, conf, self.kb.ActionOnDataSample) def test_action_on_data_collection_item(self): conf, action = self.create_action_on_data_collection_item() self.kill_list.append(action.save()) self.check_object(action, conf, self.kb.ActionOnDataSample)
class Core(object): def __init__(self, host=None, user=None, passwd=None, group=None, keep_tokens=1, study_label=None, logger=None): self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens) self.logger = logger if logger else logging.getLogger() self.record_counter = 0 self.default_study = None if study_label: s = self.kb.get_study(study_label) if not s: raise ValueError('No known study with label %s' % study_label) self.logger.info('Selecting %s[%d,%s] as default study' % (s.label, s.omero_id, s.id)) self.default_study = s @classmethod def find_action_setup_conf(klass, args): action_setup_conf = {} for x in dir(args): if not (x.startswith('_') or x.startswith('func')): action_setup_conf[x] = getattr(args, x) # HACKS action_setup_conf['ifile'] = action_setup_conf['ifile'].name action_setup_conf['ofile'] = action_setup_conf['ofile'].name action_setup_conf['report_file'] = action_setup_conf['report_file'].name return action_setup_conf @classmethod def get_action_setup_options(klass, record, action_setup_conf = None, object_history = None): options = {} if 'options' in record and record['options']: kvs = record['options'].split(',') for kv in kvs: k, v = kv.split('=') options[k] = v if action_setup_conf: options['importer_setup'] = action_setup_conf if object_history: options['object_history'] = object_history return json.dumps(options) def get_device(self, label, maker, model, release): device = self.kb.get_device(label) if not device: self.logger.debug('creating a device') device = self.kb.create_device(label, maker, model, release) return device def get_action_setup(self, label, conf): """ Return the ActionSetup corresponding to label if there is one, else create a new one using conf. """ asetup = self.kb.get_action_setup(label) if not asetup: kb_conf = { 'label': label, 'conf': json.dumps(conf), } asetup = self.kb.factory.create(self.kb.ActionSetup, kb_conf).save() return asetup def get_study(self, label): if self.default_study: return self.default_study study = self.kb.get_study(label) if not study: study = self.kb.factory.create(self.kb.Study, {'label': label}).save() return study def find_study(self, records): study_label = records[0]['study'] for r in records: if r['study'] != study_label: m = 'all records should have the same study label' self.logger.critical(m) raise ValueError(m) return self.get_study(study_label) def find_klass(self, col_name, records): o_type = records[0][col_name] for r in records: if r[col_name] != o_type: m = 'all records should have the same %s' % col_name self.logger.critical(m) raise ValueError(m) return getattr(self.kb, o_type) def __preload_items__(self, key_field, klass, preloaded): objs = self.kb.get_objects(klass) for o in objs: assert not getattr(o, key_field) in preloaded preloaded[getattr(o, key_field)] = o def preload_by_type(self, name, klass, preloaded): self.logger.info('start preloading %s' % name) self.__preload_items__('id', klass, preloaded) self.logger.info('done preloading %s' % name) def preload_studies(self, preloaded): self.logger.info('start preloading studies') self.__preload_items__('label', self.kb.Study, preloaded) self.logger.info('done preloading studies') def missing_fields(self, fields, r): for f in fields: if f not in r: return f return False
class Core(object): def __init__(self, host=None, user=None, passwd=None, group=None, keep_tokens=1, study_label=None, logger=None): self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens) self.logger = logger if logger else logging.getLogger() self.record_counter = 0 self.default_study = None if study_label: s = self.kb.get_study(study_label) if not s: raise ValueError('No known study with label %s' % study_label) self.logger.info('Selecting %s[%d,%s] as default study' % (s.label, s.omero_id, s.id)) self.default_study = s @classmethod def find_action_setup_conf(klass, args): action_setup_conf = {} for x in dir(args): if not (x.startswith('_') or x.startswith('func')): action_setup_conf[x] = getattr(args, x) # HACKS action_setup_conf['ifile'] = action_setup_conf['ifile'].name action_setup_conf['ofile'] = action_setup_conf['ofile'].name action_setup_conf['report_file'] = action_setup_conf[ 'report_file'].name return action_setup_conf @classmethod def get_action_setup_options(klass, record, action_setup_conf=None, object_history=None): options = {} if 'options' in record and record['options']: kvs = record['options'].split(',') for kv in kvs: k, v = kv.split('=') options[k] = v if action_setup_conf: options['importer_setup'] = action_setup_conf if object_history: options['object_history'] = object_history return json.dumps(options) def get_device(self, label, maker, model, release): device = self.kb.get_device(label) if not device: self.logger.debug('creating a device') device = self.kb.create_device(label, maker, model, release) return device def get_action_setup(self, label, conf): """ Return the ActionSetup corresponding to label if there is one, else create a new one using conf. """ asetup = self.kb.get_action_setup(label) if not asetup: kb_conf = { 'label': label, 'conf': json.dumps(conf), } asetup = self.kb.factory.create(self.kb.ActionSetup, kb_conf).save() return asetup def get_study(self, label): if self.default_study: return self.default_study study = self.kb.get_study(label) if not study: study = self.kb.factory.create(self.kb.Study, { 'label': label }).save() return study def find_study(self, records): study_label = records[0]['study'] for r in records: if r['study'] != study_label: m = 'all records should have the same study label' self.logger.critical(m) raise ValueError(m) return self.get_study(study_label) def find_klass(self, col_name, records): o_type = records[0][col_name] for r in records: if r[col_name] != o_type: m = 'all records should have the same %s' % col_name self.logger.critical(m) raise ValueError(m) return getattr(self.kb, o_type) def __preload_items__(self, key_field, klass, preloaded): objs = self.kb.get_objects(klass) for o in objs: assert not getattr(o, key_field) in preloaded preloaded[getattr(o, key_field)] = o def preload_by_type(self, name, klass, preloaded): self.logger.info('start preloading %s' % name) self.__preload_items__('id', klass, preloaded) self.logger.info('done preloading %s' % name) def preload_studies(self, preloaded): self.logger.info('start preloading studies') self.__preload_items__('label', self.kb.Study, preloaded) self.logger.info('done preloading studies') def missing_fields(self, fields, r): for f in fields: if f not in r: return f return False
class App(object): def __init__(self, host, user, passwd, study_label, maker, model, release): self.kb = KB(driver='omero')(host, user, passwd) self.mset = self.kb.get_snp_markers_set(maker, model, release) self.logger = logging.getLogger() if not self.mset: raise ValueError('SNPMarkersSet[%s,%s,%s] has not been defined.' % (maker, model, release)) #-- alabel = 'load_genotypes-setup-%s' % time.time() self.asetup = self.kb.factory.create(self.kb.ActionSetup, { 'label': alabel, 'conf': '' }).save() #-- dmaker, dmodel, drelease = 'CRS4', 'load_genotypes', '0.1' dlabel = '%s-%s-%s' % (dmaker, dmodel, drelease) device = self.kb.get_device(dlabel) if not device: device = self.kb.factory.create( self.kb.Device, { 'label': dlabel, 'maker': dmaker, 'model': dmodel, 'release': drelease }).save() self.device = device #-- FIXME this will break if study is not defined. self.study = self.kb.get_study(study_label) def check_snp_markers_set(self, marker_types, marker_names): self.logger.info('start checking snp_markers_set') mdefs, msetc = self.kb.get_snp_markers_set_content(self.mset) rs_labels = mdefs['rs_label'] for t, n in it.izip(marker_types, marker_names): if t == 'M': if not n in rs_labels: msg = 'marker %s is not in the specified SNPMarkersSet' % n self.logger.critical(msg) raise ValueError(msg) self.logger.info('done checking snp_markers_set') def create_action(self, target): conf = { 'setup': self.asetup, 'device': self.device, 'actionCategory': self.kb.ActionCategory.MEASUREMENT, 'operator': 'Alfred E. Neumann', 'context': self.study, 'target': target, } action = self.kb.factory.create(self.kb.ActionOnVessel, conf).save() return action def create_data_sample(self, action, label): conf = { 'snpMarkersSet': self.mset, 'label': label, 'status': self.kb.DataSampleStatus.USABLE, 'action': action } return self.kb.factory.create(self.kb.GenotypeDataSample, conf).save() def load(self, pedfile, datfile, conf_value=1.0): pr = PedReader(pedfile, datfile, conf_value) self.check_snp_markers_set(pr.marker_types, pr.marker_names) #-- self.logger.info('start loading from pedfile %s' % pedfile.name) for x in pr: sample = self.kb.get_vessel(x['sample_label']) if not sample: self.logger.error('No sample with label %s in VL' % x['sample_label']) continue action = self.create_action(sample) avid = action.id action.unload() data_sample = self.create_data_sample(action, x['label']) data_object = self.kb.add_gdo_data_object(avid, data_sample, x['probs'], x['confs']) self.logger.info('-- loaded %s' % x['label']) self.logger.info('done loading from pedfile %s' % pedfile.name)