class VidMapper(object):
def __init__(self, host, user, passwd, logger, study):
self.logger = logger
self.study = study
self.kb = KB(driver="omero")(host, user, passwd)
plates = self.kb.get_objects(self.kb.TiterPlate)
self.logger.info("fetched %d plates" % len(plates))
self.plate_map = {}
self.enroll_map = {}
for p in plates:
self.plate_map[p.omero_id] = p.barcode
s = self.kb.get_study(self.study)
enrolls = self.kb.get_enrolled(s)
self.logger.info("fetched %d enrollments" % len(enrolls))
for e in enrolls:
self.logger.debug('Retrieving wells for %s' % e.studyCode)
wells = [
w for w in self.kb.get_vessels_by_individual(
e.individual, "PlateWell")
]
self.enroll_map[e.studyCode] = wells
def map_vid(self, r):
en_code = r["source"]
pl_barcode = en_code.split("|")[1]
try:
wells = self.enroll_map[en_code]
except KeyError:
msg = "%s is not enrolled in %s" % (en_code, self.study)
self.logger.error(msg)
raise ValueError(msg)
self.logger.info("found %d wells for %s" % (len(wells), en_code))
imm_wells = [
w for w in wells
if self.plate_map[w.container.omero_id] == pl_barcode
]
if len(imm_wells) > 1:
msg = ("more than 1 (%d) immuno wells for plate %s" %
(len(imm_wells), pl_barcode))
self.logger.error(msg)
raise ValueError(msg)
elif len(imm_wells) == 0:
msg = "no immuno well for plate %s" % pl_barcode
self.logger.warn(msg)
raise ValueError(msg)
else:
r["source"] = imm_wells[0].id
class App(object):
def __init__(self, host, user, passwd):
self.kb = KB(driver='omero')(host, user, passwd)
#FIXME we need to do this to sync with the DB idea of the enums.
self.kb.Gender.map_enums_values(self.kb)
self.logger = logging.getLogger()
def get_ehr_iterator(self):
inds = self.kb.get_objects(self.kb.Individual)
inds_by_vid = dict([(i.id, i) for i in inds])
for e in self.kb.get_ehr_iterator():
if not e[0] in inds_by_vid:
#FIXME we need to do this for potential stray records left by testing
continue
yield (inds_by_vid[e[0]], e[1])
def count_males(self, inds):
return sum([i.gender == self.kb.Gender.MALE for i in inds])
def do_enrollment(self):
controls = []
affected = []
# FIXME in a future version, we will be able to issue actual AQL
# queries.
for i, ehr in self.get_ehr_iterator():
if ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1):
affected.append(i)
elif ehr.matches(EXCLUSION): # FIXME we assume that this is enough
controls.append(i)
# A more sophisticated example
# will keep only records where age of onset is between 10y and 15y
# for i, ehr in self.get_ehr_iterator():
# if (ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1)
# and
# ehr.matches(DIAGNOSIS, DIAGNOSIS_AGE_OF_ONSET, ("10y", "15y")):
# affected.append(i)
print ('there are %d controls [%d male]'
% (len(controls), self.count_males(controls)))
print ('there are %d affected [%d male] with E10'
% (len(affected), self.count_males(affected)))
class VidMapper(object):
def __init__(self, host, user, passwd, logger, study):
self.logger = logger
self.study = study
self.kb = KB(driver="omero")(host, user, passwd)
plates = self.kb.get_objects(self.kb.TiterPlate)
self.logger.info("fetched %d plates" % len(plates))
self.plate_map = {}
self.enroll_map = {}
for p in plates:
self.plate_map[p.omero_id] = p.barcode
s = self.kb.get_study(self.study)
enrolls = self.kb.get_enrolled(s)
self.logger.info("fetched %d enrollments" % len(enrolls))
for e in enrolls:
self.logger.debug("Retrieving wells for %s" % e.studyCode)
wells = [w for w in self.kb.get_vessels_by_individual(e.individual, "PlateWell")]
self.enroll_map[e.studyCode] = wells
def map_vid(self, r):
en_code = r["source"]
pl_barcode = en_code.split("|")[1]
try:
wells = self.enroll_map[en_code]
except KeyError:
msg = "%s is not enrolled in %s" % (en_code, self.study)
self.logger.error(msg)
raise ValueError(msg)
self.logger.info("found %d wells for %s" % (len(wells), en_code))
imm_wells = [w for w in wells if self.plate_map[w.container.omero_id] == pl_barcode]
if len(imm_wells) > 1:
msg = "more than 1 (%d) immuno wells for plate %s" % (len(imm_wells), pl_barcode)
self.logger.error(msg)
raise ValueError(msg)
elif len(imm_wells) == 0:
msg = "no immuno well for plate %s" % pl_barcode
self.logger.warn(msg)
raise ValueError(msg)
else:
r["source"] = imm_wells[0].id
class App(object):
def __init__(self, host, user, passwd):
self.kb = KB(driver="omero")(host, user, passwd)
# FIXME we need to do this to sync with the DB idea of the enums.
self.kb.Gender.map_enums_values(self.kb)
self.logger = logging.getLogger()
def get_ehr_iterator(self):
inds = self.kb.get_objects(self.kb.Individual)
inds_by_vid = dict([(i.id, i) for i in inds])
for e in self.kb.get_ehr_iterator():
if not e[0] in inds_by_vid:
# FIXME we need to do this for potential stray records left by testing
continue
yield (inds_by_vid[e[0]], e[1])
def count_males(self, inds):
return sum([i.gender == self.kb.Gender.MALE for i in inds])
def do_enrollment(self):
controls = []
affected = []
# FIXME in a future version, we will be able to issue actual AQL
# queries.
for i, ehr in self.get_ehr_iterator():
if ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1):
affected.append(i)
elif ehr.matches(EXCLUSION): # FIXME we assume that this is enough
controls.append(i)
# A more sophisticated example
# will keep only records where age of onset is between 10y and 15y
# for i, ehr in self.get_ehr_iterator():
# if (ehr.matches(DIAGNOSIS, DIAGNOSIS_TERM, DIABETES_TYPE_1)
# and
# ehr.matches(DIAGNOSIS, DIAGNOSIS_AGE_OF_ONSET, ("10y", "15y")):
# affected.append(i)
print("there are %d controls [%d male]" % (len(controls), self.count_males(controls)))
print("there are %d affected [%d male] with E10" % (len(affected), self.count_males(affected)))
def main():
parser = make_parser()
args = parser.parse_args()
kb = KB(driver='omero')(args.host, args.user, args.passwd)
by_label = dict(((x.label, x) for x in kb.get_objects(kb.GenotypeDataSample)))
msets = {}
itsv = csv.DictReader(args.data_samples, delimiter='\t')
otsv = csv.DictWriter(open(args., mode='w'),
fieldnames=['path', 'data_sample_label', 'mimetype',
'size', 'sha1'],
delimiter='\t')
otsv.writeheader()
for r in itsv:
ds_label = r['label']
logger.info('Gathering info on %s' % ds_label)
if ds_label not in by_label:
logger.critical('There is no GenotypeDataSample with label %s' % label)
sys.exit(1)
ds = by_label[ds_label]
# FIXME
if ds.spnMarkersSet.omero_id not in msets:
ms = ds.snpMarkersSet
ms.load_markers()
msets[ds.snpMarkersSet.omero_id] = ms
ms = msets[ds.snpMarkersSet.omero_id]
fname = ds_label + '_do.ssc'
make_data_object(ds.id, fname, ms)
size = os.stat(fname).st_size
sha1 = compute_sha1(fname)
otsv.writerow({
'path' : 'file://' + os.path.realpath(fname),
'data_sample' : ds.id,
'mimetype' : mimetypes.SSC_FILE,
'size' : size,
'sha1' : sha1,
})
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
log_level = getattr(logging, args.loglevel)
kwargs = {'format' : LOG_FORMAT,
'datefmt' : LOG_DATEFMT,
'level' : log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
kb = KB(driver='omero')(args.host, args.user, args.passwd)
logger.info('Loading GenotypeDataSample objects')
dsamples = kb.get_objects(kb.GenotypeDataSample)
logger.info('Loaded %d objects' % len(dsamples))
logger.info('Loading SNPMarkersSet')
query = 'SELECT snpm FROM SNPMarkersSet snpm WHERE snpm.label = :mset_label'
mset = kb.find_all_by_query(query, {'mset_label' : args.marker_set})[0]
if not mset:
logger.error('Unable to load SNPMarkersSet with label %s' % args.marker_set)
sys.exit(2)
else:
logger.info('Object loaded')
gdo_iterator = kb.get_gdo_iterator(mset, dsamples[:args.fetch_size])
gdos = []
logger.info('Loading GDOs')
for gdo in gdo_iterator:
logger.info(gdo['vid'])
gdos.append(gdo)
logger.debug('%d/%d GDOs loaded' % (len(gdos), args.fetch_size))
logger.info('Loaded %d GDOs' % len(gdos))
def main(argv):
parser = make_parser()
args = parser.parse_args(argv)
log_level = getattr(logging, args.loglevel)
kwargs = {'format': LOG_FORMAT, 'datefmt': LOG_DATEFMT, 'level': log_level}
if args.logfile:
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
kb = KB(driver='omero')(args.host, args.user, args.passwd)
logger.info('Loading GenotypeDataSample objects')
dsamples = kb.get_objects(kb.GenotypeDataSample)
logger.info('Loaded %d objects' % len(dsamples))
logger.info('Loading SNPMarkersSet')
query = 'SELECT snpm FROM SNPMarkersSet snpm WHERE snpm.label = :mset_label'
mset = kb.find_all_by_query(query, {'mset_label': args.marker_set})[0]
if not mset:
logger.error('Unable to load SNPMarkersSet with label %s' %
args.marker_set)
sys.exit(2)
else:
logger.info('Object loaded')
gdo_iterator = kb.get_gdo_iterator(mset, dsamples[:args.fetch_size])
gdos = []
logger.info('Loading GDOs')
for gdo in gdo_iterator:
logger.info(gdo['vid'])
gdos.append(gdo)
logger.debug('%d/%d GDOs loaded' % (len(gdos), args.fetch_size))
logger.info('Loaded %d GDOs' % len(gdos))
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Retrieving diagnosis records')
ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
DIAGNOSIS_ARCHETYPE)
ehr_records.extend(
kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
EXCL_DIAG_ARCHETYPE))
logger.info('%d records retrieved' % len(ehr_records))
logger.info('Loading actions')
actions = kb.get_objects(kb.Action)
act_map = {}
for act in actions:
act_map[act.id] = act.context.label
logger.info('%d actions loaded' % len(act_map))
with open(args.ofile, 'w') as out_file:
writer = csv.DictWriter(
out_file, ['study', 'individual', 'timestamp', 'diagnosis'],
delimiter='\t')
writer.writeheader()
for rec in ehr_records:
writer.writerow(extract_csv_row(rec, act_map))
logger.info('Job completed')
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Retrieving diagnosis records')
ehr_records = kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
DIAGNOSIS_ARCHETYPE)
ehr_records.extend(kb.get_ehr_records('(archetype == "%s") & (valid == True)' %
EXCL_DIAG_ARCHETYPE))
logger.info('%d records retrieved' % len(ehr_records))
logger.info('Loading actions')
actions = kb.get_objects(kb.Action)
act_map = {}
for act in actions:
act_map[act.id] = act.context.label
logger.info('%d actions loaded' % len(act_map))
with open(args.ofile, 'w') as out_file:
writer = csv.DictWriter(out_file, ['study', 'individual',
'timestamp', 'diagnosis'],
delimiter='\t')
writer.writeheader()
for rec in ehr_records:
writer.writerow(extract_csv_row(rec, act_map))
logger.info('Job completed')
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
enrolls_map = get_enrollments_lookup(kb, logger)
logger.info('Loading Individuals')
inds = kb.get_objects(kb.Individual)
logger.info('Loaded %d Individuals' % len(inds))
with open(args.in_file) as in_file, open(args.out_file, 'w') as out_file:
ts = long(time.time())
reader = csv.DictReader(in_file, delimiter='\t')
writer = csv.DictWriter(out_file, ['study', 'individual', 'timestamp',
'birth_date', 'birth_place',
'birth_place_district'],
delimiter='\t')
writer.writeheader()
for row in reader:
try:
writer.writerow({'study' : row['individual'].split(':')[0],
'individual' : enrolls_map[row['individual']].individual.id,
'timestamp' : ts,
class UniverseApplication(OrigUniverseApplication):
def __init__(self, **kwargs):
omero_host = vlu.ome_host()
omero_user = vlu.ome_user()
omero_passwd = vlu.ome_passwd()
self.kb = KB(driver='omero')(omero_host, omero_user, omero_passwd)
super(UniverseApplication, self).__init__(**kwargs)
self.config.omero_default_host = kwargs.get('omero_default_host')
self.config.omero_default_user = kwargs.get('omero_default_user')
self.config.omero_default_passwd = kwargs.get('omero_default_passwd')
self.config.vl_loglevel = kwargs.get('vl_loglevel', 'INFO')
self.config.vl_import_enabled_users = kwargs.get('vl_import_enabled_users')
@property
def known_studies(self):
studies = self.kb.get_objects(self.kb.Study)
if studies:
return [(s.label, s.description or '') for s in studies]
else:
return []
@property
def known_scanners(self):
scanners = self.kb.get_objects(self.kb.Scanner)
if scanners:
return [(s.id, s.label, s.physicalLocation) for s in scanners]
else:
return []
@property
def known_map_vid_source_types(self):
from bl.vl.app.kb_query.map_vid import MapVIDApp
sources = MapVIDApp.SUPPORTED_SOURCE_TYPES
if sources:
return sources
else:
return []
@property
def known_marker_sets(self):
msets = self.kb.get_objects(self.kb.SNPMarkersSet)
if msets:
return [(ms.id, ms.label) for ms in msets]
else:
return []
@property
def known_data_collections(self):
dcolls = self.kb.get_objects(self.kb.DataCollection)
if dcolls:
return [(dc.id, dc.label) for dc in dcolls]
else:
return []
@property
def known_vessels_collections(self):
vcolls = self.kb.get_objects(self.kb.VesselsCollection)
if vcolls:
return [(vc.id, vc.label) for vc in vcolls]
else:
return []
@property
def known_titer_plates(self):
plates = self.kb.get_objects(self.kb.TiterPlate)
if plates:
return [(pl.barcode, pl.label) for pl in plates if pl.barcode]
else:
return []
@property
def known_vessel_status(self):
vstatus = self.kb.get_objects(self.kb.VesselStatus)
if vstatus:
return [(v.omero_id, v.enum_label()) for v in vstatus]
else:
return []
@property
def known_data_sample_status(self):
dsstatus = self.kb.get_objects(self.kb.DataSampleStatus)
if dsstatus:
return [(ds.omero_id, ds.enum_label()) for ds in dsstatus]
else:
return []
@property
def known_hardware_devices(self):
hdev = self.kb.get_objects(self.kb.HardwareDevice)
if hdev:
return [(h.id, h.label) for h in hdev]
else:
return []
@property
def known_software_devices(self):
sdev = self.kb.get_objects(self.kb.SoftwareDevice)
if sdev:
return [(s.id, s.label) for s in sdev]
else:
return []
@property
def known_devices(self):
dev = self.kb.get_objects(self.kb.Device)
if dev:
return [(d.id, d.label) for d in dev]
else:
return []
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
enrolls_map = get_enrollments_lookup(kb, logger)
logger.info('Loading Individuals')
inds = kb.get_objects(kb.Individual)
logger.info('Loaded %d Individuals' % len(inds))
with open(args.in_file) as in_file, open(args.out_file, 'w') as out_file:
ts = long(time.time())
reader = csv.DictReader(in_file, delimiter='\t')
writer = csv.DictWriter(out_file, [
'study', 'individual', 'timestamp', 'birth_date', 'birth_place',
'birth_place_district'
],
delimiter='\t')
writer.writeheader()
for row in reader:
try:
writer.writerow({
'study':
class Core(object):
def __init__(self,
host=None,
user=None,
passwd=None,
group=None,
keep_tokens=1,
study_label=None,
logger=None):
self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens)
self.logger = logger if logger else logging.getLogger()
self.record_counter = 0
self.default_study = None
if study_label:
s = self.kb.get_study(study_label)
if not s:
raise ValueError('No known study with label %s' % study_label)
self.logger.info('Selecting %s[%d,%s] as default study' %
(s.label, s.omero_id, s.id))
self.default_study = s
@classmethod
def find_action_setup_conf(klass, args):
action_setup_conf = {}
for x in dir(args):
if not (x.startswith('_') or x.startswith('func')):
action_setup_conf[x] = getattr(args, x)
# HACKS
action_setup_conf['ifile'] = action_setup_conf['ifile'].name
action_setup_conf['ofile'] = action_setup_conf['ofile'].name
action_setup_conf['report_file'] = action_setup_conf[
'report_file'].name
return action_setup_conf
@classmethod
def get_action_setup_options(klass,
record,
action_setup_conf=None,
object_history=None):
options = {}
if 'options' in record and record['options']:
kvs = record['options'].split(',')
for kv in kvs:
k, v = kv.split('=')
options[k] = v
if action_setup_conf:
options['importer_setup'] = action_setup_conf
if object_history:
options['object_history'] = object_history
return json.dumps(options)
def get_device(self, label, maker, model, release):
device = self.kb.get_device(label)
if not device:
self.logger.debug('creating a device')
device = self.kb.create_device(label, maker, model, release)
return device
def get_action_setup(self, label, conf):
"""
Return the ActionSetup corresponding to label if there is one,
else create a new one using conf.
"""
asetup = self.kb.get_action_setup(label)
if not asetup:
kb_conf = {
'label': label,
'conf': json.dumps(conf),
}
asetup = self.kb.factory.create(self.kb.ActionSetup,
kb_conf).save()
return asetup
def get_study(self, label):
if self.default_study:
return self.default_study
study = self.kb.get_study(label)
if not study:
study = self.kb.factory.create(self.kb.Study, {
'label': label
}).save()
return study
def find_study(self, records):
study_label = records[0]['study']
for r in records:
if r['study'] != study_label:
m = 'all records should have the same study label'
self.logger.critical(m)
raise ValueError(m)
return self.get_study(study_label)
def find_klass(self, col_name, records):
o_type = records[0][col_name]
for r in records:
if r[col_name] != o_type:
m = 'all records should have the same %s' % col_name
self.logger.critical(m)
raise ValueError(m)
return getattr(self.kb, o_type)
def __preload_items__(self, key_field, klass, preloaded):
objs = self.kb.get_objects(klass)
for o in objs:
assert not getattr(o, key_field) in preloaded
preloaded[getattr(o, key_field)] = o
def preload_by_type(self, name, klass, preloaded):
self.logger.info('start preloading %s' % name)
self.__preload_items__('id', klass, preloaded)
self.logger.info('done preloading %s' % name)
def preload_studies(self, preloaded):
self.logger.info('start preloading studies')
self.__preload_items__('label', self.kb.Study, preloaded)
self.logger.info('done preloading studies')
def missing_fields(self, fields, r):
for f in fields:
if f not in r:
return f
return False
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Loading Demographic data')
demogs = kb.get_objects(kb.Demographic)
logger.info('Loaded %d items' % len(demogs))
logger.info('Loading Individuals')
inds = kb.get_objects(kb.Individual)
logger.info('Loaded %d individuals' % len(inds))
demogs_lookup = {}
for d in demogs:
demogs_lookup[d.individual] = d
enrolls_map = get_enrollments_lookup(kb, logger)
in_file = open(args.in_file)
reader = csv.DictReader(in_file, delimiter='\t')
bd_matches_file = open(args.match_files, 'w')
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Start preloading dependency tree')
kb.update_dependency_tree()
logger.info('Dependency tree loaded')
with open(args.out_file, 'w') as ofile:
writer = csv.DictWriter(
ofile, ['target', 'new_source_type', 'new_source', 'old_source'],
delimiter='\t')
writer.writeheader()
logger.info('Loading wells data')
for w in kb.get_objects(kb.PlateWell):
if type(w.action.target) == kb.Individual:
tubes = list(kb.dt.get_connected(w, kb.Tube))
if len(tubes) == 1:
record = {
'target':
'%s:%s' % (w.container.label, w.label),
'new_source_type':
tubes[0].__class__.__name__,
'new_source':
tubes[0].label,
'old_source':
'%s:%s' % (w.action.target.__class__.__name__,
w.action.target.id)
}
writer.writerow(record)
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Start preloading dependency tree')
kb.update_dependency_tree()
logger.info('Dependency tree loaded')
with open(args.out_file, 'w') as ofile:
writer = csv.DictWriter(ofile, ['target', 'new_source_type',
'new_source', 'old_source'],
delimiter='\t')
writer.writeheader()
logger.info('Loading wells data')
for w in kb.get_objects(kb.PlateWell):
if type(w.action.target) == kb.Individual:
tubes = list(kb.dt.get_connected(w, kb.Tube))
if len(tubes) == 1:
record = {'target' : '%s:%s' % (w.container.label,
w.label),
'new_source_type' : tubes[0].__class__.__name__,
'new_source' : tubes[0].label,
'old_source' : '%s:%s' % (w.action.target.__class__.__name__,
w.action.target.id)}
writer.writerow(record)
else:
logger.debug('Well %s:%s has %d tubes connected' % (w.container.label,
w.label, len(tubes)))
else:
logger.debug('Well %s:%s source is %s' % (w.container.label,
class Core(object):
def __init__(self, host=None, user=None, passwd=None, group=None,
keep_tokens=1, study_label=None, logger=None):
self.kb = KB(driver='omero')(host, user, passwd, group, keep_tokens)
self.logger = logger if logger else logging.getLogger()
self.record_counter = 0
self.default_study = None
if study_label:
s = self.kb.get_study(study_label)
if not s:
raise ValueError('No known study with label %s' % study_label)
self.logger.info('Selecting %s[%d,%s] as default study' %
(s.label, s.omero_id, s.id))
self.default_study = s
@classmethod
def find_action_setup_conf(klass, args):
action_setup_conf = {}
for x in dir(args):
if not (x.startswith('_') or x.startswith('func')):
action_setup_conf[x] = getattr(args, x)
# HACKS
action_setup_conf['ifile'] = action_setup_conf['ifile'].name
action_setup_conf['ofile'] = action_setup_conf['ofile'].name
action_setup_conf['report_file'] = action_setup_conf['report_file'].name
return action_setup_conf
@classmethod
def get_action_setup_options(klass, record, action_setup_conf = None,
object_history = None):
options = {}
if 'options' in record and record['options']:
kvs = record['options'].split(',')
for kv in kvs:
k, v = kv.split('=')
options[k] = v
if action_setup_conf:
options['importer_setup'] = action_setup_conf
if object_history:
options['object_history'] = object_history
return json.dumps(options)
def get_device(self, label, maker, model, release):
device = self.kb.get_device(label)
if not device:
self.logger.debug('creating a device')
device = self.kb.create_device(label, maker, model, release)
return device
def get_action_setup(self, label, conf):
"""
Return the ActionSetup corresponding to label if there is one,
else create a new one using conf.
"""
asetup = self.kb.get_action_setup(label)
if not asetup:
kb_conf = {
'label': label,
'conf': json.dumps(conf),
}
asetup = self.kb.factory.create(self.kb.ActionSetup, kb_conf).save()
return asetup
def get_study(self, label):
if self.default_study:
return self.default_study
study = self.kb.get_study(label)
if not study:
study = self.kb.factory.create(self.kb.Study, {'label': label}).save()
return study
def find_study(self, records):
study_label = records[0]['study']
for r in records:
if r['study'] != study_label:
m = 'all records should have the same study label'
self.logger.critical(m)
raise ValueError(m)
return self.get_study(study_label)
def find_klass(self, col_name, records):
o_type = records[0][col_name]
for r in records:
if r[col_name] != o_type:
m = 'all records should have the same %s' % col_name
self.logger.critical(m)
raise ValueError(m)
return getattr(self.kb, o_type)
def __preload_items__(self, key_field, klass, preloaded):
objs = self.kb.get_objects(klass)
for o in objs:
assert not getattr(o, key_field) in preloaded
preloaded[getattr(o, key_field)] = o
def preload_by_type(self, name, klass, preloaded):
self.logger.info('start preloading %s' % name)
self.__preload_items__('id', klass, preloaded)
self.logger.info('done preloading %s' % name)
def preload_studies(self, preloaded):
self.logger.info('start preloading studies')
self.__preload_items__('label', self.kb.Study, preloaded)
self.logger.info('done preloading studies')
def missing_fields(self, fields, r):
for f in fields:
if f not in r:
return f
return False
kwargs['filename'] = args.logfile
logging.basicConfig(**kwargs)
logger = logging.getLogger()
try:
host = args.host or vlu.ome_host()
user = args.user or vlu.ome_user()
passwd = args.passwd or vlu.ome_passwd()
except ValueError, ve:
logger.critical(ve)
sys.exit(ve)
kb = KB(driver='omero')(host, user, passwd)
logger.info('Loading Demographic data')
demogs = kb.get_objects(kb.Demographic)
logger.info('Loaded %d items' % len(demogs))
logger.info('Loading Individuals')
inds = kb.get_objects(kb.Individual)
logger.info('Loaded %d individuals' % len(inds))
demogs_lookup = {}
for d in demogs:
demogs_lookup[d.individual] = d
enrolls_map = get_enrollments_lookup(kb, logger)
in_file = open(args.in_file)
reader = csv.DictReader(in_file, delimiter='\t')
bd_matches_file = open(args.match_files, 'w')
