def save_Label(cfg, Label, dst_dir=None, db=None):
if len(Label) > 0:
## TODO
# tblname = 'LABELS'
# annonutils.write2db(db, tblname, list(Label.values()))
tblname = annonutils.get_tblname('CLASSINFO')
colors = common.random_colors(len(Label))
class_info = annonutils.get_class_info(Label, colors=colors)
log.info("len(Label): {}".format(len(Label)))
log.info("Label: {}".format(Label))
save_to_file = cfg['SAVE_TO_FILE']
if save_to_file:
lbl_filename = os.path.join(dst_dir,os.path.basename(dst_dir)+'-'+cfg['FILES']['LABELS'])
log.info("lbl_filename: {}".format(lbl_filename))
# db[tblname].insert_many(list(Label.values()))
with open(lbl_filename,'w') as fw:
fw.write(json.dumps(Label))
classinfo_filename = os.path.join(dst_dir,os.path.basename(dst_dir)+'-'+cfg['FILES']['CLASSINFO'])
log.info("classinfo_filename, tblname: {}, {}".format(classinfo_filename, tblname))
with open(classinfo_filename,'w') as fw:
json.dump(class_info,fw)
else:
log.info("tblname: {}".format(tblname))
annonutils.write2db(db, tblname, class_info, idx_col='lbl_id')
def save_Image(cfg, Image, dst_dir=None, db=None):
if len(Image) > 0:
tblname = annonutils.get_tblname('IMAGES')
save_to_file = cfg['SAVE_TO_FILE']
if save_to_file:
img_filename = os.path.join(dst_dir,os.path.basename(dst_dir)+'-'+cfg['FILES']['IMAGES'])
log.info("img_filename, tblname: {}, {}".format(img_filename, tblname))
with open(img_filename,'w') as fw:
fw.write(json.dumps(Image))
else:
log.info("tblname: {}".format(tblname))
annonutils.write2db(db, tblname, list(Image.values()), idx_col='img_id')
def save_Error(cfg, Error, dst_dir=None, db=None):
if len(Error) > 0:
# log.info("Error:\n{}".format(Error))
tblname = annonutils.get_tblname('ERRORS')
save_to_file = cfg['SAVE_TO_FILE']
if save_to_file:
err_filename = os.path.join(dst_dir,os.path.basename(dst_dir)+'-'+cfg['FILES']['ERRORS'])
log.info("err_filename, tblname: {}, {}".format(err_filename, tblname))
# db[tblname].insert_many(list(Error.values()))
with open(err_filename,'w') as fw:
fw.write(json.dumps(Error))
else:
log.info("tblname: {}".format(tblname))
# annonutils.write2db(db, tblname, list(Error.values()), idx_col='rel_filename')
annonutils.write2db(db, tblname, list(Error.values()), idx_col='rel_filepath')
def save_Annotation_Info(cfg, Annotation_Info, dst_dir=None, db=None):
if len(Annotation_Info) > 0:
tblname = annonutils.get_tblname('ANNOTATIONS')
json_str = common.numpy_to_json(Annotation_Info)
# log.info("json_str: {}".format(json_str))
save_to_file = cfg['SAVE_TO_FILE']
if save_to_file:
ant_filename = os.path.join(dst_dir,os.path.basename(dst_dir)+'-'+cfg['FILES']['ANNOTATIONS'])
log.info("ant_filename, tblname: {}, {}".format(ant_filename, tblname))
with open(ant_filename,'w') as fw:
# fw.write(json.dumps(Annotation_Info))
fw.write(json_str)
else:
log.info("tblname: {}".format(tblname))
annonutils.write2db(db, tblname, list(json.loads(json_str).values()), idx_col='ant_id')
def save_Stats(cfg, Stats, Total_Stats, dataset=None, annon_filepath=None, dst_dir=None, db=None):
stats_data = json.loads(common.numpy_to_json(Stats))
total_stats_data = json.loads(common.numpy_to_json(Total_Stats))
stats_total_stats_data = common.merge_dict([stats_data, total_stats_data])
tblname = annonutils.get_tblname('STATS')
save_to_file = cfg['SAVE_TO_FILE']
if save_to_file:
## Stats files
create_stats_files(cfg, Stats, Total_Stats, dst_dir)
## Move processed annotation file to archive folder
log.info("annon_filepath, tblname: {}, {}".format(annon_filepath, tblname))
rel_dir = cfg['BASE_PATH']['RELEASE_DIR']
with open(os.path.join(rel_dir, os.path.basename(annon_filepath)),'w') as fw:
json.dump(dataset,fw)
else:
log.info("tblname: {}".format(tblname))
# annonutils.write2db(db, tblname, [stats_total_stats_data], idx_col='rel_filename')
annonutils.write2db(db, tblname, [stats_total_stats_data], idx_col='rel_filepath')
def create_db(cfg, args, datacfg):
"""release the AIDS database i.e. creates the PXL DB (AI Datasets)
and create respective entries in AIDS table in annon database
"""
log.info("-----------------------------")
by = args.by
db_images, db_annon, latest_release_info, lbl_ids = get_annon_data(cfg)
aids, datacfg = prepare_aids(cfg, db_images, db_annon, lbl_ids, datacfg)
DBCFG = cfg['DBCFG']
mclient = MongoClient('mongodb://'+DBCFG['HOST']+':'+str(DBCFG['PORT']))
rel_timestamp = latest_release_info['timestamp']
DBNAME = 'PXL-'+rel_timestamp+'_'+cfg['TIMESTAMP']
log.info("DBNAME: {}".format(DBNAME))
db = mclient[DBNAME]
uuid_aids = None
if len(aids) > 0:
uuid_aids = common.createUUID('aids')
AIDS_SPLITS_CRITERIA = cfg['AIDS_SPLITS_CRITERIA'][cfg['AIDS_SPLITS_CRITERIA']['USE']]
splits = AIDS_SPLITS_CRITERIA[0] ## directory names
## Save aids - AI Datasets
for split in splits:
for tbl in aids[split]:
log.info("aids[{}][{}]".format(split, tbl))
tblname = annonutils.get_tblname(tbl)
log.info("tblname: {}".format(tblname))
log.info("aids[split][tbl]: {}".format(type(aids[split][tbl])))
if isinstance(aids[split][tbl], dict):
log.info('dict')
data = list(aids[split][tbl].values())
# log.info(aids[split][tbl]['img-19a68326-3468-4b1e-9fc6-5a739523c6f6'])
elif isinstance(aids[split][tbl], list):
log.info('list')
data = aids[split][tbl]
log.info("tblname, type(data): {}, {}".format(tblname, type(data)))
for doc in data:
# if tblname == 'STATS':
# log.info(doc)
doc['dbid'] = uuid_aids
doc['timestamp'] = cfg['TIMESTAMP']
doc['subset'] = split
annonutils.write2db(db, tblname, doc)
created_on = common.now()
uuid_rel = common.createUUID('rel')
datacfg['dbid'] = uuid_aids
datacfg['dbname'] = DBNAME
datacfg['created_on'] = created_on
datacfg['modified_on'] = None
datacfg['anndb_id'] = rel_timestamp
datacfg['timestamp'] = cfg['TIMESTAMP']
datacfg['anndb_rel_id'] = latest_release_info['rel_id']
datacfg['rel_id'] = uuid_rel
datacfg['log_dir'] = DBNAME
datacfg['rel_type'] = 'aids'
datacfg['creator'] = by.upper()
tblname = annonutils.get_tblname('AIDS')
annonutils.create_unique_index(db, tblname, 'created_on')
collection = db.get_collection(tblname)
collection.update_one(
{'created_on': datacfg['created_on']}
,{'$setOnInsert': datacfg}
,upsert=True
)
tblname = annonutils.get_tblname('CLASSINFO')
collection = db.get_collection(tblname)
annonutils.write2db(db, tblname, datacfg['classinfo'], idx_col='lbl_id')
save_to_annon_db(cfg, aidsdata=datacfg)
## TODO:
## generate STATS, STATSLABEL and respective SUMMARY csv files for AIDS
mclient.close()
return uuid_aids
def create_db(cfg, args, datacfg, aids):
"""release the AIDS database i.e. creates the PXL DB (AI Datasets)
and create respective entries in AIDS table in annon database
"""
log.info("-----------------------------")
by = args.by
splits = datacfg['splits']
DBCFG = cfg['DBCFG']
PXLCFG = DBCFG['PXLCFG']
mclient = MongoClient('mongodb://' + PXLCFG['host'] + ':' +
str(PXLCFG['port']))
dbname = 'PXL-' + cfg['TIMESTAMP']
log.info("dbname: {}".format(dbname))
db = mclient[dbname]
uuid_aids = None
if len(aids) > 0:
uuid_aids = common.createUUID('aids')
## Save aids - AI Datasets
for split in splits:
for tbl in aids[split]:
# log.info("aids[{}][{}]".format(split, tbl))
log.info("split: {}".format(split))
if aids[split][tbl] is not None:
tblname = annonutils.get_tblname(tbl)
log.info("tblname: {}".format(tblname))
log.info("aids[split][tbl]: {}".format(
type(aids[split][tbl])))
if isinstance(aids[split][tbl], dict):
log.info('dict')
data = list(aids[split][tbl].values())
# log.info(aids[split][tbl]['img-19a68326-3468-4b1e-9fc6-5a739523c6f6'])
elif isinstance(aids[split][tbl], list):
log.info('list')
data = aids[split][tbl]
log.info(
"tblname, type(data), len(data): {}, {}, {}".format(
tblname, type(data), len(data)))
if len(data) > 0:
for doc in data:
doc['dbid'] = uuid_aids
doc['timestamp'] = cfg['TIMESTAMP']
doc['subset'] = split
if tblname == 'STATS':
log.info('doc: {}'.format(doc))
# log.debug('doc: {}'.format(doc))
annonutils.write2db(db, tblname, doc)
created_on = common.now()
uuid_rel = common.createUUID('rel')
datacfg['dbid'] = uuid_aids
datacfg['dbname'] = dbname
datacfg['created_on'] = created_on
datacfg['modified_on'] = None
datacfg['anndb_id'] = dbname
datacfg['timestamp'] = cfg['TIMESTAMP']
datacfg['anndb_rel_id'] = None
datacfg['rel_id'] = uuid_rel
datacfg['log_dir'] = dbname
datacfg['rel_type'] = 'aids'
datacfg['creator'] = by.upper()
log.info("datacfg: {}".format(datacfg))
tblname = annonutils.get_tblname('AIDS')
annonutils.create_unique_index(db, tblname, 'created_on')
collection = db.get_collection(tblname)
collection.update_one({'created_on': datacfg['created_on']},
{'$setOnInsert': datacfg},
upsert=True)
tblname = annonutils.get_tblname('CLASSINFO')
collection = db.get_collection(tblname)
annonutils.write2db(db,
tblname,
datacfg['classinfo'],
idx_col='lbl_id')
save_to_annon_db(cfg, aidsdata=datacfg)
## TODO:
## generate STATS, STATSLABEL and respective SUMMARY csv files for AIDS
mclient.close()
return dbname