def test_json_import_export_consistency(self):
# Test consistency between import and export function
try:
json_import_export.save_json(
self.simple_object, self.test_file_path / "export_test.json")
obj = json_import_export.load_json(self.test_file_path /
"export_test.json")
self.assertEqual(self.simple_object, obj)
self.assertTrue(True)
except Exception as e:
self.assertTrue(False)
try:
json_import_export.save_json(
self.path_object, self.test_file_path / "export_test.json")
obj = json_import_export.load_json(self.test_file_path /
"export_test.json")
# TODO : Create pathlib.Path from string path. For not, can't be equal
self.assertNotEqual(self.path_object, obj)
self.assertTrue(True)
except Exception as e:
self.assertTrue(False)
def parse_conf_files(
args
) -> (database_conf.Default_database_conf, distance_engine_conf.
Default_distance_engine_conf, feature_extractor_conf.
Default_feature_extractor_conf, webservice_conf.Default_webservice_conf):
"""
Parse args to configuration files, if they exist.
Usage example : db_conf, dist_conf, fe_conf, ws_conf = arg_parser.parse_conf_files(args)
:param args: parsed arguments from command line
:return: db_conf, dist_conf, feature_conf, webservice_conf
"""
logger = logging.getLogger()
tmp_db_conf, tmp_dist_conf, tmp_fe_conf, tmp_ws_conf = None, None, None, None
# Load the provided configuration file and create back the Configuration Object
try:
conf_arg = getattr(args, ConfArgs.DB_CONF_NAME)
if conf_arg:
tmp_db_conf = database_conf.parse_from_dict(
json_import_export.load_json(pathlib.Path(conf_arg)))
except AttributeError as e:
logger.debug(
f"No DB CONF argument : {e}. This may be normal if current launch (e.g. a worker) does not require this configuration."
)
try:
conf_arg = getattr(args, ConfArgs.DIST_CONF_NAME)
if conf_arg:
tmp_dist_conf = distance_engine_conf.parse_from_dict(
json_import_export.load_json(pathlib.Path(conf_arg)))
except AttributeError as e:
logger.debug(
f"No DIST CONF argument : {e}. This may be normal if current launch (e.g. a worker) does not require this configuration."
)
try:
conf_arg = getattr(args, ConfArgs.FE_CONF_NAME)
if conf_arg:
tmp_fe_conf = feature_extractor_conf.parse_from_dict(
json_import_export.load_json(pathlib.Path(conf_arg)))
except AttributeError as e:
logger.debug(
f"No FE CONF argument : {e}. This may be normal if current launch (e.g. a worker) does not require this configuration."
)
try:
conf_arg = getattr(args, ConfArgs.WS_CONF_NAME)
if conf_arg:
tmp_ws_conf = webservice_conf.parse_from_dict(
json_import_export.load_json(pathlib.Path(conf_arg)))
except AttributeError as e:
logger.debug(
f"No WS CONF argument : {e}. This may be normal if current launch (e.g. a worker) does not require this configuration."
)
return tmp_db_conf, tmp_dist_conf, tmp_fe_conf, tmp_ws_conf
def extract(path: pathlib.Path):
tmp_json = load_json(path)
for val in tmp_json["values"]:
for entry in val["entry"]:
tmp_str = tmp_json["namespace"] + ":" + val["predicate"] + '="' + entry["value"] + '",'
print(tmp_str)
def dump(self, args):
'''
Dump the database and transmit it to the client, and save it in a file(args.dbfile)
Translate back the provided ids of the server with the filenames to id mapping
saved previously (args.mapfile). Can duplicate id of picture to their "image" and "shape" attributes. Allows to visualize the database with visjs-classificator (args.copyids)
:param args: arguments as described
:return: The database as a Dict of a graphe (visjs-classificator style)
'''
print(f"Requesting server to dump its database")
graphe_struct = self.ext_api.get_db_dump_as_graph()
db = graphe_struct.export_as_dict()
# TODO : Handle it properly with graphe structure calls ? For now, only operation on dict
# If mapfile is provided, reverse the id. Otherwise, do nothing
if args.mapfile:
print(f"Mapping file detected. Reversing the ids ... ")
mapping = load_json(args.mapfile)
db = apply_revert_mapping(db, mapping)
# TODO : graphe_struct.replace_id_from_mapping(mapping) # Cleaner
# If Copy_ids is true, we copy the value of the picture's ids
# to their image and shape attributes
if args.copyids:
print(args.copyids)
print(f"ID to image copy option detected. Copying ... ")
db = copy_id_to_image(db)
# TODO : graphe_struct.copy_ids_to_image() # Cleaner
save_json(db, args.dbfile)
return db
def test_json_import(self):
# Test json import functions
try:
obj = json_import_export.load_json(self.test_file_path /
"does_not_exist.json")
self.assertTrue(False)
except Exception as e:
self.assertTrue(True)
try:
obj = json_import_export.load_json(self.test_file_path /
"path_object.json")
# Verify if object is correct
self.assertEqual(obj["myobjectname"], "ThisIsMyObject")
self.assertEqual(obj["MyObjectList"][1], "value2")
self.assertEqual(obj["Path"], "/My/Path")
self.assertTrue(True)
except Exception as e:
self.assertTrue(False)
def load_visjs_to_graph(visjs_json_path: pathlib.Path = None) -> GraphDataStruct:
"""
Load VisJS to graph datastructure
:param visjs_json_path: path to the visjs graph storage
:return: The graph datastructure as an object
"""
if visjs_json_path is None:
raise Exception("VisJS ground truth path not set. Impossible to evaluate.")
else:
# 1. Load pictures to visjs = node server.js -i ./../douglas-quaid/datasets/MINI_DATASET/ -t ./TMP -o ./TMP
# 2. Cluster manually pictures in visjs = < Do manual stuff>
# 3. Load json graphe
visjs = json_import_export.load_json(visjs_json_path)
visjs = GraphDataStruct.load_from_dict(visjs)
return visjs
def request(self, args) -> Dict:
'''
Request the similar pictures of the provided picture (args.path)
if we get an answer before timeout (args.waittime). Translate back the provided ids
of the server with the filenames to id mapping saved previously (args.mapfile)
:param args: arguments as described
:return: A dict of results # TODO : Add an example of dict of results
'''
results = self.ext_api.request_one_picture_and_wait(args.path, args.waittime)
# If mapfile is provided, reverse the id. Otherwise, do nothing
if args.mapfile:
print(f"Mapping file detected. Reversing the ids ... ")
mapping = load_json(args.mapfile)
results = apply_revert_mapping(results, mapping)
save_json(results, args.resultfile)
return results
def load_file(self, inputfile_path: pathlib.Path):
tmp_json = load_json(inputfile_path)
pprint.pprint(tmp_json)
self.to_export = []
'''
(...),
{'annotation':
{
'labels': ['Steam'],
'note': ''
},
'content': 'fat-noxious-doubtful-walk.png',
'metadata': {'evaluation': 'NONE',
'first_done_at': 1561366939000,
'last_updated_at': 1561366939000,
'last_updated_by': 'user',
'sec_taken': 0,
'status': 'done'}}]
'''
for picture in tmp_json:
tmp_final_object = {}
tmp_final_object["picture"] = picture.get(
"content") # 'fat-noxious-doubtful-walk.png'
tmp_annotation = picture.get("annotation")
if tmp_annotation is not None:
tmp_final_object["labels"] = tmp_annotation.get(
'labels') # ['Steam']
if "to_delete" in tmp_final_object["labels"]:
# mark to be deleted
self.file_names_to_delete.add(tmp_final_object["picture"])
print(f"{tmp_final_object['picture']} mark to be deleted.")
else:
# Add to export
self.to_export.append(tmp_final_object)
# print(f"{tmp_final_object} mark to be exported.")
else:
print(
f"Annotation void for picture : {tmp_final_object['picture']}"
)
def load_file(self, inputfile_path: pathlib.Path):
tmp_json = load_json(inputfile_path)
pprint.pprint(tmp_json)
self.to_export = []
self.num_to_name_dict = {}
'''
"nodes": [
{
"id": 0,
"image": "abashed-careless-ordinary-crew.png",
"shape": "image"
},
{
"id": 1,
"image": "ablaze-jazzy-tangy-file.png",
"shape": "image"
},
'''
# Construct dict id -> name
for picture in tmp_json["nodes"]:
self.num_to_name_dict[picture.get("id")] = picture.get("image")
tmp_cluster_list = []
# Modify clusters
for cluster in tmp_json["clusters"]:
tmp_cluster = {}
tmp_cluster["cluster"] = cluster.get("label")
tmp_cluster_list.append(cluster.get("label"))
# Add tranlated member name
tmp_members = []
for members in cluster.get("members"):
tmp_members.append(self.num_to_name_dict[members])
tmp_cluster["members"] = tmp_members
self.to_export.append(tmp_cluster)
print(tmp_cluster_list)
def load_file(self, inputfile_path: pathlib.Path,
outputfile_path: pathlib.Path):
tmp_json = load_json(inputfile_path)
pprint.pprint(tmp_json)
self.to_export = []
'''
(...),
{'annotation':
{
'labels': ['Steam'],
'note': ''
},
'content': 'fat-noxious-doubtful-walk.png',
'metadata': {'evaluation': 'NONE',
'first_done_at': 1561366939000,
'last_updated_at': 1561366939000,
'last_updated_by': 'user',
'sec_taken': 0,
'status': 'done'}}]
'''
clusters = {}
labels_sets = {}
# For each picture
for picture in tmp_json:
tmp_final_object = {}
tmp_final_object["picture"] = picture.get(
"content") # 'fat-noxious-doubtful-walk.png'
tmp_annotation = picture.get("annotation")
# If there is annotation for this picture
if tmp_annotation is not None:
# Store the number of annotation of this kind
extracted_clusters = [
self.strip_tag(label)
for label in tmp_annotation.get('labels')
]
# Store each cluster frequency
for cluster in extracted_clusters: # ['Steam']
clusters[cluster] = clusters.get(cluster, 0) + 1
# Store each set of clusters frequency
tmp_set, tmp_key = self.create_set_from_labels_list(
extracted_clusters)
# We store only pairs, at least
if len(tmp_set) > 1:
labels_sets[tmp_key] = labels_sets.get(tmp_key, 0) + 1
else:
print(
f"Annotation void for picture : {tmp_final_object['picture']}"
)
# Print the labels names frequencies
list_key, list_val = self.get_list_val_list_key_from_dict(clusters)
self.draw_plot(list_key, list_val, outputfile_path / "fig_simple.pdf")
# Remove too low sets :
frequent_only = {k: v for k, v in labels_sets.items() if v > 10}
# Print the set of labels names frequencies
list_key, list_val = self.get_list_val_list_key_from_dict(
frequent_only)
self.draw_plot(list_key,
list_val,
outputfile_path / "fig_set.pdf",
log_scale=True)
def load_dataturks_json(self):
self.dataturks_json = json_io.load_json(self.dataturks)
