def setUp(self):
self.logger = logging.getLogger()
self.test_file_path = get_homedir() / "datasets" / "douglas-quaid-tests" / "test_DistanceEngine"
# Create configurations
self.test_db_conf = test_database_only_conf.TestInstance_database_conf()
self.dist_conf = distance_engine_conf.Default_distance_engine_conf()
self.fe_conf = feature_extractor_conf.Default_feature_extractor_conf()
self.fe_conf.A_HASH = Algo_conf("A_HASH", False, 0.2, 0.6, distance_weight=1)
self.fe_conf.P_HASH = Algo_conf("P_HASH", True, 0.2, 0.6, distance_weight=1)
self.fe_conf.P_HASH_SIMPLE = Algo_conf("P_HASH_SIMPLE", False, 0.2, 0.6, distance_weight=1)
self.fe_conf.D_HASH = Algo_conf("D_HASH", True, 0.2, 0.6, distance_weight=1)
self.fe_conf.D_HASH_VERTICAL = Algo_conf("D_HASH_VERTICAL", False, 0.2, 0.6, distance_weight=1)
self.fe_conf.W_HASH = Algo_conf("W_HASH", False, 0.2, 0.6, distance_weight=1)
self.fe_conf.TLSH = Algo_conf("TLSH", True, 0.2, 0.6, distance_weight=1)
self.fe_conf.ORB = Algo_conf("ORB", True, 0.2, 0.6, distance_weight=1)
self.fe_conf.list_algos = [self.fe_conf.A_HASH, self.fe_conf.P_HASH, self.fe_conf.P_HASH_SIMPLE,
self.fe_conf.D_HASH, self.fe_conf.D_HASH_VERTICAL, self.fe_conf.W_HASH,
self.fe_conf.TLSH,
self.fe_conf.ORB]
self.logger.debug(f"Configuration : {self.fe_conf.ORB}")
self.test_db_handler = test_database_handler.TestInstanceLauncher()
self.test_db_handler.create_full_instance(db_conf=self.test_db_conf, dist_conf=self.dist_conf, fe_conf=self.fe_conf)
# Create database handler from test instance
self.db_handler = self.test_db_handler.db_handler
self.picture_hasher = picture_hasher.Picture_Hasher(self.fe_conf)
self.picture_orber = picture_orber.Picture_Orber(self.fe_conf)
# Construct a worker and overwrite link to redis db
self.db_adder = database_adder.Database_Adder(self.test_db_conf, self.dist_conf, self.fe_conf)
self.distance_engine = distance_engine.Distance_Engine(self.db_adder, self.test_db_conf, self.dist_conf, self.fe_conf)
def setUp(self):
self.logger = logging.getLogger()
# Create configurations
self.test_db_conf = test_database_only_conf.TestInstance_database_conf(
)
self.test_db_handler = test_database_handler.TestInstanceLauncher()
self.test_db_handler.create_full_instance(db_conf=self.test_db_conf)
# Create database handler from test instance
self.db_handler = self.test_db_handler.db_handler
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_path = get_homedir() / "datasets" / "douglas-quaid-tests" / "API_pictures"
# Create configurations
self.db_conf = test_database_only_conf.TestInstance_database_conf()
self.dist_conf = distance_engine_conf.Default_distance_engine_conf()
self.fe_conf = feature_extractor_conf.Default_feature_extractor_conf()
self.test_db_handler = test_database_handler.TestInstanceLauncher()
self.test_db_handler.create_full_instance(db_conf=self.db_conf, dist_conf=self.dist_conf, fe_conf=self.fe_conf)
self.api = Extended_API.get_api()
def setUp(self):
self.logger = logging.getLogger()
self.test_file_path = get_homedir() / "datasets" / "douglas-quaid-tests" / "DBUtils"
# Create configurations
self.test_db_conf = test_database_only_conf.TestInstance_database_conf()
self.dist_conf = distance_engine_conf.Default_distance_engine_conf()
self.fe_conf = feature_extractor_conf.Default_feature_extractor_conf()
self.test_db_handler = test_database_handler.TestInstanceLauncher()
self.test_db_handler.create_full_instance(db_conf=self.test_db_conf, dist_conf=self.dist_conf, fe_conf=self.fe_conf)
db_access_no_decode = redis.Redis(unix_socket_path=self.test_db_handler.db_handler.get_socket_path('test'), decode_responses=False)
db_access_decode = redis.Redis(unix_socket_path=self.test_db_handler.db_handler.get_socket_path('test'), decode_responses=True)
self.db_utils = DBUtilities(db_access_decode=db_access_decode, db_access_no_decode=db_access_no_decode)
def setUp(self):
self.logger = logging.getLogger()
# Create configurations
self.db_conf = test_database_only_conf.TestInstance_database_conf()
self.dist_conf = distance_engine_conf.Default_distance_engine_conf()
self.fe_conf = feature_extractor_conf.Default_feature_extractor_conf()
self.ws_conf = Default_webservice_conf()
self.test_db_handler = test_database_handler.TestInstanceLauncher()
self.test_db_handler.create_full_instance(db_conf=self.db_conf,
dist_conf=self.dist_conf,
fe_conf=self.fe_conf)
self.api = Extended_API.get_api()
self.flask = FlaskAppWrapper('api',
tmp_ws_conf=self.ws_conf,
tmp_db_conf=self.db_conf)
self.flask.add_all_endpoints()
def get_scalability_list(
self,
list_boxes_sizes: List[int],
pictures_set: Set[
pathlib.Path], # pics_to_evaluate: Set[pathlib.Path],
dist_conf: dec.Default_distance_engine_conf = dec.
Default_distance_engine_conf(),
fe_conf: fec.Default_feature_extractor_conf = fec.
Default_feature_extractor_conf(),
output_folder: pathlib.Path = None):
# ==== Upload pictures + Make requests ====
scalability_data = ScalabilityData()
db_conf = test_database_only_conf.TestInstance_database_conf(
) # For test sockets only
# Launch a modified server
self.logger.debug(
f"Creation of a full instance of redis (Test only) ... ")
test_db_handler = test_database_handler.TestInstanceLauncher()
test_db_handler.create_full_instance(db_conf=db_conf,
dist_conf=dist_conf,
fe_conf=fe_conf)
# Get direct access to DB to retrieve statistics
db_access_no_decode = redis.Redis(
unix_socket_path=test_db_handler.db_handler.get_socket_path(
'test'),
decode_responses=False)
db_access_decode = redis.Redis(
unix_socket_path=test_db_handler.db_handler.get_socket_path(
'test'),
decode_responses=True)
db_utils = DBUtilities(db_access_decode=db_access_decode,
db_access_no_decode=db_access_no_decode)
nb_picture_total_in_db = 0
global_mapping = {}
# For each box
for i, curr_box_size in enumerate(list_boxes_sizes):
# Get a list of pictures to send
pictures_set, pics_to_request = self.biner(
pictures_set, self.scalability_conf.NB_PICS_TO_REQUEST)
pictures_set, pics_to_store = self.biner(pictures_set,
curr_box_size)
self.logger.info(
f"Nb of pictures left to be uploaded later : {len(pictures_set)}"
)
self.logger.info(
f"Nb of pictures to upload (adding) : {len(pics_to_store)}")
# If we are not out of pictures to send
if len(pics_to_store) != 0:
# Evaluate time for this database size and store it
tmp_scal_datastruct, mapping, request_list = self.evaluate_scalability_lists(
list_pictures_eval=pics_to_request, # pics_to_evaluate,
list_picture_to_up=pics_to_store,
tmp_id=i)
global_mapping = {**global_mapping, **mapping}
# Store few more values
# Nb of picture in teh database right now
nb_picture_total_in_db += tmp_scal_datastruct.nb_picture_added
tmp_scal_datastruct.nb_picture_total_in_db = db_utils.get_nb_stored_pictures(
)
# Nb of pictures sent at the beginning to be added
tmp_scal_datastruct.nb_picture_tried_to_be_added = len(
pics_to_store)
tmp_scal_datastruct.nb_picture_tried_to_be_requested = len(
pics_to_request)
# Nb of cluster and their content
tmp_scal_datastruct.nb_clusters_in_db = len(
db_utils.get_cluster_list())
tmp_scal_datastruct.clusters_sizes = db_utils.get_list_cluster_sizes(
)
# Print error
if tmp_scal_datastruct.nb_picture_total_in_db != nb_picture_total_in_db:
self.logger.error(
f"Error in scalability evaluator, number of picture really in DB and computed as should being in DB are differents : {tmp_scal_datastruct.nb_picture_total_in_db} {nb_picture_total_in_db}"
)
scalability_data.list_request_time.append(tmp_scal_datastruct)
if output_folder is not None:
save_path_json = output_folder / ("global_mapping" +
str(i) + ".json")
json_import_export.save_json(request_list, save_path_json)
save_path_json = output_folder / ("mapping" + str(i) +
".json")
json_import_export.save_json(mapping, save_path_json)
# Export graph
if output_folder is not None:
self.export_graph(output_folder, global_mapping)
'''
db_dump = self.ext_api.get_db_dump_as_graph()
db_dump_dict = db_dump.export_as_dict()
save_path_json = output_folder / "original_storage_graph_dump.json"
json_import_export.save_json(db_dump_dict, save_path_json)
# Full of new ids
save_path_json = output_folder / "global_mapping.json"
json_import_export.save_json(global_mapping, save_path_json)
# old name -> new id
db_dump_dict = dict_utilities.apply_revert_mapping(db_dump_dict, global_mapping)
# db_dump.replace_id_from_mapping(mapping)
db_dump_dict = dict_utilities.copy_id_to_image(db_dump_dict)
save_path_json = output_folder / "modified_storage_graph_dump.json"
json_import_export.save_json(db_dump_dict, save_path_json)
'''
else:
self.logger.critical("outputfolder is None ! ")
# node server.js -i ./../DATASETS/PHISHING/PHISHING-DATASET-DISTRIBUTED-DEDUPLICATED/ -t ./TMP -o ./TMP -j ./../douglas-quaid/datasets/OUTPUT_EVALUATION/threshold_0.0195/modified_storage_graph_dump.json
# Kill server instance
self.logger.debug(f"Shutting down Redis test instance")
test_db_handler.tearDown()
return scalability_data