def __init__(self, db_conf: database_conf.Default_database_conf):
# STD attributes
self.db_conf = db_conf
self.logger = logging.getLogger(__name__)
self.logger.critical("SINGLETON CREATED (worker start stop)")
# Specific attributes
# self.worker_path = get_homedir() / pathlib.Path('carlhauser_server', 'DatabaseAccessor', 'datanase_worker.py')
self.adder_worker_path = get_homedir() / pathlib.Path(
'carlhauser_server', 'DatabaseAccessor', 'database_adder.py')
self.requester_worker_path = get_homedir() / pathlib.Path(
'carlhauser_server', 'DatabaseAccessor', 'database_requester.py')
self.feature_worker_path = get_homedir() / pathlib.Path(
'carlhauser_server', 'FeatureExtractor', 'feature_worker.py')
self.flask_worker_path = get_homedir() / pathlib.Path(
'carlhauser_server', 'API', 'API_server.py')
# Mapping from workertype to worker list
self.mapping = {
WorkerTypes.ADDER:
processus_list.ProcessesList("Adder Worker", []),
WorkerTypes.REQUESTER:
processus_list.ProcessesList("Requester Worker", []),
WorkerTypes.FEATURE_ADDER:
processus_list.ProcessesList("Feature Adder Worker", []),
WorkerTypes.FEATURE_REQUESTER:
processus_list.ProcessesList("Feature Requester Worker", []),
WorkerTypes.FLASK:
processus_list.ProcessesList("Flask/API Worker", []),
}
def __init__(self):
# Please note that CERT and KEY files must be in carl-hauser/carlhauser_server (where the flask server is)
self.CERT_FILE: pathlib.Path = get_homedir(
) / 'carlhauser_server' / 'cert.pem' # './cert.pem'
self.KEY_FILE: pathlib.Path = get_homedir(
) / 'carlhauser_server' / 'key.pem' # './key.pem'
self.ip = '127.0.0.1'
self.port = 5000
def add_and_request_and_dump_pictures(self,
image_folder: pathlib.Path) -> List:
"""
Send pictures of a folder, request all pictures one by one, construct a list of results, revert the mapping to get back pictures names
:param image_folder: The folder of images to send
:return: The list of results
"""
self.logger.info(
"Automated launch : add pictures from folder, request all, and dump graph / list of results"
)
# 1- Send pictures to DB and get id mapping
mapping_old_filename_to_new_id, nb_pictures = self.add_many_pictures_and_wait_global(
image_folder)
# add_many_pictures_and_wait_global
# add_many_picture_and_wait_for_each
# TODO : To remove , debug only
json_import_export.save_json(
mapping_old_filename_to_new_id,
pathlib.Path(get_homedir() /
"mapping_old_filename_to_new_id.json"))
# 2 - Get a DB dump
list_results, nb_pictures = self.request_many_pictures_and_wait_global(
image_folder)
# request_many_pictures_and_wait_global
# request_many_pictures_and_wait_for_each
# TODO : To remove , debug only
json_import_export.save_json(
list_results,
pathlib.Path(get_homedir() / "list_result_before_reversion.json"))
list_results = dict_utilities.apply_revert_mapping(
list_results, mapping_old_filename_to_new_id)
# TODO : do it with graphes ? graphe_struct.replace_id_from_mapping(mapping)
# TODO : To remove , debug only
json_import_export.save_json(
list_results,
pathlib.Path(get_homedir() / "list_result_after_reversion.json"))
# We guarantee that each request is not None and each request has a request_id
clean_list_results = [
r for r in list_results
if r is not None and r.get("request_id", None) is not None
]
if len(list_results) != len(clean_list_results):
self.logger.critical(
f"Errors during results fetching. {abs(len(list_results) - len(clean_list_results))} elements were null or without request id."
)
return clean_list_results
def monitor_worker(self, interval: int = 1):
# Create path where to save pictures and logs
curr_name = str(self.worker_path.name) + "_at_" + str(self.start_time)
logpath = pathlib.Path(get_homedir() / (curr_name + ".log"))
graphpath = pathlib.Path(get_homedir() / (curr_name + ".pdf"))
# monitor(self.process.pid, logfile=logpath, plot=graphpath, include_children=True)
# pid, logfile = None, plot = None, duration = None, interval = None,
# Launch the monitoring process
Process(target=monitor, args=(self.process.pid, str(logpath), str(graphpath), interval, True))
def test_env_variable_presence(self):
# Verify if Environment variable are correctly set
try:
environment_variable.get_homedir()
self.assertTrue(True)
except Exception as e:
self.assertTrue(False)
raise Exception(
f"ENVIRONMENT VARIABLE CARLHAUSER_HOME MISSING. {e}")
def launch(self, db_conf: database_conf.Default_database_conf = None,
dist_conf: distance_engine_conf.Default_distance_engine_conf = None,
fe_conf: feature_extractor_conf.Default_feature_extractor_conf = None,
ws_conf: webservice_conf.Default_webservice_conf = None,
mode=None):
"""
Construct an argument list and launch the process.
:param db_conf: configuration file
:param dist_conf: configuration file
:param fe_conf: configuration file
:param ws_conf: configuration file
:param mode: configuration element
:return: Nothing
"""
# Construct an argument list to be 'popen' as a new process
arg_list = [str(self.worker_path)]
# Save current configuration in files
# Using self.worker_path.parent
if db_conf is not None:
tmp_db_conf_path = get_homedir() / "tmp_db_conf.json"
json_import_export.save_json(db_conf, file_path=tmp_db_conf_path)
arg_list.append(ConfArgs.DB_CONF_ARG)
arg_list.append(str(tmp_db_conf_path.resolve()))
if dist_conf is not None:
tmp_dist_conf_path = get_homedir() / "tmp_dist_conf.json"
json_import_export.save_json(dist_conf, file_path=tmp_dist_conf_path)
arg_list.append(ConfArgs.DIST_CONF_ARG)
arg_list.append(str(tmp_dist_conf_path.resolve()))
if fe_conf is not None:
tmp_fe_conf_path = get_homedir() / "tmp_fe_conf.json"
json_import_export.save_json(fe_conf, file_path=tmp_fe_conf_path)
arg_list.append(ConfArgs.FE_CONF_ARG)
arg_list.append(str(tmp_fe_conf_path.resolve()))
if ws_conf is not None:
tmp_ws_conf_path = get_homedir() / "tmp_ws_conf.json"
json_import_export.save_json(ws_conf, file_path=tmp_ws_conf_path)
arg_list.append(ConfArgs.WS_CONF_ARG)
arg_list.append(str(tmp_ws_conf_path.resolve()))
if mode is not None:
arg_list.append(ConfArgs.MODE_ARG)
arg_list.append(mode)
# Save starting time
self.start_time = datetime.datetime.now()
# Launch worker
self.logger.debug(f"launching process as : {arg_list}")
self.process = subprocess.Popen(arg_list)
def setUp(self):
self.logger = logging.getLogger()
# self.conf = configuration.Default_configuration()
# self.test_file_path = pathlib.Path.cwd() / pathlib.Path("tests/test_files")
self.calibrator_instance = Calibrator()
self.micro_dataset_input_path = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "Calibrator_tests" / "MICRO_DATASET"
self.micro_dataset_gt_path = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "Calibrator_tests" / "MICRO_DATASET_VISJS.json"
self.micro_dataset_output_path = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "Calibrator_tests" / "OUTPUT"
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.output_folder = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "Calibrator_conf_test"
self.quality_evaluator = threshold_calibrator.Calibrator()
self.plotmaker = TwoDimensionsPlot()
def build_list_and_evaluate_and_save_chart(
self, list_results: List, gt_graph: GraphDataStruct,
only_decisions: List[scoring_datastrutures.DecisionTypes],
output_folder: pathlib.Path):
# Generate name of the list
generated_name = "".join([d.name + "_"
for d in only_decisions]) + "only"
# Filter out results
results_list_filtered = [
self.filter_out_request_result(r, only_decisions)
for r in list_results
]
json_import_export.save_json(
results_list_filtered,
pathlib.Path(get_homedir() / (generated_name + ".json")))
perfs_list_filtered = self._compute_perfs_list(results_list_filtered,
gt_graph)
# Save to graph
twoDplot = two_dimensions_plot.TwoDimensionsPlot()
twoDplot.print_graph(perfs_list_filtered,
output_folder,
file_name=(generated_name + ".png"))
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_file_path = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "id_generator"
self.cluster_matcher = storage_quality_evaluator.InternalClusteringQualityEvaluator(
)
def setUp(self):
self.logger = logging.getLogger()
self.test_file_path = get_homedir() / "datasets" / "douglas-quaid-tests" / "test_DistanceEngine"
# Create configurations
self.test_db_conf = test_database_only_conf.TestInstance_database_conf()
self.dist_conf = distance_engine_conf.Default_distance_engine_conf()
self.fe_conf = feature_extractor_conf.Default_feature_extractor_conf()
self.fe_conf.A_HASH = Algo_conf("A_HASH", False, 0.2, 0.6, distance_weight=1)
self.fe_conf.P_HASH = Algo_conf("P_HASH", True, 0.2, 0.6, distance_weight=1)
self.fe_conf.P_HASH_SIMPLE = Algo_conf("P_HASH_SIMPLE", False, 0.2, 0.6, distance_weight=1)
self.fe_conf.D_HASH = Algo_conf("D_HASH", True, 0.2, 0.6, distance_weight=1)
self.fe_conf.D_HASH_VERTICAL = Algo_conf("D_HASH_VERTICAL", False, 0.2, 0.6, distance_weight=1)
self.fe_conf.W_HASH = Algo_conf("W_HASH", False, 0.2, 0.6, distance_weight=1)
self.fe_conf.TLSH = Algo_conf("TLSH", True, 0.2, 0.6, distance_weight=1)
self.fe_conf.ORB = Algo_conf("ORB", True, 0.2, 0.6, distance_weight=1)
self.fe_conf.list_algos = [self.fe_conf.A_HASH, self.fe_conf.P_HASH, self.fe_conf.P_HASH_SIMPLE,
self.fe_conf.D_HASH, self.fe_conf.D_HASH_VERTICAL, self.fe_conf.W_HASH,
self.fe_conf.TLSH,
self.fe_conf.ORB]
self.logger.debug(f"Configuration : {self.fe_conf.ORB}")
self.test_db_handler = test_database_handler.TestInstanceLauncher()
self.test_db_handler.create_full_instance(db_conf=self.test_db_conf, dist_conf=self.dist_conf, fe_conf=self.fe_conf)
# Create database handler from test instance
self.db_handler = self.test_db_handler.db_handler
self.picture_hasher = picture_hasher.Picture_Hasher(self.fe_conf)
self.picture_orber = picture_orber.Picture_Orber(self.fe_conf)
# Construct a worker and overwrite link to redis db
self.db_adder = database_adder.Database_Adder(self.test_db_conf, self.dist_conf, self.fe_conf)
self.distance_engine = distance_engine.Distance_Engine(self.db_adder, self.test_db_conf, self.dist_conf, self.fe_conf)
def __init__(self, tmp_db_conf: database_conf):
self.logger = logging.getLogger(__name__)
self.logger.info("Creation of a Database Accessor Worker")
# STD attributes
self.db_conf = tmp_db_conf
json_encoder = Custom_JSON_Encoder()
# Print configuration
self.logger.debug(
f"Configuration db_conf (db worker) : {pprint.pformat(json_encoder.encode(self.db_conf))}"
)
# Specific
self.input_queue = None
self.ouput_queue = None
# Specific attributes
self.redis_cache = get_homedir() / self.db_conf.DB_DATA_PATH
self.redis_storage = get_homedir() / self.db_conf.DB_DATA_PATH
# Get sockets
tmp_db_handler = database_start_stop.Database_StartStop(
db_conf=tmp_db_conf)
self.cache_db_decode = redis.Redis(
unix_socket_path=tmp_db_handler.get_socket_path('cache'),
decode_responses=True)
self.cache_db_no_decode = redis.Redis(
unix_socket_path=tmp_db_handler.get_socket_path('cache'),
decode_responses=False)
self.storage_db_decode = redis.Redis(
unix_socket_path=tmp_db_handler.get_socket_path('storage'),
decode_responses=True)
self.storage_db_no_decode = redis.Redis(
unix_socket_path=tmp_db_handler.get_socket_path('storage'),
decode_responses=False)
# Pickler with patches
self.pickler = pickle_import_export.Pickler()
# Imposible to connect => Shutdown
self.failure_nb = 0
self.FAILURE_THRESHOLD = 10
def get_custom_api(api_class):
# Generate the API access point link to the hardcoded server
cert = (get_homedir() / "carlhauser_client" / "cert.pem").resolve()
# See : https://stackoverflow.com/questions/10667960/python-requests-throwing-sslerror
# To create : openssl req -x509 -newkey rsa:4096 -nodes -out cert.pem -keyout key.pem -days 365
api = api_class(url='https://localhost:5000/',
certificate_path=cert) # TODO : Should be =cert
logging.captureWarnings(True) # TODO : Remove
return api
def __init__(self, db_conf: database_conf.Default_database_conf):
# STD attributes
self.db_conf = db_conf
self.logger = logging.getLogger(__name__)
self.logger.critical("SINGLETON CREATED (database start stop)")
self.logger.debug(
f"Only the test database will be used = {self.db_conf.ONLY_TEST_DB}"
)
# Specific attributes
# We are in a test mode, so we want the test database to be handled too.
if self.db_conf.ONLY_TEST_DB:
self.socket_test = socket.Socket(
get_homedir() / self.db_conf.DB_SOCKETS_PATH_TEST,
get_homedir() / self.db_conf.DB_SCRIPTS_PATH_TEST)
self.mapping = {
'cache': self.socket_test.socket_path,
'storage': self.socket_test.socket_path,
'test': self.socket_test.socket_path,
}
else:
self.socket_cache = socket.Socket(
get_homedir() / self.db_conf.DB_SOCKETS_PATH_CACHE,
get_homedir() / self.db_conf.DB_SCRIPTS_PATH_CACHE)
self.socket_storage = socket.Socket(
get_homedir() / self.db_conf.DB_SOCKETS_PATH_STORAGE,
get_homedir() / self.db_conf.DB_SCRIPTS_PATH_STORAGE)
self.mapping = {
'cache': self.socket_cache.socket_path,
'storage': self.socket_storage.socket_path,
}
def get_perf_list(
self,
list_results: List,
gt_graph: GraphDataStruct,
output_folder: pathlib.Path = None) -> List[perf_datastruct.Perf]:
"""
Extract a list of performance datastructure from a list of results (list_results)
compared to a ground truth file (gt_graph). Can store provided list and ground truth results if a (output_folder) is given.
:param list_results: The list of results extracted from server (one result for each node of the graph)
:param gt_graph: The ground truth file that serves as reference
:param output_folder: Faculatative output folder to save inputs
:return: a list of performance datastructure, each having a threshold and a stats datastructure. This means that for each computed threshold, we know the quality of the graph.
"""
# DEBUG purposes / Display arguments
self.logger.debug("Received requests results :")
self.logger.debug(pformat(list_results))
self.logger.debug("Received ground truth graph :")
self.logger.debug(pformat(gt_graph.export_as_dict()))
# TODO : Remove output folder ?
if output_folder is not None:
# Saving ground truth graph
json_import_export.save_json(
list_results,
get_homedir() / "requests_result.json")
# Saving list results
json_import_export.save_json(gt_graph.export_as_dict(),
get_homedir() / "gt_graph.json")
else:
self.logger.debug(
"List results and ground truth graph can't be saved : no output_folder specified."
)
perfs_list = self._compute_perfs_list(list_results, gt_graph)
return perfs_list
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_path = get_homedir() / "datasets" / "douglas-quaid-tests" / "API_pictures"
# Create configurations
self.db_conf = test_database_only_conf.TestInstance_database_conf()
self.dist_conf = distance_engine_conf.Default_distance_engine_conf()
self.fe_conf = feature_extractor_conf.Default_feature_extractor_conf()
self.test_db_handler = test_database_handler.TestInstanceLauncher()
self.test_db_handler.create_full_instance(db_conf=self.db_conf, dist_conf=self.dist_conf, fe_conf=self.fe_conf)
self.api = Extended_API.get_api()
def setUp(self):
self.logger = logging.getLogger()
self.test_file_path = get_homedir() / "datasets" / "douglas-quaid-tests" / "DBUtils"
# Create configurations
self.test_db_conf = test_database_only_conf.TestInstance_database_conf()
self.dist_conf = distance_engine_conf.Default_distance_engine_conf()
self.fe_conf = feature_extractor_conf.Default_feature_extractor_conf()
self.test_db_handler = test_database_handler.TestInstanceLauncher()
self.test_db_handler.create_full_instance(db_conf=self.test_db_conf, dist_conf=self.dist_conf, fe_conf=self.fe_conf)
db_access_no_decode = redis.Redis(unix_socket_path=self.test_db_handler.db_handler.get_socket_path('test'), decode_responses=False)
db_access_decode = redis.Redis(unix_socket_path=self.test_db_handler.db_handler.get_socket_path('test'), decode_responses=True)
self.db_utils = DBUtilities(db_access_decode=db_access_decode, db_access_no_decode=db_access_no_decode)
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_file_path = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "json_import_export"
self.simple_object = {
"myobjectname": "ThisIsMyObject",
"MyObjectList": ["value1", "value2", "value3", "value4"]
}
self.path_object = {
"myobjectname": "ThisIsMyObject",
"MyObjectList": ["value1", "value2", "value3", "value4"],
"Path": pathlib.Path("/My/Path/")
}
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_file_path = get_homedir() / pathlib.Path("carlhauser_server_tests/test_DistanceEngine/")
# Create configurations
self.dist_conf = distance_engine_conf.Default_distance_engine_conf()
self.fe_conf = feature_extractor_conf.Default_feature_extractor_conf()
self.test_db_conf = test_database_only_conf.TestInstance_database_conf()
print("[TESTS] LAUNCHING DATABASE AS TEST : NOTHING WILL BE WRITEN ON STORAGE OR CACHE DATABASES [TESTS]")
# self.test_db_handler = test_database_handler.TestDatabaseHandler()
# self.test_db_handler.setUp(db_conf=test_configuration)
# Extract what we need : a distance engine
self.db_adder = database_adder.Database_Adder(self.test_db_conf, self.dist_conf, self.fe_conf)
self.distance_engine = distance_engine.Distance_Engine(self.db_adder, self.test_db_conf, self.dist_conf, self.fe_conf)
def enqueue_and_save_picture(self, file: datastructures.FileStorage,
result_json: Dict, queue: QueueNames) -> Dict:
"""
Hash the picture, generate a BMP file, enqueue it, return the result_json with relevant values
:param queue: Queue to add the picture
:param file: the file provided by the client
:param result_json: The result json/dict to which to add standard values
:return: the modified result_json
"""
# Received : werkzeug.datastructures.FileStorage. Should use ".read()" to get picture's value
self.logger.debug(
f"Image received in server : {type(file)} ") # {f.read()}
# Compute input picture hash and convert to BMP
f_hash = id_generator.get_SHA1(file)
f_bmp = id_generator.convert_to_bmp(file) # Returns a bytes array
self.logger.debug(
f"Image transformed in BMP in server : {type(f_bmp)} ") # {f_bmp}
# Save received picture to disk
picture_import_export.save_picture(
f_bmp,
get_homedir() / 'datasets' / 'received_pictures' /
(str(f_hash) + '.bmp'))
# If the filename need to be used : secure_filename(f.filename)
# Generate uuid from SHA-1 : # Done : Create request UUID ? Or keep image hash ?
tmp_uuid = str(uuid.uuid5(uuid.NAMESPACE_URL, f_hash))
# Enqueue picture to processing
self.logger.debug(
f"Adding to feature queue : {f_hash} hash transformed into -> {tmp_uuid} uuid v5"
) # {f_bmp}
self.database_worker.add_to_queue(self.database_worker.cache_db_decode,
queue_name=queue,
input_id=tmp_uuid,
dict_to_store={"img": f_bmp})
result_json["Status"] = "Success"
result_json["id"] = tmp_uuid
return result_json
def export_db_as_graphe(self) -> Dict:
"""
Handle a dump of the database request
:return: The result json (status of the request, etc.)
"""
result_json = {}
result_json["Called_function"] = EndPoints.REQUEST_DB
result_json = self.add_std_info(result_json)
# Answer to PUT HTTP request
if flask.request.method == 'GET':
try:
# Request export of the database and save it as a json graph
self.db_utils = db_utils.DBUtilities(
db_access_decode=self.database_worker.storage_db_decode,
db_access_no_decode=self.database_worker.
storage_db_no_decode)
graph = self.db_utils.get_storage_graph()
graph_dict = graph.export_as_dict()
# Save to file
json_import_export.save_json(
graph_dict,
get_homedir() / "export_folder" / "db_graphe.json")
result_json["Status"] = "Success"
result_json["db"] = graph_dict
except Exception as e:
self.logger.error(f"Error during GET handling {e}")
result_json["Status"] = "Failure"
result_json["Error"] = "Error during db exportation to file"
else:
result_json = self.add_bad_method_info(result_json,
good_method_instead="GET")
return result_json
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_file_path = get_homedir() / pathlib.Path(
"carlhauser_server_tests/test_Helpers/pickle_import_export")
def setUp(self):
self.logger = logging.getLogger()
self.test_path = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "BowOrbVocabulary"
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import logging.config
import pathlib
import signal
import sys
import time
import traceback
from common.environment_variable import get_homedir
# ==================== ------ PREPARATION ------- ====================
# load the logging configuration
logconfig_path = (get_homedir() /
pathlib.Path("carlhauser_server", "logging.ini")).resolve()
# ==================== ------ LAUNCHER ------- ====================
class SafeLauncher:
"""
Handle a class launch with fallback method and clean exit even if brutally stopped.
"""
def __init__(self,
class_to_call=None,
launch_method: str = None,
stop_method: str = None):
self.logger = logging.getLogger(__name__)
self.class_to_call = class_to_call
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_file_path = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "id_generator"
self.perf = performance_evaluation.ClusterMatchingQualityEvaluator()
def launch(self):
# ========= INPUTS =========
# Input files folder
image_folder = get_homedir() / "datasets" / "MINI_DATASET"
# Ground truth file
gt = get_homedir() / "datasets" / "MINI_DATASET_VISJS.json"
# Output general folder
output_folder = get_homedir() / "datasets" / "OUTPUT"
output_folder.mkdir(parents=True, exist_ok=True)
# ========= GOAL =========
perfs = []
iterations_limit = 50 # Or nb of iteration if complete exploration
max_threshold = 1
min_threshold = 0
# ========= CONFIGURATION CHOSING =========
for i in range(iterations_limit):
# Computing the new threshold
curr_threshold = i * (
(max_threshold - min_threshold) / iterations_limit)
self.logger.info(
f"Current threshold computation : {curr_threshold}")
# If the instance already exist, delete it
if self.server_launcher is not None:
del self.server_launcher
# Put configuration in place
self.server_launcher = instance_handler.Instance_Handler()
self.server_launcher.dist_conf.MAX_DIST_FOR_NEW_CLUSTER = curr_threshold
# Create output folder for this configuration
tmp_output = output_folder / ''.join(
[str(curr_threshold), "_threshold"])
tmp_output.mkdir(parents=True, exist_ok=True)
# ========= CONFIGURATION LAUNCH =========
# Launch Server
self.server_launcher.launch()
time.sleep(2)
# Launch client tester
self.client_launcher = evaluator.InternalClusteringQualityEvaluator(
)
perf_overview = self.client_launcher.get_storage_graph(
image_folder, gt, tmp_output)
self.logger.warning(f"Perf overview added : {perf_overview}")
perfs.append(perf_datastruct.Perf(perf_overview, curr_threshold))
# Wait for client end
# ========= TIDY UP FOR NEXT ROUND =========
# Flush server
self.server_launcher.flush_db()
# Shutdown server
self.server_launcher.stop()
# Wait for shutdown (wait for workers to shutdown, usually longer than db)
while not self.server_launcher.check_worker():
time.sleep(1) # Enough ?
self.logger.warning("Waiting for workers to stop .. ")
# Remove all workers
self.server_launcher.flush_workers()
time.sleep(2)
# Print plot
TwoD_plot = two_dimensions_plot.TwoDimensionsPlot()
TwoD_plot.print_graph(perfs, output_folder)
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_file_path = get_homedir() / pathlib.Path("carlhauser_client_tests/evaluator/confusion_matrix")
def overwrite_socket_and_script_db_handler(db_handler: database_start_stop.Database_StartStop,
db_conf: database_conf.Default_database_conf):
"""
Replace all attributes of db_handler by test database values
:param db_handler: An instance of db handler
:param db_conf: The configuration with which to overwrite existing configuration
:return: A modified version of db handler
"""
# Specific attributes
db_handler.cache_socket_path = get_homedir() / db_conf.DB_SOCKETS_PATH / 'test.sock'
db_handler.storage_socket_path = get_homedir() / db_conf.DB_SOCKETS_PATH / 'test.sock'
# Cache scripts
db_handler.launch_cache_script_path = get_homedir() / db_conf.DB_SCRIPTS_PATH / "run.sh"
db_handler.shutdown_cache_script_path = get_homedir() / db_conf.DB_SCRIPTS_PATH / "shutdown.sh"
db_handler.flush_cache_script_path = get_homedir() / db_conf.DB_SCRIPTS_PATH / "shutdown.sh"
# Storage scripts
db_handler.launch_storage_script_path = get_homedir() / db_conf.DB_SCRIPTS_PATH / "run.sh"
db_handler.shutdown_storage_script_path = get_homedir() / db_conf.DB_SCRIPTS_PATH / "shutdown.sh"
db_handler.flush_storage_script_path = get_homedir() / db_conf.DB_SCRIPTS_PATH / "shutdown.sh"
# Setting test-database scripts, sockets, ... paths
db_handler.test_socket_path = get_homedir() / db_conf.DB_SOCKETS_PATH / 'test.sock'
db_handler.launch_test_script_path = get_homedir() / db_conf.DB_SCRIPTS_PATH / "run.sh"
db_handler.shutdown_test_script_path = get_homedir() / db_conf.DB_SCRIPTS_PATH / "shutdown.sh"
return db_handler
def test():
tmp_evaluator = GraphExtractor()
# image_folder = get_homedir() / "datasets" / "MINI_DATASET"
image_folder = get_homedir() / "datasets" / "raw_phishing_full"
output_path = get_homedir() / "carlhauser_client"
tmp_evaluator.get_proximity_graph(image_folder, output_path)
def setUp(self):
self.logger = logging.getLogger()
# self.conf = .Default_configuration()
self.test_file_path = get_homedir(
) / "datasets" / "douglas-quaid-tests" / "confusion_matrix"
self.matrix_gen = confusion_matrix_generator.ConfusionMatrixGenerator()
