def __init__(self, source_pictures_dir: pathlib.Path,
output_folder: pathlib.Path, ground_truth_json: pathlib.Path,
img_type: configuration.SUPPORTED_IMAGE_TYPE,
overwrite_folder: bool, args):
# /!\ Logging doesn't work in IDE, but works in terminal /!\
# logging.basicConfig(filename='app.log', filemode='w', format='%(name)s - %(levelname)s - %(message)s', level=logging.INFO)
self.logger = logging.getLogger(
) # See : https://stackoverflow.com/questions/50714316/how-to-use-logging-getlogger-name-in-multiple-modules
self.source_pictures_dir = source_pictures_dir
self.output_folder = output_folder
self.ground_truth_json = ground_truth_json
self.img_type = img_type
tmp_conf = configuration.Default_configuration()
tmp_conf.IMG_TYPE = img_type
self.logger.warning(
"Creation of filesystem handler : deletion of 0-sized pictures in source folder."
)
self.filesystem_handler = filesystem_lib.File_System(conf=tmp_conf)
self.filesystem_handler.clean_folder(self.source_pictures_dir)
self.overwrite_folder = overwrite_folder
self.args = args
def create_bmp_from_png():
p = pathlib.Path(DEFAULT_TARGET_DIR).glob('**/*')
files = [x for x in p if x.is_file()]
print(f"files list : {files}")
# Convert pictures
for file in files :
name_img_source = pathlib.Path(DEFAULT_TARGET_DIR) / file.name
name_img_target = pathlib.Path(BMP_TARGET_DIR) / pathlib.Path(file.name).with_suffix('.bmp')
print(name_img_source, name_img_target)
img = Image.open(name_img_source)
img.save(name_img_target)
# Convert baseline file
conf = configuration.Default_configuration()
conf.OUTPUT_DIR = BMP_BASELINE_PATH
json_handler = json_class.Json_handler(conf)
png_json = json_handler.import_json(pathlib.Path(DEFAULT_BASELINE_PATH)) # PNG one
bmp_json = json_handler.replace_type(png_json, ".bmp")
json_handler.json_to_export = bmp_json
json_handler.json_export()
def auto_launch_tlsh(self):
self.logger.info("==== ----- LAUNCHING TLSH algos ---- ==== ")
# Create conf
curr_configuration = configuration.Default_configuration()
curr_configuration.SOURCE_DIR = self.source_pictures_dir
curr_configuration.GROUND_TRUTH_PATH = self.ground_truth_json
curr_configuration.IMG_TYPE = self.img_type
if self.args.save_pictures:
curr_configuration.SAVE_PICTURE_INSTRUCTION_LIST = [
configuration.PICTURE_SAVE_MODE.TOP3
]
else:
curr_configuration.SAVE_PICTURE_INSTRUCTION_LIST = [] # No saving
curr_configuration.OUTPUT_DIR = self.output_folder
# Launch
list_to_execute = [
configuration.ALGO_TYPE.TLSH,
configuration.ALGO_TYPE.TLSH_NO_LENGTH
]
# Launch
for type in list_to_execute:
curr_configuration.ALGO = type
curr_configuration.OUTPUT_DIR = self.output_folder / tlsh.TLSH_execution_handler.conf_to_string(
curr_configuration)
# Jump to next configuration if we are not overwriting current results
if self.skip_if_already_computed(curr_configuration): continue
# Launch configuration
self.launch_exec_handler(tlsh.TLSH_execution_handler,
curr_configuration)
def auto_launch_tlsh(self):
self.logger.info("==== ----- LAUNCHING TLSH algos ---- ==== ")
# Create conf
curr_configuration = configuration.Default_configuration()
curr_configuration.SOURCE_DIR = self.source_pictures_dir
curr_configuration.GROUND_TRUTH_PATH = self.ground_truth_json
curr_configuration.IMG_TYPE = self.img_type
curr_configuration.SAVE_PICTURE = False
curr_configuration.OUTPUT_DIR = self.output_folder
# Launch
list_to_execute = [
configuration.ALGO_TYPE.TLSH,
configuration.ALGO_TYPE.TLSH_NO_LENGTH
]
# Launch
for type in list_to_execute:
curr_configuration.ALGO = type
curr_configuration.OUTPUT_DIR = self.output_folder / tlsh.TLSH_execution_handler.conf_to_string(
curr_configuration)
try:
eh = tlsh.TLSH_execution_handler(conf=curr_configuration)
eh.do_full_test()
except Exception as e:
logging.error(
f"Aborting this configuration. Current configuration thrown an error : {e} "
)
def auto_launch_image_hash(self):
self.logger.info("==== ----- LAUNCHING IMAGE HASH ALGOS ---- ==== ")
# Create conf
curr_configuration = configuration.Default_configuration()
curr_configuration.SOURCE_DIR = self.source_pictures_dir
curr_configuration.GROUND_TRUTH_PATH = self.ground_truth_json
curr_configuration.IMG_TYPE = self.img_type
curr_configuration.SAVE_PICTURE = False
curr_configuration.OUTPUT_DIR = self.output_folder
list_to_execute = [
configuration.ALGO_TYPE.A_HASH, configuration.ALGO_TYPE.P_HASH,
configuration.ALGO_TYPE.P_HASH_SIMPLE,
configuration.ALGO_TYPE.D_HASH,
configuration.ALGO_TYPE.D_HASH_VERTICAL,
configuration.ALGO_TYPE.W_HASH
]
# Launch
for type in list_to_execute:
curr_configuration.ALGO = type
curr_configuration.OUTPUT_DIR = self.output_folder / image_hash.Image_hash_execution_handler.conf_to_string(
curr_configuration)
try:
eh = image_hash.Image_hash_execution_handler(
conf=curr_configuration)
eh.do_full_test()
except Exception as e:
logging.error(
f"Aborting this configuration. Current configuration thrown an error : {e} "
)
def evaluate_graphs(target_pair_folder: pathlib.Path, ground_truth_json: pathlib.Path):
graphe_list = Graph_handler.get_graph_list(target_pair_folder.resolve())
logger = logging.getLogger(__name__)
logger.info(f"Creating merged graphs for {target_pair_folder} in {target_pair_folder}")
tmp_conf = configuration.Default_configuration()
tmp_stats = stats_lib.Stats_handler(conf=tmp_conf)
# Get grund_truth graphe
ground_truth_graphe = Graphe()
ground_truth_json_values = json_class.Json_handler.import_json(ground_truth_json.resolve())
ground_truth_graphe.load_from_json(ground_truth_json_values)
# For all pair of graphes
for curr_graphe_a in graphe_list:
# Evaluating
true_positive_rate = json_class.matching_graphe_percentage(curr_graphe_a['graphe'], ground_truth_graphe)
# define the results
tmp_results = curr_graphe_a['stats']
tmp_results["TRUE_POSITIVE_RATE"] = true_positive_rate
# define where to write the stats
tmp_conf.OUTPUT_DIR = (target_pair_folder / curr_graphe_a['name']).resolve()
tmp_stats.write_stats_to_folder(conf=tmp_conf, results=tmp_results)
def generate_merged_pairs(input_folder: pathlib.Path, target_pair_folder: pathlib.Path):
graphe_list = Graph_handler.get_graph_list(input_folder)
logger = logging.getLogger(__name__)
logger.info(f"Creating merged graphs for {input_folder} in {target_pair_folder}")
tmp_conf = configuration.Default_configuration()
tmp_stats = stats_lib.Stats_handler(conf=tmp_conf)
# For all pair of graphes
for curr_graphe_a in graphe_list:
for curr_graphe_b in graphe_list:
# Merge
tmp_graphe_1 = Graphe()
tmp_graphe_1.load_from_json(curr_graphe_a['graphe'])
tmp_graphe_2 = Graphe()
tmp_graphe_2.load_from_json(curr_graphe_b['graphe'])
merged_graphe = json_class.merge_graphes(tmp_graphe_1, tmp_graphe_2)
# Generate name
new_name = curr_graphe_a["name"] + "_AND_" + curr_graphe_b['name']
# Create folder
future_folder_path = target_pair_folder / new_name
future_folder_path.mkdir(parents=True, exist_ok=True)
tmp_file_path = future_folder_path / "graphe.py"
# Generate merged stats
tmp_results = results.RESULTS()
tmp_results.TIME_PER_PICTURE_MATCHING = curr_graphe_a['stats']["TIME_PER_PICTURE_MATCHING"] + curr_graphe_b['stats']["TIME_PER_PICTURE_MATCHING"]
tmp_results.TIME_PER_PICTURE_PRE_COMPUTING = curr_graphe_a['stats']["TIME_PER_PICTURE_PRE_COMPUTING"] + curr_graphe_b['stats']["TIME_PER_PICTURE_PRE_COMPUTING"]
tmp_results.TIME_TO_LOAD_PICTURES = curr_graphe_a['stats']["TIME_TO_LOAD_PICTURES"] + curr_graphe_b['stats']["TIME_TO_LOAD_PICTURES"]
tmp_results.NB_PICTURE = max(curr_graphe_a['stats']["NB_PICTURE"], curr_graphe_b['stats']["NB_PICTURE"])
# define where to write the stats
tmp_conf.OUTPUT_DIR = future_folder_path
tmp_stats.write_stats_to_folder(conf=tmp_conf, results=tmp_results)
# Save file
filesystem_lib.File_System.save_json(merged_graphe, file_path=tmp_file_path)
'''
f = open(str(tmp_file_path.resolve()), "w+") # Overwrite and create if does not exist
tmp_json = json.dumps(merged_graphe)
f.write(tmp_json)
f.close()
'''
# Copy the conf file
copyfile(input_folder / curr_graphe_a['name'] / "conf.txt", future_folder_path / "conf_1.txt")
copyfile(input_folder / curr_graphe_b['name'] / "conf.txt", future_folder_path / "conf_2.txt")
def auto_launch_void(self):
self.logger.info("==== ----- LAUNCHING Void baseline ---- ==== ")
# Create conf
curr_configuration = configuration.Default_configuration()
curr_configuration.SOURCE_DIR = self.source_pictures_dir
curr_configuration.GROUND_TRUTH_PATH = self.ground_truth_json
curr_configuration.IMG_TYPE = self.img_type
curr_configuration.SAVE_PICTURE_INSTRUCTION_LIST = [] # No saving
curr_configuration.OUTPUT_DIR = self.output_folder / "void_baseline"
# Launch configuration
self.launch_exec_handler(void_baseline.Void_baseline,
curr_configuration)
def __init__(self, source_pictures_dir: pathlib.Path,
output_folder: pathlib.Path, ground_truth_json: pathlib.Path,
img_type: configuration.SUPPORTED_IMAGE_TYPE):
logging.basicConfig(format='%(name)s - %(levelname)s - %(message)s',
level=logging.INFO)
self.logger = logging.getLogger(__name__)
self.source_pictures_dir = source_pictures_dir
self.output_folder = output_folder
self.ground_truth_json = ground_truth_json
self.img_type = img_type
self.logger.warning(
"Creation of filesystem handler : deletion of 0-sized pictures in source folder."
)
tmp_conf = configuration.Default_configuration()
tmp_conf.IMG_TYPE = img_type
self.filesystem_handler = filesystem_lib.File_System(conf=tmp_conf)
self.filesystem_handler.clean_folder(self.source_pictures_dir)