def test_upload_to_db_s3_KO_es_OK():
label_file = "test/resources/labels.json"
bucket_name = cluster_conf.BUCKET_NAME
key_prefix = ""
es_ip_host = cluster_conf.ES_HOST_IP
es_port_host = cluster_conf.ES_HOST_PORT
s3_success, es_success, fail = upload.upload_to_db(label_file,
es_ip_host,
es_port_host,
ES_TEST_INDEX,
bucket_name=bucket_name,
key_prefix=key_prefix,
overwrite=True)
es_utils.delete_index(ES_TEST_INDEX, es_ip_host, es_port_host)
es_utils.create_es_index(ES_HOST_IP, ES_HOST_PORT, ES_TEST_INDEX)
s3_success, es_success, fail = upload.upload_to_db(label_file,
es_ip_host,
es_port_host,
ES_TEST_INDEX,
bucket_name=bucket_name,
key_prefix=key_prefix,
overwrite=False)
update_db.delete_pic_and_index(label_file, bucket_name, key_prefix,
ES_TEST_INDEX, es_ip_host, es_port_host)
assert (s3_success, es_success, fail) == (0, 3, 4)
def test_init():
bucket_name = cluster_conf.BUCKET_NAME
key_prefix = "test_download"
es_ip_host = cluster_conf.ES_HOST_IP
es_port_host = cluster_conf.ES_HOST_PORT
es_index_name = "test_index"
output_test_folder = Path("output_test_folder")
output_test_folder.mkdir(exist_ok=True)
es_utils.delete_index(es_index_name,
host_ip=cluster_conf.ES_HOST_IP,
port=cluster_conf.ES_HOST_PORT)
es_utils.create_es_index(cluster_conf.ES_HOST_IP,
cluster_conf.ES_HOST_PORT, es_index_name)
yield output_test_folder, bucket_name, key_prefix, es_ip_host, es_port_host, es_index_name
print("--------------- Test completed ----------------")
print("Deleting test pictures from s3")
s3_utils.delete_all_in_s3_folder(bucket=bucket_name, key_prefix=key_prefix)
print("Deleting test index")
es_utils.delete_index(es_index_name,
host_ip=cluster_conf.ES_HOST_IP,
port=cluster_conf.ES_HOST_PORT)
print("Deleting test folder...")
for file in output_test_folder.iterdir():
file.unlink()
output_test_folder.rmdir()
def delete_pic_and_index(label_file,
bucket_name,
key_prefix,
index,
es_ip,
es_port,
s3_only=False):
"""
Delete all picture listed in label file from s3 and delete index passed in parameters
:param label_file: [str] Path to the labelfile, json format
:param bucket_name: [str] Name of the bucket containing the pictures
:param key_prefix: [str] key prefix to find the picture
:param index: [str] Index to delete
:param es_ip: [str] Ip of the Elastic host
:param es_port: [str] port opened for Elastic
:param s3_only: [bool] If True, only delete picture from s3
"""
d_label = utils_fct.get_label_dict_from_file(label_file)
l_pic_id = list(d_label.keys())
l_pic_s3_key = [
s3_utils.get_s3_formatted_bucket_path(bucket_name, key_prefix,
pic_id)[2] for pic_id in l_pic_id
]
log.info(f'Deleting {len(l_pic_s3_key)} picture(s) in "{bucket_name}"')
s3_utils.delete_object_s3(bucket_name, l_pic_s3_key)
if not s3_only:
log.info(f'Deleting index "{index}" from {es_ip}:{es_port}')
es_utils.delete_index(index, es_ip, es_port)
def create_and_delete_test_index():
es_utils.delete_index(ES_TEST_INDEX, host_ip=ES_HOST_IP, port=ES_HOST_PORT)
es_utils.create_es_index(ES_HOST_IP, ES_HOST_PORT, ES_TEST_INDEX)
yield True
update_db.delete_pic_and_index(LABELS, BUCKET_NAME, KEY_PREFIX,
ES_TEST_INDEX, ES_HOST_IP, ES_HOST_PORT)
update_db.delete_pic_and_index(LABELS, BUCKET_NAME, WRONG_KEY_PREFIX,
ES_TEST_INDEX, ES_HOST_IP, ES_HOST_PORT)
def create_test_index():
es_utils.delete_index(ES_TEST_INDEX,
host_ip=cluster_conf.ES_HOST_IP,
port=cluster_conf.ES_HOST_PORT)
es_utils.create_es_index(ES_HOST_IP, ES_HOST_PORT, ES_TEST_INDEX)