def test_upload_to_db_s3_OK_es_KO():
label_file = "test/resources/labels.json"
bucket_name = cluster_conf.BUCKET_NAME
key_prefix = ""
es_ip_host = cluster_conf.ES_HOST_IP
es_port_host = cluster_conf.ES_HOST_PORT
s3_success, es_success, fail = upload.upload_to_db(label_file,
es_ip_host,
es_port_host,
ES_TEST_INDEX,
bucket_name=bucket_name,
key_prefix=key_prefix,
overwrite=True)
update_db.delete_pic_and_index(label_file,
bucket_name,
key_prefix,
ES_TEST_INDEX,
es_ip_host,
es_port_host,
s3_only=True)
s3_success, es_success, fail = upload.upload_to_db(label_file,
es_ip_host,
es_port_host,
ES_TEST_INDEX,
bucket_name=bucket_name,
key_prefix=key_prefix,
overwrite=False)
update_db.delete_pic_and_index(label_file, bucket_name, key_prefix,
ES_TEST_INDEX, es_ip_host, es_port_host)
assert (s3_success, es_success, fail) == (3, 0, 4)
def upload_data():
"""
Upload and/or Delete picture in S3 and/or label in ES from a label file (json format). The label file shall be
in the same folder as the picture to upload.
The label file can contain one or more label in the form of a dictionary with the img_id as key for each label.
To see a label template, use the write_label_template.py function.
You will need credential for the upload. Access keys shall be defined in the following environment variables:
export PATATE_S3_KEY_ID="your_access_key_id"
export PATATE_S3_KEY="your_secret_key_code"
export PATATE_ES_USER_ID="your_es_user_id"
export PATATE_ES_USER_PWD="your_es_password"
"""
args = _get_args(upload_data.__doc__)
log.debug("Starting...")
label_file = args.label_file
bucket_name = None if args.es_only else args.bucket
key_prefix = args.key
update_db.delete_picture_and_label(label_file,
es_index=args.index,
bucket=bucket_name)
upload.upload_to_db(label_file,
ES_HOST_IP,
ES_HOST_PORT,
args.index,
bucket_name=bucket_name,
overwrite=args.force,
key_prefix=key_prefix)
log.debug("Execution completed.")
log.debug("Uploading log...")
logger.Logger().upload_log(index=LOG_INDEX,
es_host_ip=ES_HOST_IP,
es_host_port=ES_HOST_PORT)
def test_delete_pic_and_label_nothing_to_delete():
upload_to_db.upload_to_db(LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=BUCKET_NAME,
key_prefix=KEY_PREFIX)
success_es, fail_es, success_s3, fail_s3 = update_db.delete_picture_and_label(
LABELS,
es_index=ES_TEST_INDEX,
bucket=BUCKET_NAME,
force=True,
delete_local=False)
assert (0, 0, 0, 0) == (success_es, fail_es, success_s3, fail_s3)
def test_delete_pic_and_label_other_label_points_to_pic():
upload_to_db.upload_to_db(LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=BUCKET_NAME,
key_prefix=KEY_PREFIX)
upload_to_db.upload_to_db(LABELS_BLOCK_DELETE,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT)
success_es, fail_es, success_s3, fail_s3 = update_db.delete_picture_and_label(
LABELS_DELETE,
es_index=ES_TEST_INDEX,
bucket=BUCKET_NAME,
force=True,
delete_local=False)
assert (2, 0, 1, 1) == (success_es, fail_es, success_s3, fail_s3)
def test_delete_label_only():
s3_up_ok, es_up_ok, failed_up = upload_to_db.upload_to_db(
LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=BUCKET_NAME,
key_prefix=KEY_PREFIX)
success, fail = update_db.delete_label_only(LABELS,
es_index=ES_TEST_INDEX,
force=True)
assert es_up_ok == success
s3_up_ok, es_up_ok, failed_up = upload_to_db.upload_to_db(
LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=BUCKET_NAME,
key_prefix=KEY_PREFIX)
assert s3_up_ok == 0
assert es_up_ok == 3
def test_create_and_delete_dataset_with_just_name_filled():
dataset = {
"name": "test_dataset",
"comment": "",
"created_on_date": None,
"query": None
}
search_query = {
"dataset.name": {
"type": "match",
"field": "keyword",
"query": dataset["name"]
}
}
upload_to_db.upload_to_db(LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=None,
key_prefix=None)
d_label = utils_fct.get_label_dict_from_file(LABELS)
es_utils.append_value_to_field(d_label, "dataset", dataset, ES_TEST_INDEX,
ES_HOST_IP, ES_HOST_PORT)
time.sleep(1)
d_pic_in_dataset = get_from_db.run_search_query(search_query,
es_index=ES_TEST_INDEX,
verbose=2)
assert len(d_pic_in_dataset) == 4
update_db.delete_dataset(dataset["name"],
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_host_port=ES_HOST_PORT,
force=True)
time.sleep(1)
d_pic_in_dataset = get_from_db.run_search_query(search_query,
es_index=ES_TEST_INDEX,
verbose=2)
assert len(d_pic_in_dataset) == 0
def test_delete_pic_and_label_local_delete_true(create_delete_tmp_folder):
assert (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-574348.jpg").is_file()
assert (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-695024.jpg").is_file()
total_file_in_tmp = len(list(Path("test/.tmp").iterdir()))
upload_to_db.upload_to_db(TMP_LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=BUCKET_NAME,
key_prefix=KEY_PREFIX)
success_es, fail_es, success_s3, fail_s3 = update_db.delete_picture_and_label(
TMP_LABELS_DELETE,
es_index=ES_TEST_INDEX,
bucket=BUCKET_NAME,
force=True,
delete_local=True)
assert total_file_in_tmp - 2 == len(list(Path("test/.tmp").iterdir()))
assert not (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-574348.jpg").is_file()
assert not (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-695024.jpg").is_file()
def test_upload_to_db_key_prefix():
label_file = "test/resources/labels.json"
bucket_name = cluster_conf.BUCKET_NAME + "/"
key_prefix = "/weird/path//"
es_ip_host = cluster_conf.ES_HOST_IP
es_port_host = cluster_conf.ES_HOST_PORT
s3_success, es_success, fail = upload.upload_to_db(label_file,
es_ip_host,
es_port_host,
ES_TEST_INDEX,
bucket_name=bucket_name,
key_prefix=key_prefix,
overwrite=True)
update_db.delete_pic_and_index(label_file, bucket_name, key_prefix,
ES_TEST_INDEX, es_ip_host, es_port_host)
assert (s3_success, es_success, fail) == (3, 3, 1)
def test_download_single_file(test_init):
label_file = "test/resources/single_label.json"
output_test_folder, bucket_name, key_prefix, es_ip_host, es_port_host, es_index_name = test_init
with Path(label_file).open(mode='r', encoding='utf-8') as fp:
label = json.load(fp)
s3_success, es_success, fail = upload.upload_to_db(label_file,
es_ip_host,
es_port_host,
es_index_name,
bucket_name=bucket_name,
key_prefix=key_prefix,
overwrite=False)
if fail > 0:
raise RuntimeError(
f"Failed to upload: can't test download if upload is not working")
time.sleep(1)
test_res = s3_utils.download_from_s3(label, output_dir=output_test_folder)
assert test_res is None
def test_download_full_pipeline(test_init):
label_file = "test/resources/labels.json"
output_test_folder, bucket_name, key_prefix, es_ip_host, es_port_host, es_index_name = test_init
s3_success, es_success, fail = upload.upload_to_db(label_file,
es_ip_host,
es_port_host,
es_index_name,
bucket_name=bucket_name,
key_prefix=key_prefix,
overwrite=False)
assert fail == 1
search = {"event": {"type": "match", "field": "", "query": "unittest"}}
search_json = "tmp_search_test.json"
with Path(search_json).open(mode='w', encoding='utf-8') as fp:
json.dump(search, fp)
time.sleep(1)
l_dl_picture = get_from_db.search_and_download(search_json,
output_test_folder,
es_index=es_index_name,
force=True)
Path(search_json).unlink()
assert len(l_dl_picture) == 3
