def upload_data():
"""
Upload and/or Delete picture in S3 and/or label in ES from a label file (json format). The label file shall be
in the same folder as the picture to upload.
The label file can contain one or more label in the form of a dictionary with the img_id as key for each label.
To see a label template, use the write_label_template.py function.
You will need credential for the upload. Access keys shall be defined in the following environment variables:
export PATATE_S3_KEY_ID="your_access_key_id"
export PATATE_S3_KEY="your_secret_key_code"
export PATATE_ES_USER_ID="your_es_user_id"
export PATATE_ES_USER_PWD="your_es_password"
"""
args = _get_args(upload_data.__doc__)
log.debug("Starting...")
label_file = args.label_file
bucket_name = None if args.es_only else args.bucket
key_prefix = args.key
update_db.delete_picture_and_label(label_file,
es_index=args.index,
bucket=bucket_name)
upload.upload_to_db(label_file,
ES_HOST_IP,
ES_HOST_PORT,
args.index,
bucket_name=bucket_name,
overwrite=args.force,
key_prefix=key_prefix)
log.debug("Execution completed.")
log.debug("Uploading log...")
logger.Logger().upload_log(index=LOG_INDEX,
es_host_ip=ES_HOST_IP,
es_host_port=ES_HOST_PORT)
def test_delete_pic_and_label_local_delete_false(create_delete_tmp_folder):
total_file_in_tmp = len(list(Path("test/.tmp").iterdir()))
success_es, fail_es, success_s3, fail_s3 = update_db.delete_picture_and_label(
TMP_LABELS_DELETE,
es_index=ES_TEST_INDEX,
bucket=BUCKET_NAME,
force=True,
delete_local=False)
assert total_file_in_tmp == len(list(Path("test/.tmp").iterdir()))
def test_delete_pic_and_label_nothing_to_delete():
upload_to_db.upload_to_db(LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=BUCKET_NAME,
key_prefix=KEY_PREFIX)
success_es, fail_es, success_s3, fail_s3 = update_db.delete_picture_and_label(
LABELS,
es_index=ES_TEST_INDEX,
bucket=BUCKET_NAME,
force=True,
delete_local=False)
assert (0, 0, 0, 0) == (success_es, fail_es, success_s3, fail_s3)
def test_delete_pic_and_label_other_label_points_to_pic():
upload_to_db.upload_to_db(LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=BUCKET_NAME,
key_prefix=KEY_PREFIX)
upload_to_db.upload_to_db(LABELS_BLOCK_DELETE,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT)
success_es, fail_es, success_s3, fail_s3 = update_db.delete_picture_and_label(
LABELS_DELETE,
es_index=ES_TEST_INDEX,
bucket=BUCKET_NAME,
force=True,
delete_local=False)
assert (2, 0, 1, 1) == (success_es, fail_es, success_s3, fail_s3)
def test_delete_pic_and_label_local_delete_true_pic_not_in_db(
create_delete_tmp_folder):
assert (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-574348.jpg").is_file()
assert (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-695024.jpg").is_file()
total_file_in_tmp = len(list(Path("test/.tmp").iterdir()))
success_es, fail_es, success_s3, fail_s3 = update_db.delete_picture_and_label(
TMP_LABELS_DELETE,
es_index=ES_TEST_INDEX,
bucket=BUCKET_NAME,
force=True,
delete_local=True)
assert total_file_in_tmp - 2 == len(list(Path("test/.tmp").iterdir()))
assert not (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-574348.jpg").is_file()
assert not (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-695024.jpg").is_file()
def test_delete_pic_and_label_local_delete_true(create_delete_tmp_folder):
assert (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-574348.jpg").is_file()
assert (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-695024.jpg").is_file()
total_file_in_tmp = len(list(Path("test/.tmp").iterdir()))
upload_to_db.upload_to_db(TMP_LABELS,
es_index=ES_TEST_INDEX,
es_host_ip=ES_HOST_IP,
es_port=ES_HOST_PORT,
bucket_name=BUCKET_NAME,
key_prefix=KEY_PREFIX)
success_es, fail_es, success_s3, fail_s3 = update_db.delete_picture_and_label(
TMP_LABELS_DELETE,
es_index=ES_TEST_INDEX,
bucket=BUCKET_NAME,
force=True,
delete_local=True)
assert total_file_in_tmp - 2 == len(list(Path("test/.tmp").iterdir()))
assert not (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-574348.jpg").is_file()
assert not (Path(TMP_LABELS_DELETE).parent /
"20200204T15-23-08-695024.jpg").is_file()