def test_box_size_limit():
""" 6. Test case: Export orthophoto according restricted region size (bbox) as geoPackage
-----------------------------------------------------------------------------
Validate requirement 12 – server side – 1 Export API
-----------------------------------------------------------------------------
Test creation of geoPackage data, bounded by region size configuration value.
Default configuration parameter (100X100).
Fail if required region size is greater than this value.
"""
_log.info('Start running test: %s', test_export_geopackage.__name__)
err = 'Unknown'
request = request_sampels.get_request_by_box_size(
request_sampels.BoxSize.Big)
assert request
# sending requests with different bbox sizes
s_code, content = exc.send_export_request(
request, request_name="test_case_6_exporter_api_big" + Z_TIME)
assert config.ResponseCode.ValidationErrors.value == s_code and content[
'name'] == config.BOX_LIMIT_ERROR, f"limit box [{request_sampels.BoxSize.Medium}] test failed"
if config.ENVIRONMENT_NAME == 'qa': # on QA environment the limit size can be changes and its to prevent overload
request = request_sampels.get_request_by_box_size(
request_sampels.BoxSize.Medium)
assert request
s_code, content = exc.send_export_request(
request, request_name="test_case_6_exporter_api_medium" + Z_TIME)
assert config.ResponseCode.ValidationErrors.value == s_code and content[
'name'] == config.BOX_LIMIT_ERROR, f"limit box [{request_sampels.BoxSize.Medium}] test failed"
request = request_sampels.get_request_by_box_size(
request_sampels.BoxSize.Sanity)
assert request
s_code, content = exc.send_export_request(
request, request_name="test_case_6_exporter_api_small" + Z_TIME)
assert s_code == config.ResponseCode.Ok.value, f'Error on trigger stage [code: {s_code}], \n [message: {str(content)}]'
try:
res = exc.exporter_follower(content['uuid'])
# res = exc.exporter_follower(config.EXPORT_STORAGE_URL, content['uuid'])
uuids.append(content['uuid'])
except Exception as e:
res = None
err = str(e)
assert res, ('Error while exporting package - exporter follow stage %s' %
err)
_log.info('Finish running test: %s', test_box_size_limit.__name__)
def test_download_package():
""" 9. Test case:Download locally orthophoto geopackage from shared storage.
-----------------------------------------------------------------------------
Validate requirement 7+11 – server side – 1 Export API
-----------------------------------------------------------------------------
This test validates that export process provide download url and able to download locally
"""
_log.info('Start running test: %s', test_download_package.__name__)
""" Prerequisites - creating export package to test download process"""
# prepare request
request = request_sampels.get_request_by_box_size(
request_sampels.box_size.Sanity)
assert request, \
f'Test: [{test_download_package.__name__}] Failed: File not exist or failure on loading request json'
request = (json.loads(request))
request['fileName'] = config.EXPORT_DOWNLOAD_FILE_NAME
request['directoryName'] = config.EXPORT_DOWNLOAD_DIR_NAME
# start trigger export
s_code, content = exc.send_export_request(json.dumps(request))
assert s_code == config.ResponseCode.Ok.value, \
f'Test: [{test_download_package.__name__}] Failed: Exporter trigger return status code [{s_code}]'
# check exporting process and wait till end with results
res = None
try:
res = exc.exporter_follower(config.EXPORT_STORAGE_URL, content['uuid'])
uuids.append(content['uuid'])
except Exception as e:
err = str(e)
assert res, \
f'Test: [{test_download_package.__name__}] Failed: on follow (worker stage) with message: [{err}]'
# store data for download test
ValueStorage.gpkg_download_url = res['fileURI']
ValueStorage.file_name = request['fileName']
ValueStorage.directory_name = request['directoryName']
"""Test download actual flow"""
# validate exported file exist on storage from previous test - test_export_on_storage
_log.info(ValueStorage.gpkg_download_url)
assert ValueStorage.gpkg_download_url, \
f'Test: [{test_download_package.__name__}] Failed: Download URI not found!]'
s_code, downloaded_data = exc.send_download_request(
ValueStorage.gpkg_download_url)
assert s_code == config.ResponseCode.Ok.value, \
f'Test: [{test_download_package.__name__}] Failed: Download request failed with status code: [{s_code}])'
orig_exported = exc.load_gpkg_from_storage(ValueStorage.file_name,
ValueStorage.directory_name)
fp_orig = common.generate_unique_fingerprint(orig_exported)
fp_downloaded = common.generate_unique_fingerprint(downloaded_data)
assert fp_orig == fp_downloaded, \
f'Test: [{test_download_package.__name__}] Failed: download geopackage is not equal to stored: [{fp_orig}] != [{fp_downloaded}])'
_log.info('Finish running test: %s', test_download_package.__name__)
def test_export_geopackage(get_config, Zapi_cloud_client, test_status,
issue_id, get_execution):
""" 1. Test case: Exporting Orthophoto (raster data) as geopackage with specific layer
-----------------------------------------------------------------------------
Validate requirement 1+2 – server side – 1 Export API
-----------------------------------------------------------------------------
This test validates exporting geoPackage includes all tiles and their relative metadata based on best layer.
"""
_log.info('Start running test: %s', test_export_geopackage.__name__)
# check and load request json
request = request_sampels.get_request_sample('et_req_2')
assert request, \
f'Test: [{test_export_geopackage.__name__}] Failed: File not exist or failure on loading request json'
# sending json request and validating export process
request = json.loads(request)
request['fileName'] = 'test_case_1_exporter_api'
request = json.dumps(request)
s_code, content = exc.send_export_request(request)
assert s_code == config.ResponseCode.Ok.value, \
f'Test: [{test_export_geopackage.__name__}] Failed: Exporter trigger return status code [{s_code}] '
# worker stage - follow exporting via storage statuses API
try:
if content.get('uuid'): # for later system cleanup
uuids.append(content['uuid'])
res = exc.exporter_follower(config.EXPORT_STORAGE_URL, content['uuid'])
error_msg = None
except RuntimeError as e:
res = None
error_msg = str(e)
except Exception as e:
res = None
error_msg = str(e)
assert res, \
f'Test: [{test_export_geopackage.__name__}] Failed: on follow (worker stage) with message: [{error_msg}]'
# check geopackage file was created on storage
file_location = res['fileURI']
_log.debug('File uri expected: %s', file_location)
gpkg_exist, pkg_url = exc.is_geopackage_exist(file_location,
request=request)
assert gpkg_exist, \
f'Test: [{test_export_geopackage.__name__}] Failed: file not exist on storage [disk | S3 ]:[{pkg_url}]'
# validate the package was created properly
is_valid_package = exc.validate_geo_package(pkg_url)
assert is_valid_package, \
f'Test: [{test_export_geopackage.__name__}] Failed: package is corrupted [{pkg_url}]'
# if config.DEV_MODE and content.get('uuid',None):
# exc.delete_requests(config.EXPORT_STORAGE_URL, [content['uuid']])
_log.info('Finish running test: %s', test_export_geopackage.__name__)
def test_sanity_export_e2e():
"""
This test provide End-To-End exporting process of geopackage and use all functionality to validate deployment
environment was set properly
"""
_log.info('Start running test: %s', test_sanity_export_e2e.__name__)
# check and load request json
request = request_sampels.get_request_sample('_req_ci_cd')
assert request, \
f'Test: [{test_sanity_export_e2e.__name__}] Failed: File not exist or failure on loading request json'
# sending json request and validating export process
request = json.loads(request)
request['fileName'] = 'ci_cd_test' + Z_TIME
request = json.dumps(request)
s_code, content = exc.send_export_request(request)
message = _get_status_message(s_code)
assert s_code == config.ResponseCode.Ok.value, \
f'Test: [{test_sanity_export_e2e.__name__}] Failed: Exporter trigger return status code [{s_code}] and content: [{message}] '
# worker stage - follow exporting via storage statuses API
try:
res = exc.exporter_follower(content['uuid'])
error_msg = None
except RuntimeError as e:
res = None
error_msg = str(e)
except Exception as e:
res = None
error_msg = str(e)
assert res, \
f'Test: [{test_sanity_export_e2e.__name__}] Failed: on follow (worker stage) with message: [{error_msg}]'
# check geopackage file was created on storage
file_location = res['fileURI']
_log.debug('File uri expected: %s', file_location)
gpkg_exist, pkg_url = exc.is_geopackage_exist(file_location,
request=request)
assert gpkg_exist, \
f'Test: [{test_sanity_export_e2e.__name__}] Failed: file not exist on storage [disk | S3 ]:[{pkg_url}]'
# validate the package was created properly
is_valid_package = exc.validate_geo_package(pkg_url)
assert is_valid_package, \
f'Test: [{test_sanity_export_e2e.__name__}] Failed: package is corrupted [{pkg_url}]'
_log.info('Finish running test: %s', test_sanity_export_e2e.__name__)
# test_sanity_export_e2e()
# test_environment_validation()
def test_export_on_storage(get_config, Zapi_cloud_client, test_status,
issue_id, get_execution):
""" 8. Test case: Run export request and create geopackage on shared folder|S3
-----------------------------------------------------------------------------
Validate requirement 1 – server side – 1 Export API
-----------------------------------------------------------------------------
This test check if the package been exported to configurable directory of shared folder|S3
and local downloading functionality (depends on worker configuration
"""
_log.info('Start running test: %s', test_export_on_storage.__name__)
# loading request
request = request_sampels.get_request_by_box_size(
request_sampels.BoxSize.Sanity)
assert request, \
f'Test: [{test_export_on_storage.__name__}] Failed: File not exist or failure on loading request json'
request = (json.loads(request))
request['fileName'] = 'test_case_8_exporter_api'
# request['directoryName'] = config.EXPORT_DOWNLOAD_DIR_NAME
# start trigger export
s_code, content = exc.send_export_request(json.dumps(request))
assert s_code == config.ResponseCode.Ok.value, \
f'Test: [{test_export_on_storage.__name__}] Failed: Exporter trigger return status code [{s_code}]'
# check exporting process and wait till end with results
res = None
try:
err = "unknown"
res = exc.exporter_follower(config.EXPORT_STORAGE_URL, content['uuid'])
uuids.append(content['uuid'])
except Exception as e:
err = str(e)
assert res, \
f'Test: [{test_export_on_storage.__name__}] Failed: on follow (worker stage) with message: [{err}]'
# validate file places on storage - this is download url
file_location = res.get('fileURI')
assert file_location, \
f'Test: [{test_export_on_storage.__name__}] Failed: download link not exist | created]'
_log.debug('File uri expected: %s', file_location)
gpkg_exist, pkg_url = exc.is_geopackage_exist(file_location,
request=request)
assert gpkg_exist, \
f'Test: [{test_export_on_storage.__name__}] Failed: file not exist on storage [disk | S3 ]:[{pkg_url}]'
_log.info('Finish running test: %s', test_export_on_storage.__name__)
def test_n_multi_workers():
""" 12. Test case: validate N workers runs parallel for exporting jobs
"""
_log.info('Start running test: %s', test_n_multi_workers.__name__)
# generate export requests
n_requests = config.RUNNING_WORKERS_NUMBER
_log.info('Parallel exporting will execute %d jobs parallel', n_requests)
requests = [request_sampels.get_long_running_request()]
if n_requests > 1:
n = n_requests - 1
while n > 0:
requests.append(
request_sampels.get_lod_req(
request_sampels.ZoomLevels.default))
n -= 1
new_req_array = list()
for idx, request in enumerate(requests):
assert request, \
f'Test: [{test_n_multi_workers.__name__}] Failed: File not exist or failure on loading request json'
request = (json.loads(request))
request['fileName'] = 'test_case_exporter_api_parallel_run_' + str(
idx) + Z_TIME
new_req_array.append(request)
requests = new_req_array
for idx, request in enumerate(requests):
s_code, content = exc.send_export_request(json.dumps(request))
assert s_code == config.ResponseCode.Ok.value, \
f'Test: [{test_n_multi_workers.__name__}] Failed: Exporter trigger return status code [{s_code}]'
time.sleep(10)
try:
status, progress = exc.get_single_export_state(content['uuid'])
uuids.append(content['uuid'])
except Exception as e:
err = str(e)
raise Exception(err)
assert status != config.EXPORT_STATUS_PENDING and status != config.EXPORT_STATUS_FAILED, \
f'Test: [{test_n_multi_workers.__name__}] Failed: task not executed : [{status}]'
def test_export_by_lod(get_config, Zapi_cloud_client, test_status, issue_id,
get_execution):
""" 12. Test case: send export request by zoom level (LOD – level of details)
-----------------------------------------------------------------------------
Validate requirement 7+11 – server side – 1 Export API
-----------------------------------------------------------------------------
This test sending export requests by different max zoom level.
"""
_log.info('Start running test: %s', test_export_by_lod.__name__)
request = request_sampels.get_lod_req(request_sampels.ZoomLevels.default)
assert request, \
f'Test: [{test_export_by_lod.__name__}] Failed: File not exist or failure on loading request json'
request = (json.loads(request))
request['fileName'] = 'test_case_12_exporter_api_ZoomDefault'
# start trigger export
s_code, content = exc.send_export_request(json.dumps(request))
assert s_code == config.ResponseCode.Ok.value, \
f'Test: [{test_export_by_lod.__name__}] Failed: Exporter trigger return status code [{s_code}]'
# check exporting process and wait till end with results
res = None
try:
err = "unknown"
res = exc.exporter_follower(config.EXPORT_STORAGE_URL, content['uuid'])
uuids.append(content['uuid'])
except Exception as e:
err = str(e)
assert res, \
f'Test: [{test_export_by_lod.__name__}] Failed: on follow (worker stage) with message: [{err}]'
# check geopackage file was created on storage
file_location = res['fileURI']
_log.debug('File uri expected: %s', file_location)
gpkg_exist, pkg_url = exc.is_geopackage_exist(file_location,
request=request)
assert gpkg_exist, \
f'Test: [{test_export_by_lod.__name__}] Failed: file not exist on storage [disk | S3 ]:[{pkg_url}]'
# validate the package was created properly
is_valid_zoom = exc.validate_zoom_level(pkg_url, request['maxZoom'])
assert is_valid_zoom, \
f'Test: [{test_export_by_lod.__name__}] Failed: package is corrupted or wrong max zoom data[{pkg_url}]'
request = request_sampels.get_lod_req(request_sampels.ZoomLevels.med)
assert request, \
f'Test: [{test_export_by_lod.__name__}] Failed: File not exist or failure on loading request json'
request = (json.loads(request))
request['fileName'] = 'test_case_12_exporter_api_ZoomMed'
# start trigger export
s_code, content = exc.send_export_request(json.dumps(request))
assert s_code == config.ResponseCode.Ok.value, \
f'Test: [{test_export_by_lod.__name__}] Failed: Exporter trigger return status code [{s_code}]'
# check exporting process and wait till end with results
res = None
try:
res = exc.exporter_follower(config.EXPORT_STORAGE_URL, content['uuid'])
uuids.append(content['uuid'])
except Exception as e:
err = str(e)
assert res, \
f'Test: [{test_export_by_lod.__name__}] Failed: on follow (worker stage) with message: [{err}]'
# check geopackage file was created on storage
file_location = res['fileURI']
_log.debug('File uri expected: %s', file_location)
gpkg_exist, pkg_url = exc.is_geopackage_exist(file_location,
request=request)
assert gpkg_exist, \
f'Test: [{test_export_by_lod.__name__}] Failed: file not exist on storage [disk | S3 ]:[{pkg_url}]'
# validate the package was created properly
is_valid_zoom = exc.validate_zoom_level(pkg_url, request['maxZoom'])
assert is_valid_zoom, \
f'Test: [{test_export_by_lod.__name__}] Failed: package is corrupted or wrong max zoom data[{pkg_url}]'
