def validate_zoom_level(uri, max_zoom_level): """ check if current geopackage provide only zoom that restricted by provided zoom level value""" s_code, downloaded_data = send_download_request(uri) if config.ResponseCode.Ok.value != s_code: raise ConnectionError('Failed on downloading data') file_name = os.path.basename(uri) download_local_url = common.combine_url( config.TMP_DIR, "_".join(["zoom", str(max_zoom_level)])) full_location = common.combine_url(download_local_url, file_name) if not os.path.exists(download_local_url): os.makedirs(download_local_url) with open(full_location, "wb") as f: f.write(downloaded_data) # if config.S3_EXPORT_STORAGE_MODE: # s3_conn = s3.S3Client(config.S3_END_POINT, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) # object_key = "/".join(uri.split("/")[-2:]) # # destination_dir = os.path.join(config.S3_DOWNLOAD_DIRECTORY, object_key.split('.')[0]) # if not os.path.exists(destination_dir): # os.makedirs(destination_dir) # # s3_conn.download_from_s3(config.S3_BUCKET_NAME, object_key, os.path.join(destination_dir, destination_dir.split('/')[-1])) # uri = os.path.join(destination_dir, destination_dir.split('/')[-1]) # else: # FS # if not os.path.exists(config.PACKAGE_OUTPUT_DIR): # _logger.error( # "Output directory not exist [%s]- validate mapping and directory on config", config.PACKAGE_OUTPUT_DIR) # raise Exception("Output directory: [%s] not found ! validate config or mapping" % config.PACKAGE_OUTPUT_DIR) res = gpv.validate_zoom_levels(full_location, max_zoom_level) res = set(res) return max(res) <= max_zoom_level
def create_testing_status(directory_name, fileName): """ mock helper function that create status on storage""" current_time = datetime.utcnow() utc_curr = current_time.utcnow().strftime('%Y-%m-%d %H:%M:%SZ') utc_expierd = (current_time + timedelta(hours=1)).strftime('%Y-%m-%d %H:%M:%SZ') body = { 'taskId': common.generate_uuid(), 'userId': 'deletion_test', 'fileName': fileName.split('.')[0], 'directoryName': directory_name, 'fileURI': common.combine_url(config.DOWNLOAD_STORAGE_URL, config.DOWNLOAD_API, directory_name, fileName), 'progress': 100, 'sourceLayer': config.SOURCE_LAYER, 'status': config.EXPORT_STATUS_COMPLITED, "geometry": { "type": "Polygon", "coordinates": [[[34.8119380171075, 31.9547503375918], [34.822372617076, 31.9547503375918], [34.822372617076, 31.9642696217735], [34.8119380171075, 31.9642696217735], [34.8119380171075, 31.9547503375918]]] }, 'estimatedFileSize': 1500, 'realFileSize': 1500, 'creationTime': utc_curr, 'updatedTime': utc_curr, 'expirationTime': utc_expierd, } resp = br.send_post_request( common.combine_url(config.EXPORT_STORAGE_URL, config.STATUSES_API), body) if resp.status_code == config.ResponseCode.Ok.value: _logger.info('Created new task with uuid: %s', (body['taskId'])) _logger.debug('Task was registered as : body %s', body) return resp, body['taskId'] else: _logger.error( 'Error while trying create new task with - status: %d | error: %s', resp.status_code, resp.content) return resp, "None"
def get_tileset_from_s3(identifier, job_id): """ This method download relevant tileset.json directly from s3 by identifier """ try: s3_client = s3.S3Client(config.S3_END_POINT, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) bucket_exists = s3_client.is_bucket_exists(config.S3_BUCKET_NAME) source = common.combine_url(job_id, 'tileset.json') destination_dir = os.path.join(config.TMP_DIR, job_id) if not os.path.exists(destination_dir): os.makedirs(destination_dir) dest = os.path.join(config.TMP_DIR, source) if bucket_exists[0]: try: s3_client.download_from_s3(config.S3_BUCKET_NAME, source, dest) return os.path.join(config.TMP_DIR, job_id,'tileset.json') except Exception as e: _logger.error(f'Failed on downloading {identifier}/tileset.json from S3 with error: {str(e)}') raise Exception(f'Failed on downloading {identifier}/tileset.json from S3 with error: {str(e)}') else: _logger.error(f'Failed on connecting to bucket with error: {bucket_exists[2]}') raise Exception(f'Failed on connecting to bucket with error: {bucket_exists[2]}') except Exception as e: _logger.error(f'Failed on connecting S3 with error: {str(e)}') raise Exception(f'Failed on connecting S3 with error: {str(e)}')
def is_geopackage_exist(file_url, request=None): """ Validation of specific geopackge on S3 or File system """ if config.S3_EXPORT_STORAGE_MODE: _logger.info('Test running on s3 mode') try: s3_conn = s3.S3Client(config.S3_END_POINT, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) except Exception as e: _logger.error('Some error occur one connection to S3') raise e if isinstance(request, str): request = json.loads(request) object_key = ".".join([request['fileName'], config.PACKAGE_EXT]) object_key = "/".join([request['directoryName'], object_key]) res = s3_conn.is_file_exist(config.S3_BUCKET_NAME, object_key) pkg_url = file_url.split( '?' )[0] if '?' in file_url else file_url # todo - update after download link will be return res, pkg_url else: _logger.info('Test running on file-system mode') pkg_url = common.combine_url(config.PACKAGE_OUTPUT_DIR, *(file_url.split('/')[-2:])) _logger.info(pkg_url) res = os.path.exists(pkg_url) return res, pkg_url
def load_gpkg_from_storage(file_name, directory_name): """ - This function load to memory geopackage by provided name - Its support FS and OS by running configuration """ if config.S3_EXPORT_STORAGE_MODE: s3_conn = s3.S3Client(config.S3_END_POINT, config.S3_ACCESS_KEY, config.S3_SECRET_KEY) object_key = "/".join( [directory_name, ".".join([file_name, config.PACKAGE_EXT])]) destination_dir = os.path.join(config.S3_DOWNLOAD_DIRECTORY, object_key.split('.')[0]) if not os.path.exists(destination_dir): os.makedirs(destination_dir) s3_conn.download_from_s3( config.S3_BUCKET_NAME, object_key, os.path.join(destination_dir, destination_dir.split('/')[-1])) uri = os.path.join(destination_dir, destination_dir.split('/')[-1]) else: uri = common.combine_url( config.PACKAGE_OUTPUT_DIR, config.EXPORT_DOWNLOAD_DIR_NAME, ".".join([config.EXPORT_DOWNLOAD_FILE_NAME, config.PACKAGE_EXT])) pkg = common.load_file_as_bytearray(uri) return pkg
def get_task_status(uuid): """ This method provide task status by providing uuid (task id) """ res = su.get_uuid_status( common.combine_url(config.EXPORT_STORAGE_URL, config.STATUSES_API), uuid) return res
def get_all_statuses(url): """ This method return all statuses on db :param url: api's url :return: list[dict] """ full_url = common.combine_url(url, config.STATUSES_API) resp = conn.send_get_request(full_url) return resp
def delete_by_uuid(url, uuid): """ This delete from common storage db the status of specific uuid given as list :param url: api's url :param uuid: list of strings represent uuid of task to delete :return: status request response type """ full_url = common.combine_url(url, config.STATUSES_API, config.DELETE_API) resp = conn.send_post_request(full_url, uuid) return resp
def post_model_ingestion_job(self, request): """This method start and trigger new ingestion process of 3rd model""" if not isinstance(request, dict): _log.error( f'Request should be provided as valid json format:\n{request} => {type(request)}' ) raise TypeError('Request should be provided as valid json format') full_model_ingestion_url = common.combine_url( self._ingestion_stack_url, config.INGESTION_3RD_MODEL) resp = br.send_post_request(full_model_ingestion_url, body=request) return resp
def get_single_job_status(self, job_id): """This method return specific ingestion job status""" if not isinstance(job_id, str): _log.error( f'should be provided job id string expressed as uuid:\n{job_id} => {type(job_id)}' ) raise TypeError('Request should be provided as valid json format') job_model_ingestion_url = common.combine_url( self._ingestion_job_service_url, config.INGESTION_3RD_JOB_STATUS, job_id) try: resp = br.send_get_request(job_model_ingestion_url) return resp except Exception as e: _log.error( f'Error on get response from ingestion job progress service with error {str(e)}' )
def get_single_3rd_metadata(self, identifier): """This method return specific exists metadata from catalog db""" if not isinstance(identifier, str): _log.error( f'should be provided identifier expressed as string:\n{identifier} => {type(identifier)}' ) raise TypeError( 'identifier should be provided as valid json format') model_metadata_on_catalog_url = common.combine_url( self._ingestion_catalog_url, config.INGESTION_CATALOG_MODEL_DATA, identifier) try: resp = br.send_get_request(model_metadata_on_catalog_url) return resp except Exception as e: _log.error( f'Error on get response from catalog db service with error {str(e)}' ) raise e
def test_environment_validation(): """This test validate basic pre-running validation of exporter environment micro-services""" services_list = [] # fields to test: exporter_ui = config_dev.EXPORT_UI_URL trigger_api = common.combine_url(config_dev.EXPORT_TRIGGER_URL, config.GET_EXPORT_STATUSES_API) map_proxy = config_dev.MAP_PROXY_URL s3_end_point = config_dev.S3_END_POINT s3_access_key = config_dev.S3_ACCESS_KEY s3_secret_key = config_dev.S3_SECRET_KEY s3_bucket_name = config_dev.S3_BUCKET_NAME postgress_vm = config_dev.POSTGRESS_VM kafka_vm = config_dev.KAFKA_VM _log.info( 'Pre - Running test series that validate environment:\n' 'Will check :\n' f' 1) Exporter ui - {exporter_ui}\n' f' 2) Trigger API - {trigger_api}\n' f' 3) Map_proxy - {map_proxy}\n' f' 4) S3 client - End point: {s3_end_point}, Bucket: {s3_bucket_name}\n' f' 5) Postgress - VM alive : address: {postgress_vm}\n' f' 6) ELK - VM - kafka service: address {kafka_vm}\n') ui_ok = common.check_url_exists(exporter_ui, config_dev.HTTP_REQ_TIMEOUT) ui_ok['name'] = 'Exporter UI route' services_list.append(ui_ok) trigger_ok = common.check_url_exists(trigger_api, config_dev.HTTP_REQ_TIMEOUT) trigger_ok['name'] = 'Trigger route' services_list.append(trigger_ok) map_proxy_ok = common.check_url_exists(map_proxy, config_dev.HTTP_REQ_TIMEOUT) map_proxy_ok['name'] = 'Map-Proxy' services_list.append(map_proxy_ok) s3_ok = s3storage.check_s3_valid(s3_end_point, s3_access_key, s3_secret_key, s3_bucket_name) s3_ok['name'] = 'S3 - minio' services_list.append(s3_ok) postgress_vm_ok = common.ping_to_ip(postgress_vm) if postgress_vm_ok: ssh_conn = bash_utils.ssh_to_machine(postgress_vm, config_dev.AZURE_USER_NAME, config_dev.AZURE_PASSWORD) if ssh_conn: postgress_vm_ok = environment_validators.is_postgress_alive( ssh_conn) if postgress_vm_ok: postgress_vm_ok = { 'url_valid': True, 'status_code': None, 'content': None, 'error_msg': None } else: postgress_vm_ok = { 'url_valid': False, 'status_code': None, 'content': None, 'error_msg': 'Postgress service not running|activated' } else: postgress_vm_ok = { 'url_valid': False, 'status_code': None, 'content': None, 'error_msg': 'Postgress vm ' 'not reachable' } else: postgress_vm_ok = { 'url_valid': False, 'status_code': None, 'content': None, 'error_msg': 'Postgress vm not ' 'reachable' } postgress_vm_ok['name'] = 'Postgress DB' services_list.append(postgress_vm_ok) kafka_vm_ok = common.ping_to_ip(kafka_vm) if kafka_vm_ok: kafka_vm_ok = bash_utils.listen_to_port(kafka_vm, config_dev.KAFKA_PORT) if kafka_vm_ok: kafka_vm_ok = { 'url_valid': True, 'status_code': None, 'content': None, 'error_msg': None } else: kafka_vm_ok = { 'url_valid': False, 'status_code': None, 'content': None, 'error_msg': f"can't reached connection to kafka port {config_dev.KAFKA_PORT}, probably service not running" } else: kafka_vm_ok = { 'url_valid': False, 'status_code': None, 'content': None, 'error_msg': 'kafka vm not reachable' } kafka_vm_ok['name'] = 'KAFKA SERVICE' services_list.append(kafka_vm_ok) final_res = _analyze_system_results(services_list) passed = "--> " + "\n--> ".join( final_res['pass']) if final_res['pass'] else "\n--> ".join( final_res['pass']) failed = "--> " + "\n--> ".join( final_res['failure']) if final_res['failure'] else "\n--> ".join( final_res['failure']) _log.info( '\n-------------------------------------------------------------------------------------------' '\nEnvironment status:\n' f'\npassed:[{len(final_res["pass"])}]\n' f'{passed}\n' f'\n-------------------------------------------------------------------------------------------\n' f'failed:[{len(final_res["failure"])}]\n' f'{failed}' f'\n-------------------------------------------------------------------------------------------\n' ) assert final_res['state'], "Bad environment's prerequisites"
def get_uuid_status(url, uuid): """This method return current state of export task by uuid created""" full_url = common.combine_url(url, uuid) resp = conn.send_get_request(full_url) return resp
def __init__(self): self._base_url = config.EXPORT_TRIGGER_URL self._export_url = common.combine_url(self._base_url, config.EXPORT_GEOPACKAGE_API) self._get_status_url = common.combine_url( self._base_url, config.GET_EXPORT_STATUSES_API)