def delete(self, user_id): """Delete a specific user These methods work only if the authorized user has an admin role. Args: user_id (str): The unique name of the user Returns: flask.Response: A HTTP response with JSON payload containing the status and messages """ user = ActiniaUser(user_id) if user.exists() != 1: return make_response(jsonify(SimpleResponseModel( status="error", message="Unable to delete user %s. User does not exist." % user_id )), 400) if user.delete() is True: return make_response(jsonify(SimpleResponseModel( status="success", message="User %s deleted" % user_id )), 200) return make_response(jsonify(SimpleResponseModel( status="error", message="Unable to delete user %s" % user_id )), 400)
def delete(self, user_id, resource_id): """Request the termination of a resource.""" ret = self.check_permissions(user_id=user_id) if ret: return ret if not resource_id.startswith('resource_id-'): resource_id = 'resource_id-%s' % resource_id iteration, doc = self.resource_logger.get_latest_iteration( user_id, resource_id) if doc is None: return make_response( jsonify( SimpleResponseModel(status="error", message="Resource does not exist")), 400) self.resource_logger.commit_termination(user_id, resource_id, iteration) return make_response( jsonify( SimpleResponseModel(status="accepted", message="Termination request committed")), 200)
def delete(self, location_name): """Delete an existing location and everything inside from the user database. """ # Delete only locations from the user database location = os_path_normpath( [self.grass_user_data_base, self.user_group, location_name]) permanent_mapset = os_path_normpath([location, "PERMANENT"]) wind_file = os_path_normpath([permanent_mapset, "WIND"]) # Check the location path, only "valid" locations can be deleted if os.path.isdir(location): if os.path.isdir(permanent_mapset) and os.path.isfile(wind_file): try: shutil.rmtree(location) return make_response( jsonify( SimpleResponseModel(status="success", message="location %s deleted" % location_name)), 200) except Exception as e: return make_response( jsonify( SimpleResponseModel( status="error", message= "Unable to delete location %s Exception %s" % (location_name, str(e)))), 500) return make_response( jsonify( SimpleResponseModel(status="error", message="location %s does not exists" % location_name)), 400)
def get(self, user_id, resource_id): """Get the maximum size of mapset of a resource.""" ret = self.check_permissions(user_id=user_id) if ret: return ret response_data = self.resource_logger.get(user_id, resource_id) if response_data is not None: http_code, pc_response_model = pickle.loads(response_data) pc_status = pc_response_model['status'] if pc_status in ['accepted', 'running']: return make_response(jsonify(SimpleResponseModel( status="error", message="Resource is not ready it is %s" % pc_status)), 400) mapset_sizes = [ proc['mapset_size'] for proc in pc_response_model['process_log']] max_mapset_size = max(mapset_sizes) return make_response(jsonify(MaxMapsetSizeResponseModel( status="success", max_mapset_size=max_mapset_size)), http_code) else: return make_response(jsonify(SimpleResponseModel( status="error", message="Resource does not exist")), 400)
def _check_possibility_of_new_iteration(self, response_model, user_id, resource_id): """Check if it possible to start a new iteration of the process chain of the resource_id. A new iteration is only possible if the status of the resource is error or terminated, or running but the time_delta does not change any more. Args: response_model (ProcessingResponseModel): The processing response model of the old iteration of the resource user_id (str): The unique user name/id resource_id (str): The id of the resource """ error_msg = None if response_model is None: error_msg = "Resource has no response model" if response_model['status'] in ['accepted', 'finished']: error_msg = f"Resource is {response_model['status']} PUT not possible" elif response_model['status'] in ['running']: sleep(5) _, response_data2 = self.resource_logger.get_latest_iteration( user_id, resource_id) if response_data2 is None: error_msg = "Resource does not exist" return make_response( jsonify( SimpleResponseModel( status="error", message="Resource does not exist")), 400) _, response_model2 = pickle.loads(response_data2) if response_model2 is None: return make_response( jsonify( SimpleResponseModel( status="error", message="Resource has no response model")), 400) # check time_delta and set not running processes to error if response_model['time_delta'] == response_model2['time_delta']: iteration = response_model2['iteration'] if 'iteration' in \ response_model2 else None response_model2['status'] = 'error' response_model2['message'] = "The process no longer seems to be " \ "running and has therefore been set to error." redis_return = self.resource_logger.commit( user_id, resource_id, iteration, pickle.dumps([200, response_model2])) if redis_return is True: pass else: error_msg = "Resource is running and can not be set to error" else: error_msg = "Resource is running no restart possible" elif response_model['status'] in ['error', 'terminated']: pass return error_msg
def post(self, location_name, mapset_name, raster_name): """Create a new raster layer by uploading a GeoTIFF """ allowed_extensions = ['tif', 'tiff'] # TODO check if another content type can be used content_type = request.content_type.split(';')[0] if content_type != "multipart/form-data": return make_response(jsonify(SimpleResponseModel( status="error", message="Content type is not 'multipart/form-data'")), 400) if 'file' not in request.files: return make_response(jsonify(SimpleResponseModel( status="error", message="No file part indicated in postbody.")), 400) # create download cache path if does not exists if os.path.exists(self.download_cache): pass else: os.mkdir(self.download_cache) # save file from request id = str(uuid4()) file = request.files['file'] if file.filename == '': return make_response(jsonify(SimpleResponseModel( status="error", message="No selected file")), 400) if allowed_file(file.filename, allowed_extensions): name, extension = secure_filename(file.filename).rsplit('.', 1) filename = f"{name}_{id}.{extension}" file_path = os.path.join(self.download_cache, filename) file.save(file_path) else: os.remove(file_path) return make_response(jsonify(SimpleResponseModel( status="error", message="File has a not allowed extension. " f"Please use {','.join(allowed_extensions)}.")), 400) rdc = self.preprocess(has_json=False, has_xml=False, location_name=location_name, mapset_name=mapset_name, map_name=raster_name) if rdc: rdc.set_request_data(file_path) enqueue_job(self.job_timeout, start_create_job, rdc) http_code, response_model = pickle.loads(self.response_data) return make_response(jsonify(response_model), http_code)
def get(self, user_id, resource_id): """Render the step-by-step mapset size differences of a resource.""" ret = self.check_permissions(user_id=user_id) if ret: return ret response_data = self.resource_logger.get(user_id, resource_id) if response_data is not None: http_code, pc_response_model = pickle.loads(response_data) pc_status = pc_response_model['status'] if pc_status in ['accepted', 'running']: return make_response(jsonify(SimpleResponseModel( status="error", message="Resource is not ready, it is %s" % pc_status)), 400) mapset_sizes = [ proc['mapset_size'] for proc in pc_response_model['process_log']] diffs = compute_mapset_size_diffs(mapset_sizes) y = np.array(diffs) x = np.array(list(range(1, len(mapset_sizes) + 1))) unit = "bytes" for new_unit in ['KB', 'MB', 'GB', 'TB']: if max(y) > 1024.: y = y / 1024. unit = new_unit print(new_unit) else: break # create png result_file = create_scatter_plot( x, y, 'process chain steps', 'mapset size [%s]' % unit, 'Step-by-step mapset size differences of the resource\n%s' % resource_id) if result_file: if os.path.isfile(result_file): image = open(result_file, "rb").read() os.remove(result_file) return Response(image, mimetype='image/png') else: return make_response(jsonify(SimpleResponseModel( status="error", message="Resource does not exist")), 400)
def get(self, user_id): """Return the credentials of a single user These methods work only if the authorized user has an admin role. Args: user_id (str): The unique name of the user Returns: flask.Response: A HTTP response with JSON payload containing the credentials of the user """ user = ActiniaUser(user_id) if user.exists() != 1: return make_response(jsonify(SimpleResponseModel( status="error", message="User <%s> does not exist" % user_id )), 400) credentials = user.get_credentials() return make_response(jsonify(UserInfoResponseModel( status="success", permissions=credentials["permissions"], user_id=credentials["user_id"], user_role=credentials["user_role"], user_group=credentials["user_group"] )), 200)
def _create_ResourceDataContainer_for_resumption(self, post_url, pc_step, user_id, resource_id, iteration): """Create the ResourceDataContainer for the resumption of the resource depending on the post_url Args: post_url (str): The request url of the original resource pc_step (int): The number of the process chain step where to continue the process user_id (str): The unique user name/id resource_id (str): The id of the resource iteration (int): The number of iteration of this resource Returns: rdc (ResourceDataContainer): The data container that contains all required variables for processing processing_resource (AsyncEphemeralResource/AsyncPersistentResource/ AsyncEphemeralExportResource): The processing resource start_job (function): The start job function of the processing_resource """ interim_result = InterimResult(user_id, resource_id, iteration) if interim_result.check_interim_result_mapset(pc_step, iteration - 1) is None: return None, None, None processing_type = post_url.split('/')[-1] location = re.findall(r'locations\/(.*?)\/', post_url)[0] if processing_type.endswith( 'processing_async') and 'mapsets' not in post_url: # /locations/<string:location_name>/processing_async from .ephemeral_processing import AsyncEphemeralResource, start_job processing_resource = AsyncEphemeralResource( resource_id, iteration, post_url) rdc = processing_resource.preprocess(location_name=location) elif processing_type.endswith( 'processing_async') and 'mapsets' in post_url: # /locations/{location_name}/mapsets/{mapset_name}/processing_async from .persistent_processing import AsyncPersistentResource, start_job processing_resource = AsyncPersistentResource( resource_id, iteration, post_url) mapset = re.findall(r'mapsets\/(.*?)\/', post_url)[0] rdc = processing_resource.preprocess(location_name=location, mapset_name=mapset) elif processing_type.endswith('processing_async_export'): # /locations/{location_name}/processing_async_export from .ephemeral_processing_with_export import \ AsyncEphemeralExportResource, start_job processing_resource = AsyncEphemeralExportResource( resource_id, iteration, post_url) rdc = processing_resource.preprocess(location_name=location) else: return make_response( jsonify( SimpleResponseModel( status="error", message= f"Processing endpoint {post_url} does not support put") ), 400) return rdc, processing_resource, start_job
def get(self, user_id): """Get a list of all API calls that have been called by the provided user.""" if self.user_role not in ["admin", "superadmin" ] and user_id != self.user_id: return make_response( jsonify( SimpleResponseModel( status="error", message="You do not have the permission " "to list the API calls of the user")), 401) api_log_list = self.api_logger.list(user_id, 0, -1) return make_response( jsonify(ApiLogListModel(api_log_list=api_log_list)), 200)
def post(self, user_id): """Create a user in the database These methods work only if the authorized user has an admin role. Args: user_id (str): The unique name of the user Returns: flask.Response: A HTTP response with JSON payload containing the status and messages """ # Password parser password_parser = reqparse.RequestParser() password_parser.add_argument('password', required=True, type=str, help='The password of the new user') password_parser.add_argument('group', required=True, type=str, help='The group of the new user') args = password_parser.parse_args() password = args["password"] group = args["group"] user = ActiniaUser.create_user(user_id, group, password, "user", {}) if user is not None: if user.exists(): return make_response(jsonify(SimpleResponseModel( status="success", message="User %s created" % user_id )), 201) return make_response(jsonify(SimpleResponseModel( status="error", message="Unable to create user %s" % user_id )), 400)
def get(self): """Get a list of all available locations """ locations = [] if os.path.isdir(self.grass_data_base): dirs = os.listdir(self.grass_data_base) for dir in dirs: dir_path = os.path.join(self.grass_data_base, dir) if (os.path.isdir(dir_path) and os.access(dir_path, os.R_OK & os.X_OK)): # Check for PERMANENT mapset existence mapset_path = os.path.join(dir_path, "PERMANENT") if (os.path.isdir(mapset_path) and os.access(mapset_path, os.R_OK & os.X_OK)): # Check access rights to the global database # Super admin can see all locations if (self.has_superadmin_role or dir in self.user_credentials["permissions"] ["accessible_datasets"]): locations.append(dir) # List all locations in the user database user_database = os.path.join(self.grass_user_data_base, self.user_group) if os.path.isdir(user_database): dirs = os.listdir(user_database) for dir in dirs: dir_path = os.path.join(user_database, dir) if os.path.isdir(dir_path) and os.access( dir_path, os.R_OK & os.X_OK): # Check for PERMANENT mapset existence mapset_path = os.path.join(dir_path, "PERMANENT") if (os.path.isdir(mapset_path) and os.access(mapset_path, os.R_OK & os.X_OK)): locations.append(dir) if locations: return make_response( jsonify( LocationListResponseModel(status="success", locations=locations)), 200) else: return make_response( jsonify( SimpleResponseModel( status="error", message="Unable to access GRASS database.")), 405)
def get(self, user_id, resource_id, iteration): """Get the status of a resource of a given iteration.""" ret = self.check_permissions(user_id=user_id) if ret: return ret if not resource_id.startswith('resource_id-'): resource_id = 'resource_id-%s' % resource_id response_data = self.resource_logger.get(user_id, resource_id, int(iteration)) if response_data is not None: _, tmp_response_model = pickle.loads(response_data) response_model = {str(iteration): tmp_response_model} return make_response(jsonify(response_model), 200) else: return make_response( jsonify( SimpleResponseModel(status="error", message="Resource does not exist")), 400)
def delete(self, user_id): """Terminate all accepted and running resources of the specified user.""" ret = self.check_permissions(user_id=user_id) if ret: return ret resource_list = self._get_resource_list(user_id) termination_requests = 0 for entry in resource_list: if "status" in entry: if entry["status"] in ["accepted", "running"]: self.resource_logger.commit_termination( user_id, entry["resource_id"]) termination_requests += 1 return make_response( jsonify( SimpleResponseModel( status="finished", message="Successfully send %i termination requests" % termination_requests)), 200)
def get(self, user_id, resource_id): """Get the status of a resource.""" ret = self.check_permissions(user_id=user_id) if ret: return ret # the latest iteration should be given if resource_id.startswith('resource_id-'): _, response_data = self.resource_logger.get_latest_iteration( user_id, resource_id) else: response_data = self.resource_logger.get_all_iteration( user_id, 'resource_id-%s' % resource_id) if response_data is not None: http_code, response_model = pickle.loads(response_data) return make_response(jsonify(response_model), http_code) else: return make_response( jsonify( SimpleResponseModel(status="error", message="Resource does not exist")), 400)
def post(self, location_name, mapset_name, vector_name): """Create a new vector layer by uploading a GPKG, zipped Shapefile, or GeoJSON. """ allowed_extensions = ['gpkg', 'zip', 'json', 'geojson'] # TODO check if another content type can be used content_type = request.content_type.split(';')[0] if content_type != "multipart/form-data": return make_response( jsonify( SimpleResponseModel( status="error", message="Content type is not 'multipart/form-data'")), 400) if 'file' not in request.files: return make_response( jsonify( SimpleResponseModel( status="error", message="No file part indicated in postbody.")), 400) # create download cache path if it does not exist if os.path.exists(self.download_cache): pass else: os.mkdir(self.download_cache) # save file from request id = str(uuid4()) file = request.files['file'] if file.filename == '': return make_response( jsonify( SimpleResponseModel(status="error", message="No selected file")), 400) if allowed_file(file.filename, allowed_extensions): name, extension = secure_filename(file.filename).rsplit('.', 1) filename = f"{name}_{id}.{extension}" file_path = os.path.join(self.download_cache, filename) file.save(file_path) else: os.remove(file_path) return make_response( jsonify( SimpleResponseModel( status="error", message="File has a not allowed extension. " f"Please use {','.join(allowed_extensions)}.")), 400) # Shapefile upload as zip if extension == 'zip': unzip_folder = os.path.join(self.download_cache, f'unzip_{id}') with ZipFile(file_path, "r") as zip_ref: zip_ref.extractall(unzip_folder) shp_files = [ entry for entry in os.listdir(unzip_folder) if entry.endswith('.shp') ] if len(shp_files) == 0: return make_response( jsonify( SimpleResponseModel( status="error", message="No .shp file found in zip file.")), 400) elif len(shp_files) > 1: return make_response( jsonify( SimpleResponseModel( status="error", message= f"{len(shp_files)} .shp files found in zip file." "Please put only one in the zip file.")), 400) else: os.remove(file_path) file_path = os.path.join(unzip_folder, shp_files[0]) rdc = self.preprocess(has_json=False, has_xml=False, location_name=location_name, mapset_name=mapset_name, map_name=vector_name) if rdc: rdc.set_request_data(file_path) enqueue_job(self.job_timeout, start_create_job, rdc) http_code, response_model = pickle.loads(self.response_data) return make_response(jsonify(response_model), http_code)
def get(self): if 'status' in request.args: if request.args['status'] == "locked": if self.user.has_superadmin_role() is False: return make_response( jsonify( SimpleResponseModel( status="error", message= ("Unable to list locked mapsets You are not authorized" " for this request. " "Minimum required user role: superadmin"))), 401) redis_interface = RedisLockingInterface() kwargs = dict() kwargs["host"] = global_config.REDIS_SERVER_URL kwargs["port"] = global_config.REDIS_SERVER_PORT if (global_config.REDIS_SERVER_PW and global_config.REDIS_SERVER_PW is not None): kwargs["password"] = global_config.REDIS_SERVER_PW redis_interface.connect(**kwargs) redis_connection = redis_interface.redis_server keys_locked = redis_connection.keys("RESOURCE-LOCK*") redis_interface.disconnect() keys_locked_dec = [key.decode() for key in keys_locked] mapsets_locked = [ "/".join(key.split("/")[-2:]) for key in keys_locked_dec ] try: return make_response( jsonify( LockedMapsetListResponseModel( status="success", message="number of locked mapsets: %s" % len(mapsets_locked), locked_mapsets_list=mapsets_locked)), 200) except Exception as e: return make_response( jsonify( SimpleResponseModel( status="error", message= "Unable to list locked mapsets: Exception %s" % (str(e)))), 500) else: redis_interface = RedisUserInterface() kwargs = dict() kwargs["host"] = global_config.REDIS_SERVER_URL kwargs["port"] = global_config.REDIS_SERVER_PORT if (global_config.REDIS_SERVER_PW and global_config.REDIS_SERVER_PW is not None): kwargs["password"] = global_config.REDIS_SERVER_PW redis_interface.connect(**kwargs) if "user" in request.args: user = request.args["user"] if self.user.has_superadmin_role() is False: redis_interface.disconnect() return make_response( jsonify( SimpleResponseModel( status="error", message= (f"Unable to list mapsets for user {user}: You are not" " authorized for this request. " "Minimum required user role: superadmin"))), 401) else: user = self.user.get_id() locs_mapsets = (redis_interface.get_credentials(user) ["permissions"]["accessible_datasets"]) redis_interface.disconnect() mapsets = [] for location in locs_mapsets: for mapset in locs_mapsets[location]: mapsets.append(f"{location}/{mapset}") try: return make_response( jsonify( MapsetListResponseModel( status="success", available_mapsets=mapsets, )), 200) except Exception as e: return make_response( jsonify( SimpleResponseModel( status="error", message="Unable to list mapsets: Exception %s" % (str(e)))), 500)
def put(self, user_id, resource_id): """Updates/Resumes the status of a resource.""" global_config.read(DEFAULT_CONFIG_PATH) if global_config.SAVE_INTERIM_RESULTS is not True: return make_response( jsonify( SimpleResponseModel( status="error", message= "Interim results are not set in the configureation")), 404) ret = self.check_permissions(user_id=user_id) if ret: return ret # check if latest iteration is found old_iteration, response_data = self.resource_logger.get_latest_iteration( user_id, resource_id) old_iteration = 1 if old_iteration is None else old_iteration if response_data is None: return make_response( jsonify( SimpleResponseModel(status="error", message="Resource does not exist")), 400) # check if a new iteration is possible _, response_model = pickle.loads(response_data) err_msg = self._check_possibility_of_new_iteration( response_model, user_id, resource_id) if err_msg is not None: return make_response( jsonify(SimpleResponseModel(status="error", message=err_msg)), 404) # get step of the process chain pc_step = response_model['progress']['step'] - 1 for iter in range(old_iteration - 1, 0, -1): if iter == 1: old_response_data = self.resource_logger.get( user_id, resource_id) else: old_response_data = self.resource_logger.get( user_id, resource_id, iter) if old_response_data is None: return None _, old_response_model = pickle.loads(old_response_data) pc_step += old_response_model['progress']['step'] - 1 # start new iteration iteration = old_iteration + 1 # use post_url if iteration > 1 if old_iteration and old_iteration == 1: post_url = response_model['api_info']['request_url'] elif old_iteration and old_iteration > 1: post_url = response_model['api_info']['post_url'] else: post_url = None rdc, processing_resource, start_job = \ self._create_ResourceDataContainer_for_resumption( post_url, pc_step, user_id, resource_id, iteration) # enqueue job if rdc: enqueue_job(processing_resource.job_timeout, start_job, rdc) html_code, response_model = pickle.loads( processing_resource.response_data) return make_response(jsonify(response_model), html_code)
def check_permissions(self, user_id): """Check the access rights of the user that calls this API call Permission: - guest and user roles can only access resources of the same user id - admin role are allowed to access resources of users with the same user group, except for superusers - superdamins role can access all resources Args: user_id: Returns: None if permissions granted, a error response if permissions are not fulfilled """ # Superuser are allowed to do everything if self.user.has_superadmin_role() is True: return None # Check permissions for users and guests if self.user_role == "guest" or self.user_role == "user": if self.user_id != user_id: return make_response( jsonify( SimpleResponseModel( status="error", message= "You do not have the permission to access this resource. " "Wrong user.")), 401) new_user = ActiniaUser(user_id=user_id) # Check if the user exists if new_user.exists() != 1: return make_response( jsonify( SimpleResponseModel( status="error", message="The user <%s> does not exist" % user_id)), 400) # Check admin permissions if self.user_role == "admin": # Resources of superusers are not allowed to be accessed if new_user.has_superadmin_role() is True: return make_response( jsonify( SimpleResponseModel( status="error", message= "You do not have the permission to access this resource. " "Wrong user role.")), 401) # Only resources of the same user group are allowed to be accessed if new_user.get_group() != self.user_group: return make_response( jsonify( SimpleResponseModel( status="error", message= "You do not have the permission to access this resource. " "Wrong user group.")), 401) return None