def delete(self, user_id):
"""Delete a specific user
These methods work only if the
authorized user has an admin role.
Args:
user_id (str): The unique name of the user
Returns:
flask.Response: A HTTP response with
JSON payload containing
the status and messages
"""
user = ActiniaUser(user_id)
if user.exists() != 1:
return make_response(jsonify(SimpleResponseModel(
status="error",
message="Unable to delete user %s. User does not exist." % user_id
)), 400)
if user.delete() is True:
return make_response(jsonify(SimpleResponseModel(
status="success",
message="User %s deleted" % user_id
)), 200)
return make_response(jsonify(SimpleResponseModel(
status="error",
message="Unable to delete user %s" % user_id
)), 400)
def delete(self, user_id, resource_id):
"""Request the termination of a resource."""
ret = self.check_permissions(user_id=user_id)
if ret:
return ret
if not resource_id.startswith('resource_id-'):
resource_id = 'resource_id-%s' % resource_id
iteration, doc = self.resource_logger.get_latest_iteration(
user_id, resource_id)
if doc is None:
return make_response(
jsonify(
SimpleResponseModel(status="error",
message="Resource does not exist")),
400)
self.resource_logger.commit_termination(user_id, resource_id,
iteration)
return make_response(
jsonify(
SimpleResponseModel(status="accepted",
message="Termination request committed")),
200)
def delete(self, location_name):
"""Delete an existing location and everything inside from the user database.
"""
# Delete only locations from the user database
location = os_path_normpath(
[self.grass_user_data_base, self.user_group, location_name])
permanent_mapset = os_path_normpath([location, "PERMANENT"])
wind_file = os_path_normpath([permanent_mapset, "WIND"])
# Check the location path, only "valid" locations can be deleted
if os.path.isdir(location):
if os.path.isdir(permanent_mapset) and os.path.isfile(wind_file):
try:
shutil.rmtree(location)
return make_response(
jsonify(
SimpleResponseModel(status="success",
message="location %s deleted" %
location_name)), 200)
except Exception as e:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
"Unable to delete location %s Exception %s" %
(location_name, str(e)))), 500)
return make_response(
jsonify(
SimpleResponseModel(status="error",
message="location %s does not exists" %
location_name)), 400)
def get(self, user_id, resource_id):
"""Get the maximum size of mapset of a resource."""
ret = self.check_permissions(user_id=user_id)
if ret:
return ret
response_data = self.resource_logger.get(user_id, resource_id)
if response_data is not None:
http_code, pc_response_model = pickle.loads(response_data)
pc_status = pc_response_model['status']
if pc_status in ['accepted', 'running']:
return make_response(jsonify(SimpleResponseModel(
status="error",
message="Resource is not ready it is %s" % pc_status)),
400)
mapset_sizes = [
proc['mapset_size'] for proc in pc_response_model['process_log']]
max_mapset_size = max(mapset_sizes)
return make_response(jsonify(MaxMapsetSizeResponseModel(
status="success", max_mapset_size=max_mapset_size)), http_code)
else:
return make_response(jsonify(SimpleResponseModel(
status="error",
message="Resource does not exist")), 400)
def _check_possibility_of_new_iteration(self, response_model, user_id,
resource_id):
"""Check if it possible to start a new iteration of the process chain
of the resource_id. A new iteration is only possible if the status of
the resource is error or terminated, or running but the time_delta does
not change any more.
Args:
response_model (ProcessingResponseModel): The processing response
model of the old iteration
of the resource
user_id (str): The unique user name/id
resource_id (str): The id of the resource
"""
error_msg = None
if response_model is None:
error_msg = "Resource has no response model"
if response_model['status'] in ['accepted', 'finished']:
error_msg = f"Resource is {response_model['status']} PUT not possible"
elif response_model['status'] in ['running']:
sleep(5)
_, response_data2 = self.resource_logger.get_latest_iteration(
user_id, resource_id)
if response_data2 is None:
error_msg = "Resource does not exist"
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="Resource does not exist")), 400)
_, response_model2 = pickle.loads(response_data2)
if response_model2 is None:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="Resource has no response model")), 400)
# check time_delta and set not running processes to error
if response_model['time_delta'] == response_model2['time_delta']:
iteration = response_model2['iteration'] if 'iteration' in \
response_model2 else None
response_model2['status'] = 'error'
response_model2['message'] = "The process no longer seems to be " \
"running and has therefore been set to error."
redis_return = self.resource_logger.commit(
user_id, resource_id, iteration,
pickle.dumps([200, response_model2]))
if redis_return is True:
pass
else:
error_msg = "Resource is running and can not be set to error"
else:
error_msg = "Resource is running no restart possible"
elif response_model['status'] in ['error', 'terminated']:
pass
return error_msg
def post(self, location_name, mapset_name, raster_name):
"""Create a new raster layer by uploading a GeoTIFF
"""
allowed_extensions = ['tif', 'tiff']
# TODO check if another content type can be used
content_type = request.content_type.split(';')[0]
if content_type != "multipart/form-data":
return make_response(jsonify(SimpleResponseModel(
status="error",
message="Content type is not 'multipart/form-data'")), 400)
if 'file' not in request.files:
return make_response(jsonify(SimpleResponseModel(
status="error",
message="No file part indicated in postbody.")), 400)
# create download cache path if does not exists
if os.path.exists(self.download_cache):
pass
else:
os.mkdir(self.download_cache)
# save file from request
id = str(uuid4())
file = request.files['file']
if file.filename == '':
return make_response(jsonify(SimpleResponseModel(
status="error",
message="No selected file")), 400)
if allowed_file(file.filename, allowed_extensions):
name, extension = secure_filename(file.filename).rsplit('.', 1)
filename = f"{name}_{id}.{extension}"
file_path = os.path.join(self.download_cache, filename)
file.save(file_path)
else:
os.remove(file_path)
return make_response(jsonify(SimpleResponseModel(
status="error",
message="File has a not allowed extension. "
f"Please use {','.join(allowed_extensions)}.")), 400)
rdc = self.preprocess(has_json=False, has_xml=False,
location_name=location_name,
mapset_name=mapset_name,
map_name=raster_name)
if rdc:
rdc.set_request_data(file_path)
enqueue_job(self.job_timeout, start_create_job, rdc)
http_code, response_model = pickle.loads(self.response_data)
return make_response(jsonify(response_model), http_code)
def get(self, user_id, resource_id):
"""Render the step-by-step mapset size differences of a resource."""
ret = self.check_permissions(user_id=user_id)
if ret:
return ret
response_data = self.resource_logger.get(user_id, resource_id)
if response_data is not None:
http_code, pc_response_model = pickle.loads(response_data)
pc_status = pc_response_model['status']
if pc_status in ['accepted', 'running']:
return make_response(jsonify(SimpleResponseModel(
status="error",
message="Resource is not ready, it is %s" % pc_status)),
400)
mapset_sizes = [
proc['mapset_size'] for proc in pc_response_model['process_log']]
diffs = compute_mapset_size_diffs(mapset_sizes)
y = np.array(diffs)
x = np.array(list(range(1, len(mapset_sizes) + 1)))
unit = "bytes"
for new_unit in ['KB', 'MB', 'GB', 'TB']:
if max(y) > 1024.:
y = y / 1024.
unit = new_unit
print(new_unit)
else:
break
# create png
result_file = create_scatter_plot(
x, y, 'process chain steps', 'mapset size [%s]' % unit,
'Step-by-step mapset size differences of the resource\n%s'
% resource_id)
if result_file:
if os.path.isfile(result_file):
image = open(result_file, "rb").read()
os.remove(result_file)
return Response(image, mimetype='image/png')
else:
return make_response(jsonify(SimpleResponseModel(
status="error",
message="Resource does not exist")), 400)
def get(self, user_id):
"""Return the credentials of a single user
These methods work only if the
authorized user has an admin role.
Args:
user_id (str): The unique name of the user
Returns:
flask.Response: A HTTP response with
JSON payload containing the credentials
of the user
"""
user = ActiniaUser(user_id)
if user.exists() != 1:
return make_response(jsonify(SimpleResponseModel(
status="error",
message="User <%s> does not exist" % user_id
)), 400)
credentials = user.get_credentials()
return make_response(jsonify(UserInfoResponseModel(
status="success",
permissions=credentials["permissions"],
user_id=credentials["user_id"],
user_role=credentials["user_role"],
user_group=credentials["user_group"]
)), 200)
def _create_ResourceDataContainer_for_resumption(self, post_url, pc_step,
user_id, resource_id,
iteration):
"""Create the ResourceDataContainer for the resumption of the resource
depending on the post_url
Args:
post_url (str): The request url of the original resource
pc_step (int): The number of the process chain step where to
continue the process
user_id (str): The unique user name/id
resource_id (str): The id of the resource
iteration (int): The number of iteration of this resource
Returns:
rdc (ResourceDataContainer): The data container that contains all
required variables for processing
processing_resource (AsyncEphemeralResource/AsyncPersistentResource/
AsyncEphemeralExportResource): The processing resource
start_job (function): The start job function of the processing_resource
"""
interim_result = InterimResult(user_id, resource_id, iteration)
if interim_result.check_interim_result_mapset(pc_step,
iteration - 1) is None:
return None, None, None
processing_type = post_url.split('/')[-1]
location = re.findall(r'locations\/(.*?)\/', post_url)[0]
if processing_type.endswith(
'processing_async') and 'mapsets' not in post_url:
# /locations/<string:location_name>/processing_async
from .ephemeral_processing import AsyncEphemeralResource, start_job
processing_resource = AsyncEphemeralResource(
resource_id, iteration, post_url)
rdc = processing_resource.preprocess(location_name=location)
elif processing_type.endswith(
'processing_async') and 'mapsets' in post_url:
# /locations/{location_name}/mapsets/{mapset_name}/processing_async
from .persistent_processing import AsyncPersistentResource, start_job
processing_resource = AsyncPersistentResource(
resource_id, iteration, post_url)
mapset = re.findall(r'mapsets\/(.*?)\/', post_url)[0]
rdc = processing_resource.preprocess(location_name=location,
mapset_name=mapset)
elif processing_type.endswith('processing_async_export'):
# /locations/{location_name}/processing_async_export
from .ephemeral_processing_with_export import \
AsyncEphemeralExportResource, start_job
processing_resource = AsyncEphemeralExportResource(
resource_id, iteration, post_url)
rdc = processing_resource.preprocess(location_name=location)
else:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
f"Processing endpoint {post_url} does not support put")
), 400)
return rdc, processing_resource, start_job
def get(self, user_id):
"""Get a list of all API calls that have been called by the provided user."""
if self.user_role not in ["admin", "superadmin"
] and user_id != self.user_id:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="You do not have the permission "
"to list the API calls of the user")), 401)
api_log_list = self.api_logger.list(user_id, 0, -1)
return make_response(
jsonify(ApiLogListModel(api_log_list=api_log_list)), 200)
def post(self, user_id):
"""Create a user in the database
These methods work only if the
authorized user has an admin role.
Args:
user_id (str): The unique name of the user
Returns:
flask.Response: A HTTP response with
JSON payload containing
the status and messages
"""
# Password parser
password_parser = reqparse.RequestParser()
password_parser.add_argument('password', required=True,
type=str, help='The password of the new user')
password_parser.add_argument('group', required=True,
type=str, help='The group of the new user')
args = password_parser.parse_args()
password = args["password"]
group = args["group"]
user = ActiniaUser.create_user(user_id, group, password, "user", {})
if user is not None:
if user.exists():
return make_response(jsonify(SimpleResponseModel(
status="success",
message="User %s created" % user_id
)), 201)
return make_response(jsonify(SimpleResponseModel(
status="error",
message="Unable to create user %s" % user_id
)), 400)
def get(self):
"""Get a list of all available locations
"""
locations = []
if os.path.isdir(self.grass_data_base):
dirs = os.listdir(self.grass_data_base)
for dir in dirs:
dir_path = os.path.join(self.grass_data_base, dir)
if (os.path.isdir(dir_path)
and os.access(dir_path, os.R_OK & os.X_OK)):
# Check for PERMANENT mapset existence
mapset_path = os.path.join(dir_path, "PERMANENT")
if (os.path.isdir(mapset_path)
and os.access(mapset_path, os.R_OK & os.X_OK)):
# Check access rights to the global database
# Super admin can see all locations
if (self.has_superadmin_role
or dir in self.user_credentials["permissions"]
["accessible_datasets"]):
locations.append(dir)
# List all locations in the user database
user_database = os.path.join(self.grass_user_data_base,
self.user_group)
if os.path.isdir(user_database):
dirs = os.listdir(user_database)
for dir in dirs:
dir_path = os.path.join(user_database, dir)
if os.path.isdir(dir_path) and os.access(
dir_path, os.R_OK & os.X_OK):
# Check for PERMANENT mapset existence
mapset_path = os.path.join(dir_path, "PERMANENT")
if (os.path.isdir(mapset_path)
and os.access(mapset_path, os.R_OK & os.X_OK)):
locations.append(dir)
if locations:
return make_response(
jsonify(
LocationListResponseModel(status="success",
locations=locations)), 200)
else:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="Unable to access GRASS database.")), 405)
def get(self, user_id, resource_id, iteration):
"""Get the status of a resource of a given iteration."""
ret = self.check_permissions(user_id=user_id)
if ret:
return ret
if not resource_id.startswith('resource_id-'):
resource_id = 'resource_id-%s' % resource_id
response_data = self.resource_logger.get(user_id, resource_id,
int(iteration))
if response_data is not None:
_, tmp_response_model = pickle.loads(response_data)
response_model = {str(iteration): tmp_response_model}
return make_response(jsonify(response_model), 200)
else:
return make_response(
jsonify(
SimpleResponseModel(status="error",
message="Resource does not exist")),
400)
def delete(self, user_id):
"""Terminate all accepted and running resources of the specified user."""
ret = self.check_permissions(user_id=user_id)
if ret:
return ret
resource_list = self._get_resource_list(user_id)
termination_requests = 0
for entry in resource_list:
if "status" in entry:
if entry["status"] in ["accepted", "running"]:
self.resource_logger.commit_termination(
user_id, entry["resource_id"])
termination_requests += 1
return make_response(
jsonify(
SimpleResponseModel(
status="finished",
message="Successfully send %i termination requests" %
termination_requests)), 200)
def get(self, user_id, resource_id):
"""Get the status of a resource."""
ret = self.check_permissions(user_id=user_id)
if ret:
return ret
# the latest iteration should be given
if resource_id.startswith('resource_id-'):
_, response_data = self.resource_logger.get_latest_iteration(
user_id, resource_id)
else:
response_data = self.resource_logger.get_all_iteration(
user_id, 'resource_id-%s' % resource_id)
if response_data is not None:
http_code, response_model = pickle.loads(response_data)
return make_response(jsonify(response_model), http_code)
else:
return make_response(
jsonify(
SimpleResponseModel(status="error",
message="Resource does not exist")),
400)
def post(self, location_name, mapset_name, vector_name):
"""Create a new vector layer by uploading a GPKG, zipped Shapefile,
or GeoJSON.
"""
allowed_extensions = ['gpkg', 'zip', 'json', 'geojson']
# TODO check if another content type can be used
content_type = request.content_type.split(';')[0]
if content_type != "multipart/form-data":
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="Content type is not 'multipart/form-data'")),
400)
if 'file' not in request.files:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="No file part indicated in postbody.")), 400)
# create download cache path if it does not exist
if os.path.exists(self.download_cache):
pass
else:
os.mkdir(self.download_cache)
# save file from request
id = str(uuid4())
file = request.files['file']
if file.filename == '':
return make_response(
jsonify(
SimpleResponseModel(status="error",
message="No selected file")), 400)
if allowed_file(file.filename, allowed_extensions):
name, extension = secure_filename(file.filename).rsplit('.', 1)
filename = f"{name}_{id}.{extension}"
file_path = os.path.join(self.download_cache, filename)
file.save(file_path)
else:
os.remove(file_path)
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="File has a not allowed extension. "
f"Please use {','.join(allowed_extensions)}.")), 400)
# Shapefile upload as zip
if extension == 'zip':
unzip_folder = os.path.join(self.download_cache, f'unzip_{id}')
with ZipFile(file_path, "r") as zip_ref:
zip_ref.extractall(unzip_folder)
shp_files = [
entry for entry in os.listdir(unzip_folder)
if entry.endswith('.shp')
]
if len(shp_files) == 0:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="No .shp file found in zip file.")), 400)
elif len(shp_files) > 1:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
f"{len(shp_files)} .shp files found in zip file."
"Please put only one in the zip file.")), 400)
else:
os.remove(file_path)
file_path = os.path.join(unzip_folder, shp_files[0])
rdc = self.preprocess(has_json=False,
has_xml=False,
location_name=location_name,
mapset_name=mapset_name,
map_name=vector_name)
if rdc:
rdc.set_request_data(file_path)
enqueue_job(self.job_timeout, start_create_job, rdc)
http_code, response_model = pickle.loads(self.response_data)
return make_response(jsonify(response_model), http_code)
def get(self):
if 'status' in request.args:
if request.args['status'] == "locked":
if self.user.has_superadmin_role() is False:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
("Unable to list locked mapsets You are not authorized"
" for this request. "
"Minimum required user role: superadmin"))),
401)
redis_interface = RedisLockingInterface()
kwargs = dict()
kwargs["host"] = global_config.REDIS_SERVER_URL
kwargs["port"] = global_config.REDIS_SERVER_PORT
if (global_config.REDIS_SERVER_PW
and global_config.REDIS_SERVER_PW is not None):
kwargs["password"] = global_config.REDIS_SERVER_PW
redis_interface.connect(**kwargs)
redis_connection = redis_interface.redis_server
keys_locked = redis_connection.keys("RESOURCE-LOCK*")
redis_interface.disconnect()
keys_locked_dec = [key.decode() for key in keys_locked]
mapsets_locked = [
"/".join(key.split("/")[-2:]) for key in keys_locked_dec
]
try:
return make_response(
jsonify(
LockedMapsetListResponseModel(
status="success",
message="number of locked mapsets: %s" %
len(mapsets_locked),
locked_mapsets_list=mapsets_locked)), 200)
except Exception as e:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
"Unable to list locked mapsets: Exception %s" %
(str(e)))), 500)
else:
redis_interface = RedisUserInterface()
kwargs = dict()
kwargs["host"] = global_config.REDIS_SERVER_URL
kwargs["port"] = global_config.REDIS_SERVER_PORT
if (global_config.REDIS_SERVER_PW
and global_config.REDIS_SERVER_PW is not None):
kwargs["password"] = global_config.REDIS_SERVER_PW
redis_interface.connect(**kwargs)
if "user" in request.args:
user = request.args["user"]
if self.user.has_superadmin_role() is False:
redis_interface.disconnect()
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
(f"Unable to list mapsets for user {user}: You are not"
" authorized for this request. "
"Minimum required user role: superadmin"))),
401)
else:
user = self.user.get_id()
locs_mapsets = (redis_interface.get_credentials(user)
["permissions"]["accessible_datasets"])
redis_interface.disconnect()
mapsets = []
for location in locs_mapsets:
for mapset in locs_mapsets[location]:
mapsets.append(f"{location}/{mapset}")
try:
return make_response(
jsonify(
MapsetListResponseModel(
status="success",
available_mapsets=mapsets,
)), 200)
except Exception as e:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="Unable to list mapsets: Exception %s" %
(str(e)))), 500)
def put(self, user_id, resource_id):
"""Updates/Resumes the status of a resource."""
global_config.read(DEFAULT_CONFIG_PATH)
if global_config.SAVE_INTERIM_RESULTS is not True:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
"Interim results are not set in the configureation")),
404)
ret = self.check_permissions(user_id=user_id)
if ret:
return ret
# check if latest iteration is found
old_iteration, response_data = self.resource_logger.get_latest_iteration(
user_id, resource_id)
old_iteration = 1 if old_iteration is None else old_iteration
if response_data is None:
return make_response(
jsonify(
SimpleResponseModel(status="error",
message="Resource does not exist")),
400)
# check if a new iteration is possible
_, response_model = pickle.loads(response_data)
err_msg = self._check_possibility_of_new_iteration(
response_model, user_id, resource_id)
if err_msg is not None:
return make_response(
jsonify(SimpleResponseModel(status="error", message=err_msg)),
404)
# get step of the process chain
pc_step = response_model['progress']['step'] - 1
for iter in range(old_iteration - 1, 0, -1):
if iter == 1:
old_response_data = self.resource_logger.get(
user_id, resource_id)
else:
old_response_data = self.resource_logger.get(
user_id, resource_id, iter)
if old_response_data is None:
return None
_, old_response_model = pickle.loads(old_response_data)
pc_step += old_response_model['progress']['step'] - 1
# start new iteration
iteration = old_iteration + 1
# use post_url if iteration > 1
if old_iteration and old_iteration == 1:
post_url = response_model['api_info']['request_url']
elif old_iteration and old_iteration > 1:
post_url = response_model['api_info']['post_url']
else:
post_url = None
rdc, processing_resource, start_job = \
self._create_ResourceDataContainer_for_resumption(
post_url, pc_step, user_id, resource_id, iteration)
# enqueue job
if rdc:
enqueue_job(processing_resource.job_timeout, start_job, rdc)
html_code, response_model = pickle.loads(
processing_resource.response_data)
return make_response(jsonify(response_model), html_code)
def check_permissions(self, user_id):
"""Check the access rights of the user that calls this API call
Permission:
- guest and user roles can only access resources of the same user id
- admin role are allowed to access resources of users with the same
user group, except for superusers
- superdamins role can access all resources
Args:
user_id:
Returns:
None if permissions granted, a error response if permissions are
not fulfilled
"""
# Superuser are allowed to do everything
if self.user.has_superadmin_role() is True:
return None
# Check permissions for users and guests
if self.user_role == "guest" or self.user_role == "user":
if self.user_id != user_id:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
"You do not have the permission to access this resource. "
"Wrong user.")), 401)
new_user = ActiniaUser(user_id=user_id)
# Check if the user exists
if new_user.exists() != 1:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message="The user <%s> does not exist" % user_id)),
400)
# Check admin permissions
if self.user_role == "admin":
# Resources of superusers are not allowed to be accessed
if new_user.has_superadmin_role() is True:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
"You do not have the permission to access this resource. "
"Wrong user role.")), 401)
# Only resources of the same user group are allowed to be accessed
if new_user.get_group() != self.user_group:
return make_response(
jsonify(
SimpleResponseModel(
status="error",
message=
"You do not have the permission to access this resource. "
"Wrong user group.")), 401)
return None