def complete_validation(self,
run_id: int,
user_id: int = None,
group_id: int = None):
""" Handles the completed validation process.
Handler for completing the validation process. This method
takes care of deleting the PeriodicTask and adding a
new GroupActivity.
Note: This method should be used to handle the completed validation
process (passed, failed). Use complete_validation_error for
handling any kind of unexpected exceptions during the validation.
Args:
run_id (int): The id of the ConformityCheckRun.
Keyword arguments:
user_id (int): The id of the user that triggered the run.
group_id (int): the id of the group that the validated metadata
object belongs to.
Returns:
nothing
"""
parent_task_id = complete_validation.request.parent_id
if parent_task_id is not None:
try:
pt = PendingTask.objects.get(task_id=parent_task_id)
pt.progress = 100
pt.save()
sleep(2)
pt.delete()
except PendingTask.DoesNotExist:
pass
try:
run = ConformityCheckRun.objects.get(pk=run_id)
# task is still running
if run.passed is None:
return
group = MrMapGroup.objects.get(pk=group_id)
user = MrMapUser.objects.get(pk=user_id)
title = _(f'Validation {"succeeded" if run.passed else "failed"}')
href = reverse('resource:detail', args=(run.metadata.pk, ))
content = _(f'for <a href="{href}">'
f'{run.metadata.title}</a> '
f'with <i>{run.conformity_check_configuration}</i>.')
create_group_activity(group=group,
user=user,
msg=title,
metadata_title=content)
except (ConformityCheckRun.DoesNotExist, MrMapGroup.DoesNotExist,
MrMapUser.DoesNotExist) as e:
logger.error("Could not complete validation. ", e)
def complete_validation_error(request,
exc,
traceback,
user_id: int = None,
group_id: int = None,
config_id: int = None,
metadata_id: str = None):
""" Handles the aborted validation process.
Handler for completing the aborted validation process. This method
takes care of deleting the PeriodicTask and adding a
new GroupActivity.
Note: This method should be used to handle the aborted validation
process (e.g. unhandled Exceptions). Use complete_validation for
handling the completed validation process.
Args:
*args: positional arguments
Keyword arguments:
user_id (int): The id of the user that triggered the run.
group_id (int): the id of the group that the validated metadata
object belongs to.
config_id (int): The id of the ConformityCheckConfiguration.
metadata_id (uuid): The id of the validated metadata object.
Returns:
nothing
"""
try:
config = ConformityCheckConfiguration.objects.get(pk=config_id)
metadata = Metadata.objects.get(pk=metadata_id)
group = MrMapGroup.objects.get(pk=group_id)
user = MrMapUser.objects.get(pk=user_id)
# delete run, if it was manually aborted
if isinstance(exc, AbortedException):
try:
run = ConformityCheckRun.objects.get_latest_check(metadata)
run.delete()
except ConformityCheckRun.DoesNotExist:
pass
title = _(f'Validation aborted')
content = _(f'for <a href="resource/detail/{metadata.id}">'
f'{metadata.title}</a> '
f'with <i>{config}</i>.')
create_group_activity(group=group,
user=user,
msg=title,
metadata_title=content)
except (ConformityCheckRun.DoesNotExist, Metadata.DoesNotExist,
MrMapUser.DoesNotExist) as e:
logger.error("Could not complete error task. ", e)
def process_restore_dataset_metadata(self):
ext_auth = self.instance.get_external_authentication_object()
if not self.instance.is_custom:
messages.add_message(self.request, messages.INFO, METADATA_IS_ORIGINAL)
return HttpResponseRedirect(reverse(self.request.GET.get('current-view', 'home')), status=303)
if self.instance.is_custom:
self.instance.restore(self.instance.identifier, external_auth=ext_auth)
self.instance.save()
messages.add_message(self.request, messages.SUCCESS, METADATA_RESTORING_SUCCESS)
user_helper.create_group_activity(self.instance.created_by, self.requesting_user, SERVICE_MD_RESTORED,
"{}".format(self.instance.title, ))
def form_valid(self, form):
self.object = self.get_object()
ext_auth = self.object.get_external_authentication_object()
self.object.restore(self.object.identifier, external_auth=ext_auth)
# Todo: add last_changed_by_user field to Metadata and move this piece of code to Metadata.restore()
if self.object.is_metadata_type(MetadataEnum.DATASET):
user_helper.create_group_activity(self.object.created_by, self.request.user, SERVICE_MD_RESTORED,
"{}".format(self.object.title, ))
else:
user_helper.create_group_activity(self.object.created_by, self.request.user, SERVICE_MD_RESTORED,
"{}: {}".format(self.object.get_root_metadata().title,
self.object.title))
success_url = self.get_success_url()
return HttpResponseRedirect(success_url)
def run_update_service(request: HttpRequest, metadata_id):
if request.method == 'POST':
current_service = get_object_or_404(Service.objects.select_related(
'metadata').prefetch_related('metadata__documents'),
metadata__id=metadata_id)
new_service = get_object_or_404(
Service.objects.select_related('metadata').prefetch_related(
'metadata__documents'),
is_update_candidate_for=current_service)
if not current_service.is_service_type(OGCServiceEnum.WFS):
new_service.root_layer = get_object_or_404(
Layer, parent_service=new_service, parent=None)
current_service.root_layer = get_object_or_404(
Layer, parent_service=current_service, parent=None)
comparator = ServiceComparator(service_a=new_service,
service_b=current_service)
diff = comparator.compare_services()
diff_elements = diff.get("layers", None) or diff.get(
"feature_types", {})
# We need to extract the linkage of new->old elements from the request by hand
# key identifies the new element and it's identifier (not id!) and choice identifies the existing element's id!
links = {}
prefix = "new_elem_"
for key, choice in request.POST.items():
if prefix in key:
links[key.replace(prefix, "")] = choice
update_confirmation_form = UpdateOldToNewElementsForm(
request.POST,
new_elements=diff_elements.get("new"),
removed_elements=diff_elements.get("removed"),
choices=links,
current_service=current_service,
)
update_confirmation_form.action_url = reverse("resource:run-update",
args=[metadata_id])
if update_confirmation_form.is_valid():
# UPDATE
# First update the metadata of the whole service
md = update_helper.update_metadata(current_service.metadata,
new_service.metadata,
new_service.keep_custom_md)
md.save()
current_service.metadata = md
current_service.save()
# Then update the service object
current_service = update_helper.update_service(
current_service, new_service)
current_service.save()
# Update the subelements
if new_service.is_service_type(OGCServiceEnum.WFS):
current_service = update_helper.update_wfs_elements(
current_service, new_service, diff, links,
new_service.keep_custom_md)
elif new_service.is_service_type(OGCServiceEnum.WMS):
# takes long time | todo proof again after using django-mptt
current_service = update_helper.update_wms_elements(
current_service, new_service, diff, links,
new_service.keep_custom_md)
current_service.save()
update_helper.update_capability_document(current_service,
new_service)
current_service.save()
user_helper.create_group_activity(
current_service.metadata.created_by, request.user,
SERVICE_UPDATED, current_service.metadata.title)
new_service.delete()
messages.success(request, SERVICE_UPDATED)
return HttpResponseRedirect(reverse("resource:detail",
args=(metadata_id, )),
status=303)
else:
params = {
"update_confirmation_form": update_confirmation_form,
}
return pending_update_service(request=request,
metadata_id=metadata_id,
update_params=params,
status_code=422)
else:
return HttpResponseRedirect(reverse("resource:pending-update",
args=(metadata_id, )),
status=303)
def async_new_service(url_dict: dict, user_id: int, register_group_id: int,
register_for_organization_id: int, external_auth: dict):
""" Async call of new service creation
Since redis is used as broker, the objects can not be passed directly into the function. They have to be resolved using
their ids, since the objects are not easily serializable using json
Args:
url_dict (dict): Contains basic information about the service like connection uri
user_id (int): Id of the performing user
register_group_id (int): Id of the group which wants to register
register_for_organization_id (int): Id of the organization for which the service is registered
Returns:
nothing
"""
# create ExternalAuthentication object
if external_auth is not None:
external_auth = ExternalAuthentication(
username=external_auth["username"],
password=external_auth["password"],
auth_type=external_auth["auth_type"],
)
# get current task id
curr_task_id = async_new_service.request.id
# set progress for current task to 0
if curr_task_id is not None:
task_helper.update_progress(async_new_service, 0)
# restore objects from ids
user = MrMapUser.objects.get(id=user_id)
url_dict["service"] = service_helper.resolve_service_enum(
url_dict["service"])
url_dict["version"] = service_helper.resolve_version_enum(
url_dict["version"])
register_group = MrMapGroup.objects.get(id=register_group_id)
if utils.resolve_none_string(
str(register_for_organization_id)) is not None:
register_for_organization = Organization.objects.get(
id=register_for_organization_id)
else:
register_for_organization = None
try:
t_start = time.time()
service = service_helper.create_service(url_dict.get("service"),
url_dict.get("version"),
url_dict.get("base_uri"),
user,
register_group,
register_for_organization,
async_task=async_new_service,
external_auth=external_auth)
# update progress
if curr_task_id is not None:
task_helper.update_progress(async_new_service,
PROGRESS_STATUS_AFTER_PARSING)
# get db object
if curr_task_id is not None:
pending_task = PendingTask.objects.get(task_id=curr_task_id)
# update db pending task information
pending_task.description = json.dumps({
"service": service.metadata.title,
"phase": "Persisting",
})
pending_task.save()
# update progress
if curr_task_id is not None:
task_helper.update_progress(async_new_service, 95)
# after service AND documents have been persisted, we can now set the service being secured if needed
if external_auth is not None:
service.metadata.set_proxy(True)
metadatas = Metadata.objects.filter(pk=service.metadata.pk)
sub_elements = service.get_subelements().select_related('metadata')
for sub_element in sub_elements:
metadatas |= Metadata.objects.filter(pk=sub_element.metadata.pk)
metadatas |= sub_element.metadata.get_related_dataset_metadatas()
service_logger.debug(EXEC_TIME_PRINT %
("total registration", time.time() - t_start))
user_helper.create_group_activity(service.metadata.created_by, user,
SERVICE_REGISTERED,
service.metadata.title)
if curr_task_id is not None:
task_helper.update_progress(async_new_service, 100)
# delete pending task from db
if curr_task_id is not None:
pending_task = PendingTask.objects.get(task_id=curr_task_id)
pending_task.delete()
except (BaseException, XMLSyntaxError, XPathEvalError, InvalidURL,
ConnectionError) as e:
url = url_dict[
'base_uri'] + f"SERVICE={url_dict['service'].value}&VERSION={url_dict['version'].value}&request={url_dict['request']}"
error_msg = f"Error while trying to register new resource for url: {url}\n"
response = requests.get(url)
if response.status_code == 200:
cap_doc = "-----------------------------------------------------------\n"\
f"We could receive the following capabilities document:\n{response.text}"
error_msg += cap_doc
service_logger.error(msg=error_msg)
service_logger.exception(e, stack_info=True, exc_info=True)
if curr_task_id is not None:
pending_task = PendingTask.objects.get(task_id=curr_task_id)
register_group = MrMapGroup.objects.get(id=register_group_id)
error_report = ErrorReport(message=error_msg,
traceback=traceback.format_exc(),
created_by=register_group)
error_report.save()
descr = json.loads(pending_task.description)
pending_task.description = json.dumps({
"service":
descr.get("service", None),
"info": {
"current": "0",
},
"exception":
e.__str__(),
"phase":
"ERROR: Something went wrong! Click on generate error report to inform your serveradmin about this error.",
})
pending_task.error_report = error_report
pending_task.save()
raise e
def async_new_service(url_dict: dict, user_id: int, register_group_id: int,
register_for_organization_id: int, external_auth: dict):
""" Async call of new service creation
Since redis is used as broker, the objects can not be passed directly into the function. They have to be resolved using
their ids, since the objects are not easily serializable using json
Args:
url_dict (dict): Contains basic information about the service like connection uri
user_id (int): Id of the performing user
register_group_id (int): Id of the group which wants to register
register_for_organization_id (int): Id of the organization for which the service is registered
Returns:
nothing
"""
if current_task:
current_task.update_state(state=states.STARTED,
meta={
'current': 0,
'total': 100,
'phase': 'pre configure task...',
})
# create ExternalAuthentication object
if external_auth is not None:
external_auth = ExternalAuthentication(
username=external_auth["username"],
password=external_auth["password"],
auth_type=external_auth["auth_type"],
)
# restore objects from ids
user = MrMapUser.objects.get(id=user_id)
url_dict["service"] = service_helper.resolve_service_enum(
url_dict["service"])
url_dict["version"] = service_helper.resolve_version_enum(
url_dict["version"])
register_group = MrMapGroup.objects.get(id=register_group_id)
if utils.resolve_none_string(
str(register_for_organization_id)) is not None:
register_for_organization = Organization.objects.get(
id=register_for_organization_id)
else:
register_for_organization = None
t_start = time.time()
service = service_helper.create_service(url_dict.get("service"),
url_dict.get("version"),
url_dict.get("base_uri"),
user,
register_group,
register_for_organization,
external_auth=external_auth)
# after service AND documents have been persisted, we can now set the service being secured if needed
if external_auth is not None:
#todo: check this......
if current_task:
current_task.update_state(state=states.STARTED,
meta={
'current':
PROGRESS_STATUS_AFTER_PARSING,
'phase': 'Securing...',
'service': service.metadata.title
})
service.metadata.set_proxy(True)
service_logger.debug(EXEC_TIME_PRINT %
("total registration", time.time() - t_start))
user_helper.create_group_activity(service.metadata.created_by, user,
SERVICE_REGISTERED,
service.metadata.title)
return {
'msg':
'Done. New service registered.',
'id':
str(service.metadata.pk),
'absolute_url':
service.metadata.get_absolute_url(),
'absolute_url_html':
f'<a href={service.metadata.get_absolute_url()}>{service.metadata.title}</a>'
}
