def complete_validation(self, run_id: int, user_id: int = None, group_id: int = None): """ Handles the completed validation process. Handler for completing the validation process. This method takes care of deleting the PeriodicTask and adding a new GroupActivity. Note: This method should be used to handle the completed validation process (passed, failed). Use complete_validation_error for handling any kind of unexpected exceptions during the validation. Args: run_id (int): The id of the ConformityCheckRun. Keyword arguments: user_id (int): The id of the user that triggered the run. group_id (int): the id of the group that the validated metadata object belongs to. Returns: nothing """ parent_task_id = complete_validation.request.parent_id if parent_task_id is not None: try: pt = PendingTask.objects.get(task_id=parent_task_id) pt.progress = 100 pt.save() sleep(2) pt.delete() except PendingTask.DoesNotExist: pass try: run = ConformityCheckRun.objects.get(pk=run_id) # task is still running if run.passed is None: return group = MrMapGroup.objects.get(pk=group_id) user = MrMapUser.objects.get(pk=user_id) title = _(f'Validation {"succeeded" if run.passed else "failed"}') href = reverse('resource:detail', args=(run.metadata.pk, )) content = _(f'for <a href="{href}">' f'{run.metadata.title}</a> ' f'with <i>{run.conformity_check_configuration}</i>.') create_group_activity(group=group, user=user, msg=title, metadata_title=content) except (ConformityCheckRun.DoesNotExist, MrMapGroup.DoesNotExist, MrMapUser.DoesNotExist) as e: logger.error("Could not complete validation. ", e)
def complete_validation_error(request, exc, traceback, user_id: int = None, group_id: int = None, config_id: int = None, metadata_id: str = None): """ Handles the aborted validation process. Handler for completing the aborted validation process. This method takes care of deleting the PeriodicTask and adding a new GroupActivity. Note: This method should be used to handle the aborted validation process (e.g. unhandled Exceptions). Use complete_validation for handling the completed validation process. Args: *args: positional arguments Keyword arguments: user_id (int): The id of the user that triggered the run. group_id (int): the id of the group that the validated metadata object belongs to. config_id (int): The id of the ConformityCheckConfiguration. metadata_id (uuid): The id of the validated metadata object. Returns: nothing """ try: config = ConformityCheckConfiguration.objects.get(pk=config_id) metadata = Metadata.objects.get(pk=metadata_id) group = MrMapGroup.objects.get(pk=group_id) user = MrMapUser.objects.get(pk=user_id) # delete run, if it was manually aborted if isinstance(exc, AbortedException): try: run = ConformityCheckRun.objects.get_latest_check(metadata) run.delete() except ConformityCheckRun.DoesNotExist: pass title = _(f'Validation aborted') content = _(f'for <a href="resource/detail/{metadata.id}">' f'{metadata.title}</a> ' f'with <i>{config}</i>.') create_group_activity(group=group, user=user, msg=title, metadata_title=content) except (ConformityCheckRun.DoesNotExist, Metadata.DoesNotExist, MrMapUser.DoesNotExist) as e: logger.error("Could not complete error task. ", e)
def process_restore_dataset_metadata(self): ext_auth = self.instance.get_external_authentication_object() if not self.instance.is_custom: messages.add_message(self.request, messages.INFO, METADATA_IS_ORIGINAL) return HttpResponseRedirect(reverse(self.request.GET.get('current-view', 'home')), status=303) if self.instance.is_custom: self.instance.restore(self.instance.identifier, external_auth=ext_auth) self.instance.save() messages.add_message(self.request, messages.SUCCESS, METADATA_RESTORING_SUCCESS) user_helper.create_group_activity(self.instance.created_by, self.requesting_user, SERVICE_MD_RESTORED, "{}".format(self.instance.title, ))
def form_valid(self, form): self.object = self.get_object() ext_auth = self.object.get_external_authentication_object() self.object.restore(self.object.identifier, external_auth=ext_auth) # Todo: add last_changed_by_user field to Metadata and move this piece of code to Metadata.restore() if self.object.is_metadata_type(MetadataEnum.DATASET): user_helper.create_group_activity(self.object.created_by, self.request.user, SERVICE_MD_RESTORED, "{}".format(self.object.title, )) else: user_helper.create_group_activity(self.object.created_by, self.request.user, SERVICE_MD_RESTORED, "{}: {}".format(self.object.get_root_metadata().title, self.object.title)) success_url = self.get_success_url() return HttpResponseRedirect(success_url)
def run_update_service(request: HttpRequest, metadata_id): if request.method == 'POST': current_service = get_object_or_404(Service.objects.select_related( 'metadata').prefetch_related('metadata__documents'), metadata__id=metadata_id) new_service = get_object_or_404( Service.objects.select_related('metadata').prefetch_related( 'metadata__documents'), is_update_candidate_for=current_service) if not current_service.is_service_type(OGCServiceEnum.WFS): new_service.root_layer = get_object_or_404( Layer, parent_service=new_service, parent=None) current_service.root_layer = get_object_or_404( Layer, parent_service=current_service, parent=None) comparator = ServiceComparator(service_a=new_service, service_b=current_service) diff = comparator.compare_services() diff_elements = diff.get("layers", None) or diff.get( "feature_types", {}) # We need to extract the linkage of new->old elements from the request by hand # key identifies the new element and it's identifier (not id!) and choice identifies the existing element's id! links = {} prefix = "new_elem_" for key, choice in request.POST.items(): if prefix in key: links[key.replace(prefix, "")] = choice update_confirmation_form = UpdateOldToNewElementsForm( request.POST, new_elements=diff_elements.get("new"), removed_elements=diff_elements.get("removed"), choices=links, current_service=current_service, ) update_confirmation_form.action_url = reverse("resource:run-update", args=[metadata_id]) if update_confirmation_form.is_valid(): # UPDATE # First update the metadata of the whole service md = update_helper.update_metadata(current_service.metadata, new_service.metadata, new_service.keep_custom_md) md.save() current_service.metadata = md current_service.save() # Then update the service object current_service = update_helper.update_service( current_service, new_service) current_service.save() # Update the subelements if new_service.is_service_type(OGCServiceEnum.WFS): current_service = update_helper.update_wfs_elements( current_service, new_service, diff, links, new_service.keep_custom_md) elif new_service.is_service_type(OGCServiceEnum.WMS): # takes long time | todo proof again after using django-mptt current_service = update_helper.update_wms_elements( current_service, new_service, diff, links, new_service.keep_custom_md) current_service.save() update_helper.update_capability_document(current_service, new_service) current_service.save() user_helper.create_group_activity( current_service.metadata.created_by, request.user, SERVICE_UPDATED, current_service.metadata.title) new_service.delete() messages.success(request, SERVICE_UPDATED) return HttpResponseRedirect(reverse("resource:detail", args=(metadata_id, )), status=303) else: params = { "update_confirmation_form": update_confirmation_form, } return pending_update_service(request=request, metadata_id=metadata_id, update_params=params, status_code=422) else: return HttpResponseRedirect(reverse("resource:pending-update", args=(metadata_id, )), status=303)
def async_new_service(url_dict: dict, user_id: int, register_group_id: int, register_for_organization_id: int, external_auth: dict): """ Async call of new service creation Since redis is used as broker, the objects can not be passed directly into the function. They have to be resolved using their ids, since the objects are not easily serializable using json Args: url_dict (dict): Contains basic information about the service like connection uri user_id (int): Id of the performing user register_group_id (int): Id of the group which wants to register register_for_organization_id (int): Id of the organization for which the service is registered Returns: nothing """ # create ExternalAuthentication object if external_auth is not None: external_auth = ExternalAuthentication( username=external_auth["username"], password=external_auth["password"], auth_type=external_auth["auth_type"], ) # get current task id curr_task_id = async_new_service.request.id # set progress for current task to 0 if curr_task_id is not None: task_helper.update_progress(async_new_service, 0) # restore objects from ids user = MrMapUser.objects.get(id=user_id) url_dict["service"] = service_helper.resolve_service_enum( url_dict["service"]) url_dict["version"] = service_helper.resolve_version_enum( url_dict["version"]) register_group = MrMapGroup.objects.get(id=register_group_id) if utils.resolve_none_string( str(register_for_organization_id)) is not None: register_for_organization = Organization.objects.get( id=register_for_organization_id) else: register_for_organization = None try: t_start = time.time() service = service_helper.create_service(url_dict.get("service"), url_dict.get("version"), url_dict.get("base_uri"), user, register_group, register_for_organization, async_task=async_new_service, external_auth=external_auth) # update progress if curr_task_id is not None: task_helper.update_progress(async_new_service, PROGRESS_STATUS_AFTER_PARSING) # get db object if curr_task_id is not None: pending_task = PendingTask.objects.get(task_id=curr_task_id) # update db pending task information pending_task.description = json.dumps({ "service": service.metadata.title, "phase": "Persisting", }) pending_task.save() # update progress if curr_task_id is not None: task_helper.update_progress(async_new_service, 95) # after service AND documents have been persisted, we can now set the service being secured if needed if external_auth is not None: service.metadata.set_proxy(True) metadatas = Metadata.objects.filter(pk=service.metadata.pk) sub_elements = service.get_subelements().select_related('metadata') for sub_element in sub_elements: metadatas |= Metadata.objects.filter(pk=sub_element.metadata.pk) metadatas |= sub_element.metadata.get_related_dataset_metadatas() service_logger.debug(EXEC_TIME_PRINT % ("total registration", time.time() - t_start)) user_helper.create_group_activity(service.metadata.created_by, user, SERVICE_REGISTERED, service.metadata.title) if curr_task_id is not None: task_helper.update_progress(async_new_service, 100) # delete pending task from db if curr_task_id is not None: pending_task = PendingTask.objects.get(task_id=curr_task_id) pending_task.delete() except (BaseException, XMLSyntaxError, XPathEvalError, InvalidURL, ConnectionError) as e: url = url_dict[ 'base_uri'] + f"SERVICE={url_dict['service'].value}&VERSION={url_dict['version'].value}&request={url_dict['request']}" error_msg = f"Error while trying to register new resource for url: {url}\n" response = requests.get(url) if response.status_code == 200: cap_doc = "-----------------------------------------------------------\n"\ f"We could receive the following capabilities document:\n{response.text}" error_msg += cap_doc service_logger.error(msg=error_msg) service_logger.exception(e, stack_info=True, exc_info=True) if curr_task_id is not None: pending_task = PendingTask.objects.get(task_id=curr_task_id) register_group = MrMapGroup.objects.get(id=register_group_id) error_report = ErrorReport(message=error_msg, traceback=traceback.format_exc(), created_by=register_group) error_report.save() descr = json.loads(pending_task.description) pending_task.description = json.dumps({ "service": descr.get("service", None), "info": { "current": "0", }, "exception": e.__str__(), "phase": "ERROR: Something went wrong! Click on generate error report to inform your serveradmin about this error.", }) pending_task.error_report = error_report pending_task.save() raise e
def async_new_service(url_dict: dict, user_id: int, register_group_id: int, register_for_organization_id: int, external_auth: dict): """ Async call of new service creation Since redis is used as broker, the objects can not be passed directly into the function. They have to be resolved using their ids, since the objects are not easily serializable using json Args: url_dict (dict): Contains basic information about the service like connection uri user_id (int): Id of the performing user register_group_id (int): Id of the group which wants to register register_for_organization_id (int): Id of the organization for which the service is registered Returns: nothing """ if current_task: current_task.update_state(state=states.STARTED, meta={ 'current': 0, 'total': 100, 'phase': 'pre configure task...', }) # create ExternalAuthentication object if external_auth is not None: external_auth = ExternalAuthentication( username=external_auth["username"], password=external_auth["password"], auth_type=external_auth["auth_type"], ) # restore objects from ids user = MrMapUser.objects.get(id=user_id) url_dict["service"] = service_helper.resolve_service_enum( url_dict["service"]) url_dict["version"] = service_helper.resolve_version_enum( url_dict["version"]) register_group = MrMapGroup.objects.get(id=register_group_id) if utils.resolve_none_string( str(register_for_organization_id)) is not None: register_for_organization = Organization.objects.get( id=register_for_organization_id) else: register_for_organization = None t_start = time.time() service = service_helper.create_service(url_dict.get("service"), url_dict.get("version"), url_dict.get("base_uri"), user, register_group, register_for_organization, external_auth=external_auth) # after service AND documents have been persisted, we can now set the service being secured if needed if external_auth is not None: #todo: check this...... if current_task: current_task.update_state(state=states.STARTED, meta={ 'current': PROGRESS_STATUS_AFTER_PARSING, 'phase': 'Securing...', 'service': service.metadata.title }) service.metadata.set_proxy(True) service_logger.debug(EXEC_TIME_PRINT % ("total registration", time.time() - t_start)) user_helper.create_group_activity(service.metadata.created_by, user, SERVICE_REGISTERED, service.metadata.title) return { 'msg': 'Done. New service registered.', 'id': str(service.metadata.pk), 'absolute_url': service.metadata.get_absolute_url(), 'absolute_url_html': f'<a href={service.metadata.get_absolute_url()}>{service.metadata.title}</a>' }