def storagesetup(request): # Display the dashboard UUID on the storage service setup page dashboard_uuid = helpers.get_setting('dashboard_uuid', None) assert dashboard_uuid is not None # Prefill the storage service URL inital_data = { 'storage_service_url': helpers.get_setting('storage_service_url', 'http://localhost:8000'), 'storage_service_user': helpers.get_setting('storage_service_user', 'test'), 'storage_service_apikey': helpers.get_setting('storage_service_apikey', None) } storage_form = StorageSettingsForm(request.POST or None, initial=inital_data) if not storage_form.is_valid(): return render(request, 'installer/storagesetup.html', locals()) storage_form.save() try: use_default_config = storage_form.cleaned_data[ 'storage_service_use_default_config'] setup_pipeline_in_ss(use_default_config) except Exception: messages.warning( request, _('Error creating pipeline: is the storage server running? Please contact an administrator.' )) return redirect('main.views.home')
def storagesetup(request): # Display the dashboard UUID on the storage service setup page dashboard_uuid = helpers.get_setting("dashboard_uuid", None) assert dashboard_uuid is not None # Prefill the storage service URL inital_data = { "storage_service_url": helpers.get_setting("storage_service_url", "http://localhost:8000"), "storage_service_user": helpers.get_setting("storage_service_user", "test"), "storage_service_apikey": helpers.get_setting("storage_service_apikey", None), } storage_form = StorageSettingsForm(request.POST or None, initial=inital_data) if not storage_form.is_valid(): return render(request, "installer/storagesetup.html", locals()) storage_form.save() try: use_default_config = storage_form.cleaned_data[ "storage_service_use_default_config"] setup_pipeline_in_ss(use_default_config) except Exception: messages.warning( request, _("Error creating pipeline: is the storage server running? Please contact an administrator." ), ) return redirect("main.views.home")
def storagesetup(request): # Display the dashboard UUID on the storage service setup page dashboard_uuid = helpers.get_setting('dashboard_uuid', None) assert dashboard_uuid is not None # Prefill the storage service URL inital_data = { 'storage_service_url': helpers.get_setting('storage_service_url', 'http://localhost:8000') } storage_form = StorageSettingsForm(request.POST or None, initial=inital_data) if storage_form.is_valid(): # Set storage service URL storage_form.save() if "use_default" in request.POST: shared_path = helpers.get_server_config_value('sharedDirectory') # Post first user & API key user = User.objects.all()[0] api_key = ApiKey.objects.get(user=user) # Create pipeline, tell it to use default setup try: storage_service.create_pipeline( create_default_locations=True, shared_path=shared_path, api_username=user.username, api_key=api_key.key, ) except Exception: messages.warning( request, 'Error creating pipeline: is the storage server running? Please contact an administrator.' ) else: # Add the storage service URL to the API whitelist ss_url = urlparse.urlparse( helpers.get_setting('storage_service_url')) whitelist = helpers.get_setting('api_whitelist', '127.0.0.1') whitelist = '\n'.join([whitelist, ss_url.hostname]) helpers.set_setting('api_whitelist', whitelist) else: # Storage service manually set up, just register Pipeline if # possible. Do not provide additional information about the shared # path, or API, as this is probably being set up in the storage # service manually. try: storage_service.create_pipeline() except Exception: pass return redirect('main.views.home') else: return render(request, 'installer/storagesetup.html', locals())
def submit_fpr_agent(): agent = get_agent() url = django_settings.FPR_URL + 'agent/' resp = {} headers = {'Content-Type': 'application/json'} payload = { 'uuid': helpers.get_setting('dashboard_uuid'), 'agentType': 'new install', 'agentName': agent.name, 'clientIP': get_my_ip(), 'agentIdentifierType': agent.identifiertype, 'agentIdentifierValue': agent.identifiervalue } try: logger.info("FPR Server URL: {}".format(django_settings.FPR_URL)) r = requests.post(url, data=json.dumps(payload), headers=headers, timeout=django_settings.FPR_CLIENT_TIMEOUT, verify=True) if r.status_code == 201: resp['result'] = 'success' else: resp['result'] = 'failed to fetch from ' + url except: resp['result'] = 'failed to post to ' + url return resp
def general(request): toggleableSettings = { 'dashboard_administration_atom_dip_enabled': 'Hide AtoM DIP upload link', 'dashboard_administration_dspace_enabled': 'Hide DSpace transfer type', } initial_data = _intial_settings_data() interface_form = SettingsForm(request.POST or None, prefix='interface', reverse_checkboxes=toggleableSettings) storage_form = StorageSettingsForm(request.POST or None, prefix='storage', initial=initial_data) if interface_form.is_valid() and storage_form.is_valid(): interface_form.save() storage_form.save() messages.info(request, 'Saved.') dashboard_uuid = helpers.get_setting('dashboard_uuid') try: pipeline = storage_service._get_pipeline(dashboard_uuid) except Exception: messages.warning( request, "Storage server inaccessible. Please contact an administrator or update storage service URL below." ) else: if not pipeline: messages.warning( request, "This pipeline is not registered with the storage service or has been disabled in the storage service. Please contact an administrator." ) hide_features = helpers.hidden_features() return render(request, 'administration/general.html', locals())
def fprupload(request): response_data = {} agent = Agent.objects.get(pk=2) url = django_settings.FPR_URL + 'Agent/' #url = 'https://fpr.archivematica.org/fpr/api/v1/Agent/' payload = { 'uuid': helpers.get_setting('dashboard_uuid'), 'agentType': 'new install', 'agentName': agent.name, 'clientIP': get_my_ip(), 'agentIdentifierType': agent.identifiertype, 'agentIdentifierValue': agent.identifiervalue } headers = {'Content-Type': 'application/json'} try: r = requests.post(url, data=json.dumps(payload), headers=headers, timeout=10, verify=True) if r.status_code == 201: response_data['result'] = 'success' else: response_data['result'] = 'failed to fetch from ' + url except: response_data['result'] = 'failed to post to ' + url return HttpResponse(json.dumps(response_data), content_type="application/json")
def test_save_datatable_state(self): """Test ability to save DataTable state""" response = self.client.post( "/backlog/save_state/transfers/", self.data, content_type="application/json" ) assert response.status_code == 200 saved_state = helpers.get_setting("transfers_datatable_state") assert json.dumps(self.data) == saved_state
def administration_api(request): if request.method == 'POST': whitelist = request.POST.get('whitelist', '') helpers.set_setting('api_whitelist', whitelist) else: whitelist = helpers.get_setting('api_whitelist', '127.0.0.1') return render(request, 'administration/api.html', locals())
def general(request): initial_data = _intial_settings_data() initial_data['storage_service_use_default_config'] = { 'False': False}.get( initial_data.get('storage_service_use_default_config', True), True) general_form = GeneralSettingsForm(request.POST or None, prefix='general', initial=initial_data) storage_form = StorageSettingsForm(request.POST or None, prefix='storage', initial=initial_data) checksum_form = ChecksumSettingsForm(request.POST or None, prefix='checksum algorithm', initial=initial_data) forms = (general_form, storage_form, checksum_form) if all(map(lambda form: form.is_valid(), forms)): map(lambda form: form.save(), forms) messages.info(request, _('Saved.')) dashboard_uuid = helpers.get_setting('dashboard_uuid') not_created_yet = False try: pipeline = storage_service.get_pipeline(dashboard_uuid) except Exception as err: if err.response is not None and err.response.status_code == 404: # The server has returned a 404, we're going to assume that this is # the Storage Service telling us that the pipeline is unknown. not_created_yet = True else: messages.warning(request, _('Storage Service inaccessible. Please' ' contact an administrator or update' ' the Storage Sevice URL below.' '<hr />%(error)s' % {'error': err})) if not_created_yet: if storage_form.is_valid(): try: setup_pipeline_in_ss( storage_form.cleaned_data[ 'storage_service_use_default_config']) except Exception as err: messages.warning(request, _('Storage Service failed to create the' ' pipeline. This can happen if' ' the pipeline exists but it is' ' disabled. Please contact an' ' administrator.' '<hr />%(error)s' % {'error': err})) else: messages.warning(request, _('Storage Service returned a 404 error.' ' Has the pipeline been disabled or is' ' it not registered yet? Submitting' ' form will attempt to register the' ' pipeline.')) return render(request, 'administration/general.html', locals())
def api(request): if request.method == 'POST': whitelist = request.POST.get('whitelist', '') helpers.set_setting('api_whitelist', whitelist) messages.info(request, _('Saved.')) else: whitelist = helpers.get_setting('api_whitelist', '') return render(request, 'administration/api.html', locals())
def api(request): if request.method == "POST": whitelist = request.POST.get("whitelist", "") helpers.set_setting("api_whitelist", whitelist) else: whitelist = helpers.get_setting("api_whitelist", "127.0.0.1") hide_features = hidden_features() return render(request, "administration/api.html", locals())
def api(request): if request.method == "POST": whitelist = request.POST.get("whitelist", "") helpers.set_setting("api_whitelist", whitelist) messages.info(request, _("Saved.")) else: whitelist = helpers.get_setting("api_whitelist", "") return render(request, "administration/api.html", locals())
def ingest_upload_destination_url_check(request): url = helpers.get_setting('dip_upload_atom_url') # add target to URL url = urljoin(url, request.GET.get('target', '')) # make request for URL response = requests.request('GET', url) # return resulting status code from request return HttpResponse(response.status_code)
def process_request(self, request): # The presence of the UUID is an indicator of whether we've already set up. dashboard_uuid = helpers.get_setting('dashboard_uuid') if not dashboard_uuid: # Start off the installer if reverse('installer.views.welcome') != request.path_info: return redirect('installer.views.welcome') elif not request.user.is_authenticated(): # Installation already happened - make sure the user is logged in. path = request.path_info.lstrip('/') if not any(m.match(path) for m in EXEMPT_URLS): return redirect(settings.LOGIN_URL)
def allowed_by_whitelist(ip_address): whitelist = [ip.strip() for ip in helpers.get_setting("api_whitelist", "").split()] # If there's no whitelist, allow all through if not whitelist: return True LOGGER.debug("looking for ip %s in whitelist %s", ip_address, whitelist) # There is a whitelist - check the IP address against it if ip_address in whitelist: LOGGER.debug("API called by trusted IP %s", ip_address) return True return False
def authenticate_request(request): api_auth = ApiKeyAuthentication() authorized = api_auth.is_authenticated(request) if authorized == True: client_ip = request.META['REMOTE_ADDR'] whitelist = helpers.get_setting('api_whitelist', '127.0.0.1').split("\r\n") try: whitelist.index(client_ip) return True except: pass return False
def load_state(request, table): """Retrieve DataTable state JSON object stored in DashboardSettings. :param request: Django request. :param table: Name of table to store state for. :return: JSON state """ setting_name = "{}_datatable_state".format(table) state = helpers.get_setting(setting_name) if state: return HttpResponse( json.loads(state), content_type="application/json", status=200 ) return helpers.json_response( {"error": True, "message": "Setting not found"}, status_code=404 )
def process_request(self, request): dashboard_uuid = helpers.get_setting("dashboard_uuid") # Start off the installer unless the user is already there. if not dashboard_uuid: if reverse(welcome) == request.path_info: return return redirect(welcome) # Send the user to the login page if needed. if not request.user.is_authenticated(): path = request.path_info.lstrip("/") if not any(m.match(path) for m in EXEMPT_URLS): return redirect(settings.LOGIN_URL) # Share the ID of the pipeline with the application views. request.dashboard_uuid = dashboard_uuid
def authenticate_request(request): error = None api_auth = ApiKeyAuthentication() authorized = api_auth.is_authenticated(request) # 'authorized' can be True, False or tastypie.http.HttpUnauthorized # Check explicitly for True, not just truthiness if authorized is True: client_ip = request.META['REMOTE_ADDR'] whitelist = helpers.get_setting('api_whitelist', '127.0.0.1').split() if client_ip not in whitelist: error = 'Host/IP ' + client_ip + ' not authorized.' else: error = 'API key not valid.' return error
def authenticate_request(request): error = None api_auth = ApiKeyAuthentication() authorized = api_auth.is_authenticated(request) if authorized == True: client_ip = request.META['REMOTE_ADDR'] whitelist = helpers.get_setting('api_whitelist', '127.0.0.1').split("\r\n") try: whitelist.index(client_ip) return except: error = 'Host/IP ' + client_ip + ' not authorized.' else: error = 'API key not valid.' return error
def fprupload(request): response_data = {} agent = Agent.objects.get(pk=2) url = 'https://fpr.artefactual.com/fpr/api/v1/Agent/' payload = {'uuid': helpers.get_setting('dashboard_uuid'), 'agentType': 'new install', 'agentName': agent.name, 'clientIP': get_my_ip(), 'agentIdentifierType': agent.identifiertype, 'agentIdentifierValue': agent.identifiervalue } headers = {'Content-Type': 'application/json'} r = requests.post(url, data=json.dumps(payload), headers=headers) if r.status_code == 201: response_data['result'] = 'success' else: response_data['result'] = 'failed' return HttpResponse(json.dumps(response_data), content_type="application/json")
def fprupload(request): response_data = {} agent = Agent.objects.get(pk=2) url = 'https://fpr.artefactual.com/fpr/api/v1/Agent/' payload = { 'uuid': helpers.get_setting('dashboard_uuid'), 'agentType': 'new install', 'agentName': agent.name, 'clientIP': get_my_ip(), 'agentIdentifierType': agent.identifiertype, 'agentIdentifierValue': agent.identifiervalue } headers = {'Content-Type': 'application/json'} r = requests.post(url, data=json.dumps(payload), headers=headers) if r.status_code == 201: response_data['result'] = 'success' else: response_data['result'] = 'failed' return HttpResponse(json.dumps(response_data), content_type="application/json")
def setup_pipeline_in_ss(use_default_config=False): # Check if pipeline is already registered on SS dashboard_uuid = helpers.get_setting("dashboard_uuid") try: storage_service.get_pipeline(dashboard_uuid) except Exception: logger.warning("SS inaccessible or pipeline not registered.") else: # If pipeline is already registered on SS, then exit logger.warning("This pipeline is already configured on SS.") return if not use_default_config: # Storage service manually set up, just register Pipeline if # possible. Do not provide additional information about the shared # path, or API, as this is probably being set up in the storage # service manually. storage_service.create_pipeline() return # Post first user & API key user = User.objects.all()[0] api_key = ApiKey.objects.get(user=user) # Retrieve remote name try: setting = DashboardSetting.objects.get(name="site_url") except DashboardSetting.DoesNotExist: remote_name = None else: remote_name = setting.value # Create pipeline, tell it to use default setup storage_service.create_pipeline( create_default_locations=True, shared_path=django_settings.SHARED_DIRECTORY, remote_name=remote_name, api_username=user.username, api_key=api_key.key, )
def welcome(request): # This form will be only accessible when there is no uuid dashboard_uuid = helpers.get_setting('dashboard_uuid') if dashboard_uuid: return redirect('main.views.home') # Do we need to set up a user? set_up_user = not User.objects.exists() if request.method == 'POST': # save organization PREMIS agent if supplied setup_pipeline( org_name=request.POST.get('org_name', ''), org_identifier=request.POST.get('org_identifier', ''), site_url=request.POST.get('site_url'), ) if set_up_user: form = SuperUserCreationForm(request.POST) if form.is_valid(): user = form.save() api_key = ApiKey.objects.create(user=user) api_key.key = api_key.generate_key() api_key.save() user = authenticate(username=user.username, password=form.cleaned_data['password1']) if user is not None: login(request, user) request.session['first_login'] = True return redirect('installer.views.storagesetup') else: request.session['first_login'] = True return redirect('installer.views.storagesetup') else: form = SuperUserCreationForm() if set_up_user else OrganizationForm() return render(request, 'installer/welcome.html', { 'form': form, })
def setup_pipeline(org_name, org_identifier, site_url): dashboard_uuid = helpers.get_setting("dashboard_uuid") # Setup pipeline only if dashboard_uuid doesn't already exists if dashboard_uuid: return # Assign UUID to Dashboard dashboard_uuid = str(uuid.uuid4()) helpers.set_setting("dashboard_uuid", dashboard_uuid) # Update Archivematica version in DB set_agent_code("Archivematica-" + get_version(), pk=1) if org_name != "" or org_identifier != "": agent = get_agent() agent.name = org_name agent.identifiertype = "repository code" agent.identifiervalue = org_identifier agent.save() if site_url: helpers.set_setting("site_url", site_url)
def welcome(request): # This form will be only accessible when there is no uuid dashboard_uuid = helpers.get_setting("dashboard_uuid") if dashboard_uuid: return redirect("main:main_index") # Do we need to set up a user? set_up_user = not User.objects.exists() if request.method == "POST": # save organization PREMIS agent if supplied setup_pipeline( org_name=request.POST.get("org_name", ""), org_identifier=request.POST.get("org_identifier", ""), site_url=request.POST.get("site_url"), ) if set_up_user: form = SuperUserCreationForm(request.POST) if form.is_valid(): user = form.save() api_key = ApiKey.objects.create(user=user) api_key.key = api_key.generate_key() api_key.save() user = authenticate( username=user.username, password=form.cleaned_data["password1"] ) if user is not None: login(request, user) request.session["first_login"] = True return redirect("installer:storagesetup") else: request.session["first_login"] = True return redirect("installer:storagesetup") else: form = SuperUserCreationForm() if set_up_user else OrganizationForm() return render(request, "installer/welcome.html", {"form": form})
def fprupload(request): response_data = {} agent = Agent.objects.get(pk=2) url = django_settings.FPR_URL + 'Agent/' #url = 'https://fpr.archivematica.org/fpr/api/v1/Agent/' payload = {'uuid': helpers.get_setting('dashboard_uuid'), 'agentType': 'new install', 'agentName': agent.name, 'clientIP': get_my_ip(), 'agentIdentifierType': agent.identifiertype, 'agentIdentifierValue': agent.identifiervalue } headers = {'Content-Type': 'application/json'} try: r = requests.post(url, data=json.dumps(payload), headers=headers, timeout=10, verify=True) if r.status_code == 201: response_data['result'] = 'success' else: response_data['result'] = 'failed to fetch from ' + url except: response_data['result'] = 'failed to post to ' + url return HttpResponse(json.dumps(response_data), content_type="application/json")
def report(uuid): """ Generate normalization report using Django's template module and send it to every active user. """ recipient_list = User.objects.filter( is_active=True, userprofile__system_emails=True).values_list("email", flat=True) if not recipient_list: logger.info( "Normalization report is not being sent because the recipient list is empty." ) return 0 logger.info("Sending report to %s.", ", ".join(recipient_list)) try: sip = SIP.objects.get(uuid=uuid) except SIP.DoesNotExist: logger.error("SIP with UUID %s not found.", uuid) return 1 failed_tasks = {} for jobtype in ("Normalize for preservation", "Normalize for access"): try: job = Job.objects.filter( sipuuid=uuid, jobtype=jobtype).order_by("-createdtime")[0] except IndexError: logger.info( 'No normalization failures have been detected in type "%s"', jobtype) continue tasks = Task.objects.filter(job=job).exclude(exitcode__in=[0, 2]) if not tasks.exists(): logger.info( 'No normalization failures have been detected in type "%s"', jobtype) continue failed_tasks[jobtype] = tasks.values("filename", "fileuuid", "exitcode") for item in failed_tasks[jobtype]: try: item["location"] = File.objects.get( uuid=item["fileuuid"]).currentlocation.replace( "%SIPDirectory%", "") except File.DoesNotExist: pass if not len(failed_tasks): logger.info( "Normalization report is not being sent because no failures have been detected." ) return 0 ctxdict = { "uuid": uuid, "name": os.path.basename(sip.currentpath.rstrip("/")).replace( "-" + sip.uuid, ""), "pipeline_uuid": helpers.get_setting("dashboard_uuid"), "failed_tasks": failed_tasks, } logger.info("Building HTML message") ctx = Context(ctxdict) tmpl = Template(EMAIL_TEMPLATE) html_message = tmpl.render(ctx) logger.info("Storing report in database") Report.objects.create( content=html_message, unittype="SIP", unitname=ctxdict["name"], unitidentifier=ctxdict["uuid"], ) try: logger.info("Sending email...") send_mail( subject="Normalization failure report for {} ({})".format( ctxdict["name"], ctxdict["uuid"]), message="Please see the attached HTML document", from_email=mcpclient_settings.DEFAULT_FROM_EMAIL, recipient_list=recipient_list, html_message=html_message, ) except: logger.exception("Report email was not delivered") return 1 else: logger.info("Report sent successfully!") return 0
def report(uuid): """ Generate normalization report using Django's template module and send it to every active user. """ recipient_list = User.objects.filter(is_active=True).values_list( 'email', flat=True).exclude(email__in=['*****@*****.**', '']) if not recipient_list: logger.info( 'Normalization report is not being sent because the recipient list is empty.' ) return 0 logger.info('Sending report to %s.', ', '.join(recipient_list)) try: sip = SIP.objects.get(uuid=uuid) except SIP.DoesNotExist: logger.error('SIP with UUID %s not found.', uuid) return 1 failed_tasks = {} for jobtype in ('Normalize for preservation', 'Normalize for access'): try: job = Job.objects.filter( sipuuid=uuid, jobtype=jobtype).order_by('-createdtime')[0] except IndexError: logger.info( 'No normalization failures have been detected in type "%s"', jobtype) continue tasks = Task.objects.filter(job=job).exclude(exitcode__in=[0, 2]) if not tasks.exists(): logger.info( 'No normalization failures have been detected in type "%s"', jobtype) continue failed_tasks[jobtype] = tasks.values('filename', 'fileuuid', 'exitcode') for item in failed_tasks[jobtype]: try: item['location'] = File.objects.get( uuid=item['fileuuid']).currentlocation.replace( '%SIPDirectory%', '') except File.DoesNotExist: pass if not len(failed_tasks): logger.info( 'Normalization report is not being sent because no failures have been detected.' ) return 0 ctxdict = { 'uuid': uuid, 'name': os.path.basename(sip.currentpath.rstrip('/')).replace( '-' + sip.uuid, ''), 'pipeline_uuid': helpers.get_setting('dashboard_uuid'), 'failed_tasks': failed_tasks } logger.info('Building HTML message') ctx = Context(ctxdict) tmpl = Template(EMAIL_TEMPLATE) html_message = tmpl.render(ctx) logger.info('Storing report in database') Report.objects.create(content=html_message, unittype='SIP', unitname=ctxdict['name'], unitidentifier=ctxdict['uuid']) try: logger.info('Sending email...') send_mail( subject='Normalization failure report for {} ({})'.format( ctxdict['name'], ctxdict['uuid']), message='Please see the attached HTML document', from_email=mcpclient_settings.DEFAULT_FROM_EMAIL, recipient_list=recipient_list, html_message=html_message, ) except: logger.exception('Report email was not delivered') return 1 else: logger.info('Report sent successfully!') return 0
def general(request): initial_data = _intial_settings_data() initial_data["storage_service_use_default_config"] = {"False": False}.get( initial_data.get("storage_service_use_default_config", True), True ) general_form = GeneralSettingsForm( request.POST or None, prefix="general", initial=initial_data ) storage_form = StorageSettingsForm( request.POST or None, prefix="storage", initial=initial_data ) checksum_form = ChecksumSettingsForm( request.POST or None, prefix="checksum algorithm", initial=initial_data ) forms = (general_form, storage_form, checksum_form) if all(map(lambda form: form.is_valid(), forms)): for item in forms: item.save() messages.info(request, _("Saved.")) dashboard_uuid = helpers.get_setting("dashboard_uuid") not_created_yet = False try: pipeline = storage_service.get_pipeline(dashboard_uuid) except Exception as err: if err.response is not None and err.response.status_code == 404: # The server has returned a 404, we're going to assume that this is # the Storage Service telling us that the pipeline is unknown. not_created_yet = True else: messages.warning( request, _( "Storage Service inaccessible. Please" " contact an administrator or update" " the Storage Sevice URL below." "<hr />%(error)s" % {"error": err} ), ) if not_created_yet: if storage_form.is_valid(): try: setup_pipeline_in_ss( storage_form.cleaned_data["storage_service_use_default_config"] ) except Exception as err: messages.warning( request, _( "Storage Service failed to create the" " pipeline. This can happen if" " the pipeline exists but it is" " disabled. Please contact an" " administrator." "<hr />%(error)s" % {"error": err} ), ) else: messages.warning( request, _( "Storage Service returned a 404 error." " Has the pipeline been disabled or is" " it not registered yet? Submitting" " form will attempt to register the" " pipeline." ), ) return render(request, "administration/general.html", locals())