def decorated(*args, **kwargs): try: ulapd_api = UlapdAPI() api_key = request.headers.get('Authorization') name = kwargs['dataset'] dataset = ulapd_api.get_dataset_by_name(name) # Prevent 'open' datasets being accessed via API if dataset['type'] == 'open': raise ApplicationError(*errors.get('ulapd_ui', 'DATASET_NOT_FOUND', filler=name), http_code=404) # Prevent 'confidential' datasets being available via API if user doesn't have access if dataset['type'] == 'confidential': try: user_access = ulapd_api.get_user_details( 'api_key', api_key)['datasets'] except ApplicationError: raise ApplicationError(*errors.get('ulapd_ui', 'API_KEY_ERROR', filler=api_key), http_code=404) if not user_access.get(name, False): raise ApplicationError(*errors.get('ulapd_ui', 'DATASET_NOT_FOUND', filler=name), http_code=404) return f(*args, **kwargs) except ApplicationError as e: return jsonify({'success': False, 'error': e.message}), e.http_code
def view_licence(dataset_id, licence_type=None): try: if licence_type: current_app.logger.info( 'Displaying view {} licence page for dataset: {}'.format( licence_type, dataset_id)) return render_template("app/datasets/licence.html", licence_type=licence_type, dataset_id=dataset_id) else: licence_file = open( os.path.join( directory, '../documents/datasets/{}/licence.md').format(dataset_id), "r") md_licence = licence_file.read() md_renderer = markdown.Markdown() md_html = md_renderer.convert(md_licence) current_app.logger.info( 'Displaying view licence page for dataset: {}'.format( dataset_id)) return render_template("app/datasets/licence.html", dataset_id=dataset_id, md=md_html) except Exception as e: raise ApplicationError( 'Something went wrong when retrieving licence view page: {}'. format(e))
def get_api_dataset_history(name): ulapd_api = UlapdAPI() # authenticate _authenticate(ulapd_api) history_details = ulapd_api.get_dataset_history(name) if history_details: history_data = [] for history in history_details['dataset_history']: for file in history['resource_list']: history_info = { "last_updated": history["last_updated"], "unsorted_date": history["unsorted_date"], "filename": file['file_name'], "file_size": file['file_size'] } history_data.append(history_info) response = { "success": True, "dataset": name, "dataset_history": history_data } return response else: current_app.logger.error('Dataset {} not found'.format(name)) raise ApplicationError(*errors.get('ulapd_ui', 'DATASET_NOT_FOUND', filler=name), http_code=404)
def get_api_datasets(external=False): ulapd_api = UlapdAPI() dataset_list = ulapd_api.get_datasets() user_details = _authenticate(ulapd_api) user_access = user_details['datasets'] if dataset_list: # Filter out 'open' datasets dataset_list = [d for d in dataset_list if d['type'] != 'open'] result_data = [] for dataset in dataset_list: # Don't show 'confidential' datasets unless user has access if dataset['type'] == 'confidential': if user_access.get(dataset['name']): result_data.append( dict(name=dataset['name'], title=dataset['title'])) else: result_data.append( dict(name=dataset['name'], title=dataset['title'])) response = {"success": True, "result": result_data} return response else: raise ApplicationError(*errors.get('ulapd_ui', 'NO_DATASETS_FOUND'), http_code=404)
def decorated(*args, **kwargs): try: if request.headers.get('Authorization') is None: raise ApplicationError( 'Access denied: You need to provide your API Key to perform this operation', http_code=403) return func(*args, **kwargs) except ApplicationError as error: response = {"success": False, "error": error.message} return jsonify(response), error.http_code
def _request(self, uri, data=None): url = '{}/{}'.format(self.base_url, uri) headers = {'Accept': 'application/json'} timeout = current_app.config['DEFAULT_TIMEOUT'] try: if data is None: response = g.requests.get(url, headers=headers, timeout=timeout) else: headers['Content-Type'] = 'application/json' response = g.requests.post(url, headers=headers, timeout=timeout, data=data) status = response.status_code if status == 204: return {} if status == 404: raise ApplicationError(*errors.get('ulapd_ui', 'RESOURCE_NOT_FOUND', filler=uri), http_code=status) response.raise_for_status() return response.json() except requests.exceptions.HTTPError as e: raise ApplicationError(*errors.get('ulapd_ui', 'API_HTTP_ERROR', filler=e), http_code=status) except requests.exceptions.ConnectionError as e: raise ApplicationError(*errors.get('ulapd_ui', 'API_CONN_ERROR', filler=e), http_code=status) except requests.exceptions.Timeout as e: raise ApplicationError(*errors.get('ulapd_ui', 'API_TIMEOUT', filler=e), http_code=status)
def _authenticate(ulapd_api): api_key = request.headers.get('Authorization') try: user_details = ulapd_api.get_user_details('api_key', api_key) except ApplicationError: raise ApplicationError(*errors.get('ulapd_ui', 'API_KEY_ERROR', filler=api_key), http_code=404) return user_details
def reset_api_key(): try: session = dps_session.get_state() ulapd_api = UlapdAPI() user_id = session['user']['user_details']['user_details_id'] ulapd_api.update_api_key(user_id) return redirect('/#my-api-key') except ApplicationError as e: raise ApplicationError( 'Something went wrong when resetting API key - error: {}'.format( e))
def get_agree_licence(dataset_id): try: ulapd_api = UlapdAPI() dataset_details = ulapd_api.get_dataset_by_name(dataset_id) accepted = check_agreement(dataset_id) if dataset_details['type'] == 'freemium': accepted = True session = dps_session.get_state() dataset = session['user']['datasets'].get(dataset_id) if not dataset: accepted = False if dataset: if 'Direct Use' not in dataset['licences']: accepted = False if len(dataset['licences']) == 1: if 'Direct' in dataset['licences'][ 0] and not dataset['valid_licence']: accepted = False if not accepted: return render_template("app/datasets/licence.html", agree=True, dataset_id=dataset_id, licence_type='direct') if accepted: current_app.logger.info( 'Redirecting to download page for dataset: {}'.format( dataset_id)) return redirect(url_for('.get_details', dataset_id=dataset_id)) licence_file = open( os.path.join( directory, '../documents/datasets/{}/licence.md').format(dataset_id), "r") md_licence = licence_file.read() md_renderer = markdown.Markdown() md_html = md_renderer.convert(md_licence) current_app.logger.info( 'Displaying agree licence page for dataset: {}'.format(dataset_id)) return render_template("app/datasets/licence.html", agree=True, dataset_id=dataset_id, md=md_html) except Exception as e: raise ApplicationError( 'Something went wrong when retrieving licence agree page: {}'. format(e))
def post_agree_licence(dataset_id): try: ulapd_api = UlapdAPI() dataset_details = ulapd_api.get_dataset_by_name(dataset_id) if request.form.get('agree-licence') is None: is_freemium = dataset_details['type'] == 'freemium' md_html = '' # Until we convert licence MD files to HTML if not is_freemium: licence_file = open( os.path.join( directory, f'../documents/datasets/{dataset_id}/licence.md'), "r") md_licence = licence_file.read() md_renderer = markdown.Markdown() md_html = md_renderer.convert(md_licence) current_app.logger.info( 'Displaying licence page with errors for dataset: {}'.format( dataset_id)) error_msg = 'You need to agree to the terms and conditions to download data' return render_template( "app/datasets/licence.html", agree=True, dataset_id=dataset_id, licence_type='direct', md=md_html, error_title="There are errors on this page", fields={'agree-licence': { 'data': '', 'error': [error_msg] }}) else: # Prevent users signing licences for nps/dad etc via the service if dataset_details['type'] not in ['confidential', 'restricted']: accept_licence(dataset_id) current_app.logger.info( 'Redirecting to download page for dataset: {}'.format( dataset_id)) session = dps_session.get_state() if dataset_id == 'nps_sample': return redirect(url_for('.get_details', dataset_id='nps')) return redirect(url_for('general.get_list')) except Exception as e: raise ApplicationError( 'Something went wrong when retrieving licence agree page: {}'. format(e))
def test_authenticate_fail(self, mock_request): error = ('ulapd_ui', 'API_KEY_ERROR') m = Mock() m.get_user_details.side_effect = ApplicationError('Test') mock_request.headers = {} mock_request.headers['Authorization'] = '1234' with self.assertRaises(ApplicationError) as cm: _authenticate(m) self.assertEqual(cm.exception.message, errors.get_message(*error, filler='1234')) self.assertEqual(cm.exception.code, errors.get_code(*error))
def decorated(*args, **kwargs): try: if g.user: ulapd_api = UlapdAPI() session = dps_session.get_state() user_details = ulapd_api.get_user_details('email', g.user) session['user'].update(user_details) dps_session.commit() return f(*args, **kwargs) except Exception as e: raise ApplicationError( 'Something went wrong when refreshing user session: {}'.format( e))
def get_api_dataset_by_name(name): ulapd_api = UlapdAPI() # authenticate _authenticate(ulapd_api) dataset_details = ulapd_api.get_dataset_by_name(name) if dataset_details: response = {"success": True, "result": dataset_details} return response else: raise ApplicationError(*errors.get('ulapd_ui', 'DATASET_NOT_FOUND', filler=name), http_code=404)
def accept_licence(dataset_id): session = dps_session.get_state() try: ulapd_api = UlapdAPI() user_details = session['user']['user_details'] send_metric(dataset_id, 'licence agreed', user_details['user_details_id'], user_details, None) data = { 'user_details_id': user_details['user_details_id'], 'licence_id': dataset_id } ulapd_api.create_licence_agreement(data) except Exception as e: raise ApplicationError('Error accepting licence: {}'.format(str(e)))
def build_download_history(): download_history = [] session = dps_session.get_state() if g.user: try: ulapd_api = UlapdAPI() user_id = session['user']['user_details']['user_details_id'] download_activities = ulapd_api.get_user_download_activity(user_id) download_activities = get_latest_download_activities(download_activities) if download_activities: for download_activity in download_activities: dataset_id = download_activity['dataset_id'] package_details = ulapd_api.get_dataset_by_name(dataset_id) download_datetime = datetime.strptime(download_activity['timestamp'], '%a, %d %b %Y %H:%M:%S %Z') last_update_datetime = datetime.strptime(package_details['last_updated'], '%d %B %Y') is_latest_download = False if download_datetime < last_update_datetime else True licence_agree_string = None license_exists = package_details.get('licence_id') if license_exists: licence_agree_date = session['user']['datasets'][dataset_id]['date_agreed'] licence_agree_datetime = datetime.strptime(licence_agree_date, '%a, %d %b %Y %H:%M:%S %Z') licence_agree_string = datetime.strftime(licence_agree_datetime, '%d %B %Y') download_history.append({ 'dataset_id': dataset_id, 'dataset_title': package_details['title'], 'last_download_date': datetime.strftime(download_datetime, '%d %B %Y'), 'last_update_date': datetime.strftime(last_update_datetime, '%d %B %Y'), 'licence_exists': license_exists, 'is_latest_download': is_latest_download, 'licence_agree_date': licence_agree_string, 'is_licence_agreed': check_agreement(dataset_id), 'resources': package_details['resources'] }) except Exception as e: raise ApplicationError('Error building download history: {}'.format(e)) current_app.logger.info('Returning history of file downloads: {}'.format(download_history)) return download_history
def build_dataset_details(dataset_id, full_info=True): try: dataset = {} if full_info is False: return dataset # If full info is set to True the method will continue to fetch more information from documents directory dataset_sample = json.loads(open(directory + "/documents/datasets/{}/sample.json".format( dataset_id), 'r').read()) details = { "example_data": dataset_sample.get('example_data', {}) } # Add details to dataset dataset.update(details) return dataset except Exception as e: raise ApplicationError('Error building dataset details: {}'.format(e))
def __call__(self, field, **kwargs): if self.multiple: raise ApplicationError( 'Please do not render mutliselect elements as a select box' ' - you should use checkboxes instead in order to comply with' ' the GOV.UK service manual') kwargs.setdefault("id", field.id) if "required" not in kwargs and "required" in getattr( field, "flags", []): kwargs["required"] = True kwargs['items'] = [] # Construct select box choices for val, label, selected in field.iter_choices(): item = {'text': label, 'value': val, 'selected': selected} kwargs['items'].append(item) return super().__call__(field, **kwargs)
def get_api_download_link(dataset_name, file_name, date=None): try: ulapd_api = UlapdAPI() dataset_details = ulapd_api.get_dataset_by_name(dataset_name) # authenticate user_details = _authenticate(ulapd_api) # check agreement agreement = user_details['datasets'].get(dataset_name) if not agreement: if dataset_details['private'] is True: raise ApplicationError(*errors.get('ulapd_ui', 'NO_DATASET_ACCESS', filler=dataset_name), http_code=403) raise ApplicationError(*errors.get('ulapd_ui', 'NO_LICENCE_SIGNED', filler=dataset_name), http_code=403) if agreement['valid_licence'] is False: raise ApplicationError(*errors.get('ulapd_ui', 'NO_LICENCE_SIGNED', filler=dataset_name), http_code=403) # check to see if the filename exists for history files if date: resource_exists = False history_details = ulapd_api.get_dataset_history(dataset_name) for history in history_details['dataset_history']: exist = any( map(lambda resource: resource['file_name'] == file_name, history['resource_list'])) if exist: resource_exists = True break else: # check to see if the filename exists for latest files resource_exists = any( map(lambda resource: resource['file_name'] == file_name, dataset_details['resources'])) if not resource_exists: # check to see if the filename exists for public files if 'public_resources' in dataset_details: public_exists = any( map(lambda public: public['file_name'] == file_name, dataset_details['public_resources'])) if not public_exists: raise ApplicationError(*errors.get('ulapd_ui', 'FILE_DOES_NOT_EXIST', filler=file_name), http_code=404) else: raise ApplicationError(*errors.get('ulapd_ui', 'FILE_DOES_NOT_EXIST', filler=file_name), http_code=404) if date: link = ulapd_api.get_history_download_link(dataset_name, file_name, date) else: link = ulapd_api.get_download_link(dataset_name, file_name) if link: response = { "success": True, "result": { "resource": file_name, "valid_for_seconds": 10, "download_url": link["link"] } } # Activity create ulapd_api.create_activity( user_details['user_details']['user_details_id'], 'download', request.remote_addr, True, file_name, dataset_name) send_metric(dataset_name, 'download api', user_details['user_details']['user_details_id'], user_details['user_details'], file_name) return response else: current_app.logger.error( 'There was a problem getting the resource: '.format(file_name)) raise ApplicationError(*errors.get('ulapd_ui', 'DATASET_NOT_FOUND', filler=dataset_name), http_code=404) except ApplicationError as error: raise error except Exception as e: raise e
def download(dataset_id, file_name, last_updated): try: ulapd_api = UlapdAPI() dataset = ulapd_api.get_dataset_by_name(dataset_id) # First check to see if its a public resource if last_updated is None: for resource in dataset['public_resources']: current_app.logger.info("Public: " + str(resource['file_name'])) if resource['file_name'] == file_name: current_app.logger.info( "Public file download, skipping checks") url = ulapd_api.get_download_link(dataset_id, resource['file_name']) return redirect(url['link']) if dataset['type'] != 'open': # Need the session to get infor about the dataset and user session = dps_session.get_state() user_id = session['user']['user_details']['user_details_id'] user_data = session['user']['user_details'] # 1. Check if user is authenticated if not dps_session.is_logged_in(): return '/sign-in' # 2. Check if user has signed the correct licence if check_agreement(dataset_id) is not True: current_app.logger.info("User has no access to dataset") return url_for('datasets.get_agree_licence', dataset_id=dataset_id) # 3. Generate link if last_updated: last_updated = historic_date_formatter( last_updated, dataset['update_frequency']) url = ulapd_api.get_history_download_link( dataset_id, file_name, last_updated) activity = 'history download' else: url = ulapd_api.get_download_link(dataset_id, file_name) activity = 'download' # 4. Track the download and return (create activity) ulapd_api.create_activity( session['user']['user_details']['user_details_id'], "download", request.remote_addr, False, file_name, dataset_id) send_metric(dataset_id, activity + " ui", user_id, user_data, file_name) else: # 1. Generate link if last_updated: last_updated = historic_date_formatter( last_updated, dataset['update_frequency']) url = ulapd_api.get_history_download_link( dataset_id, file_name, last_updated) else: url = ulapd_api.get_download_link(dataset_id, file_name) return redirect(url['link']) except Exception as e: raise ApplicationError( 'Tracking download has failed - error: {}'.format(e))
def get_details(dataset_id): try: ulapd_api = UlapdAPI() session = dps_session.get_state() if dataset_id == 'nps_sample': return redirect(url_for('.get_details', dataset_id='nps')) # Go get the individual dataset details dataset_details = ulapd_api.get_dataset_by_name(dataset_id) # Fail nicely if the dataset doesnt exist if dataset_id not in dataset_details['name']: raise ApplicationError( 'Unable to display dataset details: dataset does not exist', http_code=404) # Now get the example json and to our dataset list extras = build_dataset_details(dataset_id) # Add details to dataset dataset_details.update(extras) # get dataset for dataset_id to check private boolean, if false is non restricted, if true is restricted is_restricted = dataset_details['private'] licence_signed = check_agreement(dataset_id) if dataset_id == 'nps': licence_signed = { 'nps': check_agreement(dataset_id), 'nps_sample': check_agreement('nps_sample') } # Handle freemium licencing: # If a user has signed exploratory/commercial licences they should still be able to sign direct licence if dataset_details['type'] == 'freemium': if g.user: licence_signed = True dataset = session['user']['datasets'].get(dataset_id) if not dataset: licence_signed = False if dataset: if 'Direct Use' not in dataset['licences']: licence_signed = False if len(dataset['licences']) == 1: if 'Direct' in dataset['licences'][ 0] and not dataset['valid_licence']: licence_signed = False current_app.logger.info( 'Displaying details for user requested dataset: {}'.format( dataset_id)) breadcrumb_links = [{ "label": "Home", "href": "/" }, { "label": dataset_details['title'], "href": None }] return render_template( "app/datasets/{}/details.html".format(dataset_id), dataset_details=dataset_details, licence_signed=licence_signed, is_restricted=is_restricted, readable_date=dataset_details['last_updated'], breadcrumb_links=breadcrumb_links) except ApplicationError as e: raise ApplicationError( 'Something went wrong when retrieving dataset details: {}'.format( e))
def get_list(): try: ulapd_api = UlapdAPI() session = dps_session.get_state() internal_datasets = ulapd_api.get_datasets() external_datasets = ulapd_api.get_external_datasets() # Filter out sample internal_datasets = [ d for d in internal_datasets if '_sample' not in d['name'] ] # Add internal/external datasets together dataset_list = internal_datasets + external_datasets # Sort alphabetically putting datasets starting with numeric characters last dataset_list.sort( key=lambda d: 'z' if d['title'][0].isdigit() else d['title']) # User specific data user_access = {} api_key = '' user_has_activity = False if g.user: api_key = session['user']['user_details']['api_key'] # Dictionary of datasets user has access to user_access = {d: True for d in session['user']['datasets']} # Check if user has downloaded anything for 'agreed licence but not downloaded' state user_activity = ulapd_api.get_user_download_activity( session['user']['user_details']['user_details_id']) user_has_activity = bool(user_activity) # Get dataset history for agreed datasets agreed_dataset_list = [] for dataset in dataset_list: if check_agreement(dataset['name']): dataset['history'] = ulapd_api.get_dataset_history( dataset['name']) agreed_dataset_list.append(dataset) # Filter out confidential (e.g. DAD dataset) from listings page dataset_list = [d for d in dataset_list if 'confidential' != d['type']] freemium_licences = {} for dataset in user_access: if dataset == 'res_cov' or dataset == 'leases': dataset_licence = session['user']['datasets'][dataset][ 'licences'] if len(dataset_licence) == 1: licence_string = '{} licence'.format(dataset_licence[0]) freemium_licences[dataset] = licence_string else: start = ", ".join(dataset_licence[:-1]) licence_string = '{} and {} licences'.format( start, dataset_licence[-1]) freemium_licences[dataset] = licence_string return render_template('app/datasets/index.html', datasets_list=dataset_list, api_key=api_key, user_access=user_access, agreed_dataset_list=agreed_dataset_list, dps_session=session, user_has_activity=user_has_activity, freemium_licences=freemium_licences) except ApplicationError as e: raise ApplicationError( 'Something went wrong when retrieving the datasets - error: {}'. format(e))