def get_download_page(dataset_id):
ulapd_api = UlapdAPI()
if dataset_id == 'rfi':
dataset, history = build_rfi_dataset_for_download(
ulapd_api.get_dataset_by_name(dataset_id),
ulapd_api.get_dataset_history(dataset_id))
else:
dataset = ulapd_api.get_dataset_by_name(dataset_id)
history = ulapd_api.get_dataset_history(dataset_id)
breadcrumb_links = [{
"label": "Home",
"href": "/"
}, {
"label": dataset['title'],
"href": "/datasets/" + dataset['name']
}, {
"label": "Download dataset",
"href": None
}]
return render_template('app/datasets/{}/download.html'.format(dataset_id),
dataset=dataset,
history=history,
breadcrumb_links=breadcrumb_links)
def get_api_dataset_history(name):
ulapd_api = UlapdAPI()
# authenticate
_authenticate(ulapd_api)
history_details = ulapd_api.get_dataset_history(name)
if history_details:
history_data = []
for history in history_details['dataset_history']:
for file in history['resource_list']:
history_info = {
"last_updated": history["last_updated"],
"unsorted_date": history["unsorted_date"],
"filename": file['file_name'],
"file_size": file['file_size']
}
history_data.append(history_info)
response = {
"success": True,
"dataset": name,
"dataset_history": history_data
}
return response
else:
current_app.logger.error('Dataset {} not found'.format(name))
raise ApplicationError(*errors.get('ulapd_ui',
'DATASET_NOT_FOUND',
filler=name),
http_code=404)
def test_get_dataset_history(self, mock_get):
mock_get.return_value.json.return_value = {'name': 'ccod'}
mock_get.return_value.status_code = 200
ulapd_api = UlapdAPI()
response = ulapd_api.get_dataset_history('ccod')
self.assertEqual(response, {'name': 'ccod'})
def get_api_download_link(dataset_name, file_name, date=None):
try:
ulapd_api = UlapdAPI()
dataset_details = ulapd_api.get_dataset_by_name(dataset_name)
# authenticate
user_details = _authenticate(ulapd_api)
# check agreement
agreement = user_details['datasets'].get(dataset_name)
if not agreement:
if dataset_details['private'] is True:
raise ApplicationError(*errors.get('ulapd_ui',
'NO_DATASET_ACCESS',
filler=dataset_name),
http_code=403)
raise ApplicationError(*errors.get('ulapd_ui',
'NO_LICENCE_SIGNED',
filler=dataset_name),
http_code=403)
if agreement['valid_licence'] is False:
raise ApplicationError(*errors.get('ulapd_ui',
'NO_LICENCE_SIGNED',
filler=dataset_name),
http_code=403)
# check to see if the filename exists for history files
if date:
resource_exists = False
history_details = ulapd_api.get_dataset_history(dataset_name)
for history in history_details['dataset_history']:
exist = any(
map(lambda resource: resource['file_name'] == file_name,
history['resource_list']))
if exist:
resource_exists = True
break
else:
# check to see if the filename exists for latest files
resource_exists = any(
map(lambda resource: resource['file_name'] == file_name,
dataset_details['resources']))
if not resource_exists:
# check to see if the filename exists for public files
if 'public_resources' in dataset_details:
public_exists = any(
map(lambda public: public['file_name'] == file_name,
dataset_details['public_resources']))
if not public_exists:
raise ApplicationError(*errors.get('ulapd_ui',
'FILE_DOES_NOT_EXIST',
filler=file_name),
http_code=404)
else:
raise ApplicationError(*errors.get('ulapd_ui',
'FILE_DOES_NOT_EXIST',
filler=file_name),
http_code=404)
if date:
link = ulapd_api.get_history_download_link(dataset_name, file_name,
date)
else:
link = ulapd_api.get_download_link(dataset_name, file_name)
if link:
response = {
"success": True,
"result": {
"resource": file_name,
"valid_for_seconds": 10,
"download_url": link["link"]
}
}
# Activity create
ulapd_api.create_activity(
user_details['user_details']['user_details_id'], 'download',
request.remote_addr, True, file_name, dataset_name)
send_metric(dataset_name, 'download api',
user_details['user_details']['user_details_id'],
user_details['user_details'], file_name)
return response
else:
current_app.logger.error(
'There was a problem getting the resource: '.format(file_name))
raise ApplicationError(*errors.get('ulapd_ui',
'DATASET_NOT_FOUND',
filler=dataset_name),
http_code=404)
except ApplicationError as error:
raise error
except Exception as e:
raise e
def get_list():
try:
ulapd_api = UlapdAPI()
session = dps_session.get_state()
internal_datasets = ulapd_api.get_datasets()
external_datasets = ulapd_api.get_external_datasets()
# Filter out sample
internal_datasets = [
d for d in internal_datasets if '_sample' not in d['name']
]
# Add internal/external datasets together
dataset_list = internal_datasets + external_datasets
# Sort alphabetically putting datasets starting with numeric characters last
dataset_list.sort(
key=lambda d: 'z' if d['title'][0].isdigit() else d['title'])
# User specific data
user_access = {}
api_key = ''
user_has_activity = False
if g.user:
api_key = session['user']['user_details']['api_key']
# Dictionary of datasets user has access to
user_access = {d: True for d in session['user']['datasets']}
# Check if user has downloaded anything for 'agreed licence but not downloaded' state
user_activity = ulapd_api.get_user_download_activity(
session['user']['user_details']['user_details_id'])
user_has_activity = bool(user_activity)
# Get dataset history for agreed datasets
agreed_dataset_list = []
for dataset in dataset_list:
if check_agreement(dataset['name']):
dataset['history'] = ulapd_api.get_dataset_history(
dataset['name'])
agreed_dataset_list.append(dataset)
# Filter out confidential (e.g. DAD dataset) from listings page
dataset_list = [d for d in dataset_list if 'confidential' != d['type']]
freemium_licences = {}
for dataset in user_access:
if dataset == 'res_cov' or dataset == 'leases':
dataset_licence = session['user']['datasets'][dataset][
'licences']
if len(dataset_licence) == 1:
licence_string = '{} licence'.format(dataset_licence[0])
freemium_licences[dataset] = licence_string
else:
start = ", ".join(dataset_licence[:-1])
licence_string = '{} and {} licences'.format(
start, dataset_licence[-1])
freemium_licences[dataset] = licence_string
return render_template('app/datasets/index.html',
datasets_list=dataset_list,
api_key=api_key,
user_access=user_access,
agreed_dataset_list=agreed_dataset_list,
dps_session=session,
user_has_activity=user_has_activity,
freemium_licences=freemium_licences)
except ApplicationError as e:
raise ApplicationError(
'Something went wrong when retrieving the datasets - error: {}'.
format(e))