def table_data_view(request, database, schema, table):
logger.info(
"table_data_view attempt: %s %s %s %s",
request.user.email,
database,
schema,
table,
)
log_event(
request.user,
EventLog.TYPE_DATASET_TABLE_DATA_DOWNLOAD,
extra={
"path": request.get_full_path(),
"database": database,
"schema": schema,
"table": table,
},
)
if request.method != "GET":
return HttpResponseNotAllowed(["GET"])
elif not can_access_schema_table(request.user, database, schema, table):
return HttpResponseForbidden()
elif not (view_exists(database, schema, table)
or table_exists(database, schema, table)):
return HttpResponseNotFound()
else:
return table_data(request.user.email, database, schema, table)
def get(self, request, *args, **kwargs):
dataset = find_dataset(self.kwargs.get('dataset_uuid'), request.user)
if not dataset.user_has_access(self.request.user):
return HttpResponseForbidden()
query = get_object_or_404(self.model,
id=self.kwargs.get('query_id'),
dataset=dataset)
if not query.reviewed and not request.user.is_superuser:
return HttpResponseForbidden()
log_event(
request.user,
EventLog.TYPE_DATASET_CUSTOM_QUERY_DOWNLOAD,
query.dataset,
extra={
'path': request.get_full_path(),
**serializers.serialize('python', [query])[0],
},
)
dataset.number_of_downloads = F('number_of_downloads') + 1
dataset.save(update_fields=['number_of_downloads'])
return streaming_query_response(
request.user.email,
query.database.memorable_name,
sql.SQL(query.query),
query.get_filename(),
)
def table_data_view(request, database, schema, table):
logger.info(
'table_data_view attempt: %s %s %s %s',
request.user.email,
database,
schema,
table,
)
log_event(
request.user,
EventLog.TYPE_DATASET_TABLE_DATA_DOWNLOAD,
extra={
'path': request.get_full_path(),
'database': database,
'schema': schema,
'table': table,
},
)
if request.method != 'GET':
return HttpResponseNotAllowed(['GET'])
elif not can_access_schema_table(request.user, database, schema, table):
return HttpResponseForbidden()
elif not (view_exists(database, schema, table)
or table_exists(database, schema, table)):
return HttpResponseNotFound()
else:
return table_data(request.user.email, database, schema, table)
def save(self,
force_insert=False,
force_update=False,
using=None,
update_fields=None):
if self._initial_approved is False and self.approved is True:
raise ValueError(
"A new record must be created for a new approval - you cannot flip a rescinded approval."
)
elif self._initial_approved is self.approved and self.modified_date is not None:
raise ValueError(
"The only change that can be made to an approval is to unapprove it."
)
super().save(force_insert, force_update, using, update_fields)
if self.approved:
log_event(self.approver,
EventLog.TYPE_VISUALISATION_APPROVED,
related_object=self)
else:
log_event(
self.approver,
EventLog.TYPE_VISUALISATION_UNAPPROVED,
related_object=self,
)
self._initial_approved = self.approved
def post(self, request, *args, **kwargs):
action = request.POST.get('action', '')
play_sql = get_playground_sql_from_request(request)
if action == 'edit':
if not play_sql:
play_sql = PlaygroundSQL(sql=request.POST.get("sql", ""),
created_by_user=request.user)
play_sql.save()
redirect_url = reverse('explorer:index')
return redirect(redirect_url + f"?play_id={play_sql.id}")
if action == 'save':
ret = super().post(request)
if self.get_form().is_valid():
query, form = QueryView.get_instance_and_form(
request, self.object.id, play_sql=play_sql)
if form.is_valid():
form.save()
vm = query_viewmodel(
request,
query,
form=form,
run_query=False,
rows=url_get_rows(request),
page=url_get_page(request),
message=None,
)
if vm['form'].errors:
self.object.delete()
del vm['query']
vm['form_action'] = request.get_full_path()
return render(request, self.template_name, vm)
messages.success(request, "Your query has been saved.")
log_event(
request.user,
EventLog.TYPE_DATA_EXPLORER_SAVED_QUERY,
related_object=query,
extra={"sql": query.sql},
)
return HttpResponseRedirect(
reverse_lazy('explorer:query_detail',
kwargs={'query_id': self.object.id}))
return ret
elif action == 'edit':
query_params = (('sql', request.POST.get('sql')), )
return HttpResponseRedirect(
reverse('explorer:index') + f"?{urlencode(query_params)}")
else:
return HttpResponse(f"Unknown form action: {action}", 400)
def get(self, request, *args, **kwargs):
dataset = find_dataset(
self.kwargs.get('group_slug'), self.kwargs.get('set_slug')
)
if not dataset.user_has_access(self.request.user):
return HttpResponseForbidden()
query = get_object_or_404(
self.model, id=self.kwargs.get('query_id'), dataset=dataset
)
log_event(
request.user,
EventLog.TYPE_DATASET_CUSTOM_QUERY_DOWNLOAD,
query.dataset,
extra={'path': request.get_full_path(), **model_to_dict(query)},
)
dataset.number_of_downloads = F('number_of_downloads') + 1
dataset.save(update_fields=['number_of_downloads'])
return streaming_query_response(
request.user.email,
query.database.memorable_name,
sql.SQL(query.query),
query.get_filename(),
)
def request_access_view(request, dataset_uuid):
dataset = find_dataset_or_visualisation(dataset_uuid, request.user)
if request.method == 'POST':
form = RequestAccessForm(request.POST)
if form.is_valid():
goal = form.cleaned_data['goal']
contact_email = form.cleaned_data['email']
user_edit_relative = reverse('admin:auth_user_change',
args=[request.user.id])
user_url = request.build_absolute_uri(user_edit_relative)
dataset_url = request.build_absolute_uri(
dataset.get_absolute_url())
if (isinstance(dataset, VisualisationCatalogueItem)
and dataset.visualisation_template):
ticket_reference = _notify_visualisation_access_request(
request, dataset, dataset_url, contact_email, goal)
else:
ticket_reference = create_zendesk_ticket(
contact_email,
request.user,
goal,
user_url,
dataset.name,
dataset_url,
dataset.information_asset_owner,
dataset.information_asset_manager,
)
log_event(
request.user,
EventLog.TYPE_DATASET_ACCESS_REQUEST,
dataset,
extra={
'contact_email': contact_email,
'goal': goal,
'ticket_reference': ticket_reference,
},
)
url = reverse('datasets:request_access_success',
args=[dataset_uuid])
return HttpResponseRedirect(f'{url}?ticket={ticket_reference}')
return render(
request,
'request_access.html',
{
'dataset': dataset,
'authenticated_user': request.user,
'is_visualisation': isinstance(dataset,
VisualisationCatalogueItem),
},
)
def get(self, request, *args, **kwargs):
dl_format = self.kwargs.get('format')
if dl_format not in ['json', 'csv']:
raise Http404
ref_dataset = self.get_object()
records = []
for record in ref_dataset.get_records():
record_data = {}
for field in ref_dataset.fields.all():
field_name = field.name
value = getattr(record, field.column_name)
# If this is a linked field display the display name and id of that linked record
if field.data_type == ReferenceDatasetField.DATA_TYPE_FOREIGN_KEY:
record_data[get_linked_field_identifier_name(field)] = (
value.get_identifier() if value is not None else None
)
record_data[get_linked_field_display_name(field)] = (
value.get_display_name() if value is not None else None
)
else:
record_data[field_name] = value
records.append(record_data)
response = HttpResponse()
response['Content-Disposition'] = 'attachment; filename={}-{}.{}'.format(
ref_dataset.slug, ref_dataset.published_version, dl_format
)
log_event(
request.user,
EventLog.TYPE_REFERENCE_DATASET_DOWNLOAD,
ref_dataset,
extra={
'path': request.get_full_path(),
'reference_dataset_version': ref_dataset.published_version,
'download_format': dl_format,
},
)
ref_dataset.number_of_downloads = F('number_of_downloads') + 1
ref_dataset.save(update_fields=['number_of_downloads'])
if dl_format == 'json':
response['Content-Type'] = 'application/json'
response.write(json.dumps(list(records), cls=DjangoJSONEncoder))
else:
response['Content-Type'] = 'text/csv'
with closing(io.StringIO()) as outfile:
writer = csv.DictWriter(
outfile,
fieldnames=ref_dataset.export_field_names,
quoting=csv.QUOTE_NONNUMERIC,
)
writer.writeheader()
writer.writerows(records)
response.write(outfile.getvalue()) # pylint: disable=no-member
return response
def get(self, request, *args, **kwargs):
dataset = find_dataset(self.kwargs.get('dataset_uuid'), request.user)
if not dataset.user_has_access(self.request.user):
return HttpResponseForbidden()
source_link = get_object_or_404(SourceLink,
id=self.kwargs.get('source_link_id'),
dataset=dataset)
log_event(
request.user,
EventLog.TYPE_DATASET_SOURCE_LINK_DOWNLOAD,
source_link.dataset,
extra={
'path': request.get_full_path(),
**serializers.serialize('python', [source_link])[0],
},
)
dataset.number_of_downloads = F('number_of_downloads') + 1
dataset.save(update_fields=['number_of_downloads'])
if source_link.link_type == source_link.TYPE_EXTERNAL:
return HttpResponseRedirect(source_link.url)
client = boto3.client('s3')
try:
file_object = client.get_object(Bucket=settings.AWS_UPLOADS_BUCKET,
Key=source_link.url)
except ClientError as ex:
try:
return HttpResponse(
status=ex.response['ResponseMetadata']['HTTPStatusCode'])
except KeyError:
return HttpResponseServerError()
response = StreamingHttpResponseWithoutDjangoDbConnection(
file_object['Body'].iter_chunks(chunk_size=65536),
content_type=file_object['ContentType'],
)
response[
'Content-Disposition'] = f'attachment; filename="{source_link.get_filename()}"'
response['Content-Length'] = file_object['ContentLength']
return response
def get(self, request, *args, **kwargs):
dataset = find_dataset(
self.kwargs.get('group_slug'), self.kwargs.get('set_slug')
)
if not dataset.user_has_access(self.request.user):
return HttpResponseForbidden()
source_link = get_object_or_404(
SourceLink, id=self.kwargs.get('source_link_id'), dataset=dataset
)
log_event(
request.user,
EventLog.TYPE_DATASET_SOURCE_LINK_DOWNLOAD,
source_link.dataset,
extra={'path': request.get_full_path(), **model_to_dict(source_link)},
)
dataset.number_of_downloads = F('number_of_downloads') + 1
dataset.save(update_fields=['number_of_downloads'])
if source_link.link_type == source_link.TYPE_EXTERNAL:
return HttpResponseRedirect(source_link.url)
client = boto3.client('s3')
try:
file_object = client.get_object(
Bucket=settings.AWS_UPLOADS_BUCKET, Key=source_link.url
)
except ClientError as ex:
try:
return HttpResponse(
status=ex.response['ResponseMetadata']['HTTPStatusCode']
)
except KeyError:
return HttpResponseServerError()
response = StreamingHttpResponse(
file_object['Body'].iter_chunks(), content_type=file_object['ContentType']
)
response['Content-Disposition'] = 'attachment; filename="{}"'.format(
os.path.split(source_link.url)[-1]
)
return response
def get(self, request, *args, **kwargs):
access_request = self.get_object()
if not access_request.zendesk_reference_number:
catalogue_item = (find_dataset(access_request.catalogue_item_id,
self.request.user)
if access_request.catalogue_item_id else None)
if (isinstance(catalogue_item, VisualisationCatalogueItem)
and catalogue_item.visualisation_template is not None):
access_request.zendesk_reference_number = (
zendesk.notify_visualisation_access_request(
request,
access_request,
catalogue_item,
))
else:
access_request.zendesk_reference_number = zendesk.create_zendesk_ticket(
request,
access_request,
catalogue_item,
)
access_request.save()
if catalogue_item:
log_event(
request.user,
EventLog.TYPE_DATASET_ACCESS_REQUEST,
catalogue_item,
extra={
"ticket_reference":
access_request.zendesk_reference_number,
},
)
else:
log_event(
request.user,
EventLog.TYPE_TOOLS_ACCESS_REQUEST,
extra={
"ticket_reference":
access_request.zendesk_reference_number,
},
)
return super().get(request, *args, **kwargs)
def post(self, request, query_id):
action = request.POST.get("action", "")
play_sql = get_playground_sql_from_request(request)
if action == 'save':
show = url_get_show(request)
query, form = QueryView.get_instance_and_form(request,
query_id,
play_sql=play_sql)
success = form.is_valid() and form.save()
vm = query_viewmodel(
request.user,
query,
form=form,
run_query=show,
rows=url_get_rows(request),
page=url_get_page(request),
message=None,
)
if success:
messages.success(request, "Your query has been updated.")
log_event(
request.user,
EventLog.TYPE_DATA_EXPLORER_SAVED_QUERY,
related_object=query,
extra={"sql": query.sql},
)
return redirect(
reverse('explorer:query_detail',
kwargs={"query_id": query.id}))
vm['form_action'] = request.get_full_path()
return render(self.request, 'explorer/query.html', vm)
elif action == 'edit':
query, _ = QueryView.get_instance_and_form(request,
query_id,
play_sql=play_sql)
return HttpResponseRedirect(self.get_edit_sql_url(request, query))
else:
return HttpResponseBadRequest(f"Unknown form action: {action}")
def post(self, request, subscription_id):
if subscription_id == "__all__":
# Ignore the post values and just delete all subscriptions
unsubscribed = unsubscribe_from_all(request.user)
else:
unsubscribed = [unsubscribe(subscription_id, request.user)]
for subscription in unsubscribed:
log_event(
request.user,
EventLog.TYPE_DATASET_NOTIFICATIONS_UNSUBSCRIBED,
related_object=subscription,
)
return render(
request,
"datasets/subscriptions/unsubscribe_confirm.html",
context={"subscriptions": unsubscribed},
)
def form_valid(self, form):
if form.cleaned_data["notification_type"] == NotificationType.COLUMNS:
form.instance.notify_on_schema_change = True
form.instance.notify_on_data_change = False
elif form.cleaned_data[
"notification_type"] == NotificationType.ALL_CHANGES:
form.instance.notify_on_schema_change = True
form.instance.notify_on_data_change = True
log_event(
self.request.user,
EventLog.TYPE_DATASET_NOTIFICATIONS_SUBSCRIBED,
related_object=form.instance,
extra={
"notify_on_schema_change":
form.instance.notify_on_schema_change,
"notify_on_data_change": form.instance.notify_on_data_change,
},
)
return super().form_valid(form)
def get(self, request, *_, **__):
dataset = find_dataset(
self.kwargs.get('group_slug'), self.kwargs.get('set_slug')
)
db_object = get_object_or_404(
self.model, id=self.kwargs.get('source_id'), dataset=dataset
)
if not db_object.dataset.user_has_access(self.request.user):
return HttpResponseForbidden()
if not self.db_object_exists(db_object):
return HttpResponseNotFound()
log_event(
request.user,
self.event_log_type,
db_object.dataset,
extra={'path': request.get_full_path(), **model_to_dict(db_object)},
)
dataset.number_of_downloads = F('number_of_downloads') + 1
dataset.save(update_fields=['number_of_downloads'])
return self.get_table_data(db_object)
