def load_nester_service(request):
s1 = Service.objects.get(pk=settings.WEC_LOCATION_EVALUATION_SERVICE_ID)
s2 = Service.objects.get(pk=settings.LOCATION_EVALUATION_SERVICE_ID)
access_decision1 = PEP.access_to_service(request, s1.id)
access_decision2 = PEP.access_to_service(request, s2.id)
return render(request, 'services/services/load_nester_service.html', {
'access_decision1': access_decision1,
'access_decision2': access_decision2
})
def load_hcmr_service(request):
s1 = Service.objects.get(pk=settings.OIL_SPILL_FORECAST_SERVICE_ID)
s2 = Service.objects.get(pk=settings.HIGH_RISK_POLLUTION_SERVICE_ID)
s3 = Service.objects.get(pk=settings.UNDERWATER_ACCIDENT_SERVICE_ID)
access_decision1 = PEP.access_to_service(request, s1.id)
access_decision2 = PEP.access_to_service(request, s2.id)
access_decision3 = PEP.access_to_service(request, s3.id)
return render(
request, 'services/services/load_hcmr_service.html', {
'access_decision1': access_decision1,
'access_decision2': access_decision2,
'access_decision3': access_decision3
})
def save_dashboard(request, pk=None):
# create or update
if not pk:
user = request.user
if user.is_authenticated():
dashboard = Dashboard(user=user)
else:
dashboard = Dashboard(user=User.objects.get(
username='******'))
else:
dashboard = Dashboard.objects.get(pk=pk)
# check access
try:
access_decision = PEP.access_to_edit_dashboard(
request, dashboard.id)
if access_decision is False:
raise PermissionDenied
except:
return HttpResponseForbidden()
dashboard.title = 'BDO Dashboard'
print request.POST
dashboard_data = request.POST.dict()
for order in dashboard_data.keys():
dashboard_data = json.loads(order)
print "We are now printing dashboard data"
print dashboard_data
print "end of data"
title = dashboard_data.pop('title', None)
description = dashboard_data.pop('description', None)
private = dashboard_data.pop('private', None)
private = True
can_be_shared = dashboard_data.pop('can_be_shared', None)
for order in dashboard_data.keys():
print order
print dashboard_data[order]
dashboard.viz_components = dashboard_data
dashboard.title = title
dashboard.description = str(description).strip()
dashboard.private = private
dashboard.can_be_shared = can_be_shared
print dashboard.title
dashboard.save()
return JsonResponse({
'pk': dashboard.pk,
})
def load_service(request, pk):
access_decision = PEP.access_to_service(request, pk)
if access_decision is False:
raise PermissionDenied
service = Service.objects.get(pk=pk)
title = service.title
notebook_id = service.notebook_id
queries = service.queries
arguments = service.arguments
output_html = service.output_html
output_css = service.output_css
output_js = service.output_js
# if settings.TEST_SERVICES:
# from bdo_platform.settings import BASE_DIR
# import os
# path = os.path.join(BASE_DIR + '\\service_builder\\templates\\service_builder\\', 'service_template_1.html')
# with open(path, 'r') as f:
# output_html = f.read()
# path = os.path.join(BASE_DIR + '\\service_builder\\static\\service_builder\\css\\', 'service_template_1.css')
# with open(path, 'r') as f:
# output_css = f.read()
# path = os.path.join(BASE_DIR + '\\service_builder\\static\\service_builder\\js\\', 'service_template_1.js')
# with open(path, 'r') as f:
# output_js = f.read()
output_html = output_html.replace('iframe src', 'iframe src-a')
# check if user is the owner or just has been granted access
owner = False
if service.user_id == request.user.id:
owner = True
return render(
request, 'service_builder/load_service.html', {
'service_title': service.title,
'output_html': output_html,
'output_css': output_css,
'output_js': output_js,
'arguments': json.dumps(arguments),
'service_id': pk,
'is_owner': owner,
'published': service.published
})
def test_generating_access_to_dashboard(self):
dashboardTitle = "A title"
dashboardPrivate = True
tempVis2 = [
"", "1", "1", "1", "1", "temptitle",
"<div>Hello this is a test</div>"
]
payload = {
"title": dashboardTitle,
"private": dashboardPrivate,
"0": tempVis2,
"can_be_shared": True
}
payload = json.dumps(payload)
payloadContainer = {payload: ''}
user = User(username='******')
user.save()
cl = Client()
cl.force_login(user)
url = "http://localhost:8000/dashboards/save/"
pk = cl.post(url, data=payloadContainer)
pk = pk.content
pk = json.loads(pk)
accessible_dashboard_pk = pk["pk"]
from django.http import HttpRequest
request1 = HttpRequest()
request1.user = user
decision = PEP.access_to_dashboard(request1, accessible_dashboard_pk)
self.assertEqual(str(decision), str(True))
another_user = User(username='******')
another_user.save()
cl.logout()
cl = Client()
cl.force_login(another_user)
url = "http://localhost:8000/access_control/request_access_to_resource/dashboard/"
payload = {"resource_id": accessible_dashboard_pk}
response = cl.post(url, data=payload)
self.assertEqual(response.status_code, 200)
def view_service(request, pk):
user = request.user
service = Service.objects.get(pk=pk)
# check for the access
try:
access_decision = PEP.access_to_service(request, service.id)
if access_decision is False:
raise PermissionDenied
except:
return HttpResponseForbidden()
return render(
request, 'service_builder/load_service.html', {
'service_id': service.id,
'output_html': service.output_html,
'output_css': service.output_css,
'output_js': service.output_js,
'published': service.published
})
def view_dashboard(request, pk):
user = request.user
try:
dashboard = Dashboard.objects.get(pk=pk)
except ObjectDoesNotExist:
return render(
request, 'error_page.html', {
'message':
'You cannot view this Dashboard!\nThe Dashboard does not exist or has already been deleted!'
})
# check for the access
try:
access_decision = PEP.access_to_dashboard(request, dashboard.id)
if access_decision is False:
raise PermissionDenied
except:
return HttpResponseForbidden()
# check if user is the owner or just has been granted access
owner = False
if dashboard.user_id == user.id:
owner = True
# check if user has rights to edit
can_edit = False
for da in DashboardAccess.objects.filter(dashboard=dashboard, user=user):
if da.can_edit:
can_edit = True
dashboard.viz_components = convert_unicode_json(dashboard.viz_components)
print dashboard.viz_components
dashboard_display(dashboard)
unique_dashboard_display(dashboard, user)
return render(
request, 'services/services/view_dashboard.html', {
'dashboard': dashboard,
'dashboard_json': json.dumps(dashboard.viz_components),
'pk': pk,
'is_owner': owner,
'can_edit': can_edit
})
def edit_dashboard(request, pk=None):
if request.method == 'GET':
user = request.user
if request.user.is_authenticated():
saved_queries = Query.objects.filter(user=user).exclude(
document__from=[])
else:
saved_queries = []
for q in saved_queries:
doc = q.document
changed = False
for f in doc['from']:
for s in f['select']:
if s['type'] == "VALUE":
if 'datatype' not in s.keys():
try:
s['datatype'] = Variable.objects.get(
pk=int(f['type'])).dataType
changed = True
except:
s['datatype'] = 'FLOAT'
pass
else:
if 'datatype' not in s.keys():
try:
s['datatype'] = Dimension.objects.get(
pk=int(s['type'])).dataType
changed = True
except:
s['datatype'] = 'FLOAT'
pass
if changed:
q.document = doc
q.save()
try:
dashboard = Dashboard.objects.get(pk=pk)
except ObjectDoesNotExist:
message = 'You cannot edit this Dashboard!\nThe Dashboard does not exist or has already been deleted!'
return render(request, 'error_page.html', {'message': message})
# try:
# if dashboard.user_id != user.id:
# raise PermissionDenied
# except:
# return HttpResponseForbidden()
# check for the access
try:
access_decision = PEP.access_to_edit_dashboard(
request, dashboard.id)
if access_decision is False:
raise PermissionDenied
except:
return HttpResponseForbidden()
# variables_list = []
# dimensions_list = []
# var_list = Variable.objects.all()
# dim_list = Dimension.objects.all()
# for el in var_list:
# if not (el.name in variables_list):
# variables_list.append(el.name.encode('ascii'))
# for el in dim_list:
# if not (el.name in dimensions_list):
# dimensions_list.append(el.name.encode('ascii'))
# toCreate = dashboard.title
form_class = forms.CkEditorForm
dashboard.viz_components = convert_unicode_json(
dashboard.viz_components)
print dashboard.viz_components
conf_viz_json = ''
try:
with open(
'visualizer/static/visualizer/visualisations_settings.json'
) as f:
conf_viz_json = json.dumps(json.load(f))
except:
pass
# check if user is the owner or just has been granted access
owner = False
if dashboard.user_id == user.id:
owner = True
return render(
request,
'dashboard_builder/dashboard_editor_new.html',
{
'dashboard':
dashboard,
'dashboard_json':
json.dumps(dashboard.viz_components),
'dashboard_pk':
pk,
'dashboard_title':
dashboard.title,
'is_owner':
owner,
'sidebar_active':
'products',
'saved_queries':
saved_queries,
'available_viz':
Visualization.objects.filter(hidden=False).order_by(
'-type', '-title'),
'form':
form_class,
# 'toCreate': toCreate,
# 'variables_list': variables_list,
# 'dimensions_list': dimensions_list,
'visualisation_configuration':
conf_viz_json
})
return None
def execute_query(request, pk=None):
user = request.user
if user.is_authenticated:
doc_str = request.POST.get('query', '')
# get or fake query object
if not pk:
q = AbstractQuery(document=json.loads(doc_str))
else:
q = AbstractQuery.objects.get(pk=pk)
# try:
# q.document = json.loads(doc_str)
# except ValueError:
# pass
q.document['limit'] = 1000000
# print q.document
# print q.raw_query
# get POST params
dimension_values = request.POST.get('dimension_values', '')
variable = request.POST.get('variable', '')
only_headers = request.POST.get('only_headers', '').lower() == 'true'
# check for the access
try:
dataset_list = []
doc = q.document
print q.document
for el in doc['from']:
dataset = Variable.objects.get(id=int(el['type'])).dataset
if dataset.id not in dataset_list:
dataset_list.append(dataset.id)
print dataset_list
for dataset_id in dataset_list:
access_decision = PEP.access_to_dataset_for_query(
request, dataset_id)
if access_decision is False:
raise PermissionDenied
except:
return HttpResponseForbidden()
try:
check_api_calls(user)
except Exception as e:
print 'API call failed because user exceeded the number of allowed API calls or does not have a plan'
traceback.print_exc()
# return render(request, 'error_page.html', {'message': e.message})
return JsonResponse({"error_message": e.message})
try:
# execute
result = q.execute(dimension_values=dimension_values,
variable=variable,
only_headers=only_headers,
with_encoder=True)
except ValueError as ve:
return JsonResponse({"error_message": str(ve.message)})
except Exception as e:
if e.args[0] == 'max_memory_exceeded':
return JsonResponse({
"error_message":
'Your query execution exceeded the available memory. Please limit your selection on space and time and try again.'
})
else:
return JsonResponse({"error_message": str(e.message)})
response, encoder = result
# import pdb
# pdb.set_trace()
# send results
for dataset_list_el_id in dataset_list:
try:
dataset_obj = Dataset.objects.get(id=dataset_list_el_id)
dataset_exploration(dataset_obj)
if len(dataset_list) > 1:
dataset_join(dataset_obj)
except:
pass
return JsonResponse(response, encoder=encoder)
else:
return HttpResponseForbidden()
def submit_service_args(request, service_id):
access_decision = PEP.access_to_service(request, service_id)
if access_decision is False:
raise PermissionDenied
service = Service.objects.get(pk=int(service_id))
if service.title == 'Oil Spill Simulation Service':
output_html = service.output_html
soup = BeautifulSoup(str(output_html), 'html.parser')
service_result_container = soup.find(id="service_result_container")
innerHTML = ''
for c in service_result_container.contents:
innerHTML += str(c)
context = Context({"result": ''})
template = Template(innerHTML)
return HttpResponse(template.render(context))
livy = service.through_livy
service_exec = ServiceInstance(service=service,
user=request.user,
time=datetime.now())
service_exec.save()
# 1.GATHER THE SERVICE ARGUMENTS
service_args = convert_unicode_json(service.arguments)
service_filter_args = service_args["filter-arguments"]
print 'original args:'
print service_filter_args
for filter_arg in service_filter_args:
filter_arg['filter_b'] = request.GET.get(filter_arg['name'],
filter_arg['default'])
print 'user args:'
print service_filter_args
service_algorithm_args = service_args["algorithm-arguments"]
print 'original algorithm args:'
print service_algorithm_args
args_to_note = dict()
for algorithm_arg in service_algorithm_args:
args_to_note[algorithm_arg['name']] = request.GET.get(
algorithm_arg['name'], algorithm_arg['default'])
print 'user algorithm args:'
print args_to_note
service_exec.arguments = {
'filter-arguments': service_filter_args,
'algorithm-arguments': service_algorithm_args
}
service_exec.save()
# 2.CUSTOMIZE THE QUERIES BASED ON THE GATHERED USER ARGUMENTS (AND CREATE TEMPQUERIES)
# query_mapping is a dict that maps the original queries of the template service to the tempQueries created after the user customisation
query_mapping = dict()
queries = convert_unicode_json(service.queries)
if queries is not None:
for name, q in queries.iteritems():
query_id = int(q['query_id'])
query = Query.objects.get(pk=query_id)
doc = query.document
for arg in service_filter_args:
arg_query_id = int(queries[arg['query']]['query_id'])
if arg_query_id == query_id:
filters = convert_unicode_json(doc['filters'])
doc['filters'] = update_filter(filters, arg)
q_temp = TempQuery(original=query,
document=doc,
user_id=query.user_id)
q_temp.save()
q['temp_q'] = q_temp.id
query_mapping[query_id] = q_temp.id
print queries
# 3.BRING THE CUSTOMISED QUERIES TO THE SERVICE CODE
original_notebook_id = service.notebook_id
if settings.TEST_SERVICES:
new_notebook_id = original_notebook_id
excluded_paragraphs = []
new_created_paragraphs = []
else:
excluded_paragraphs = []
new_notebook_id = clone_zep_note(original_notebook_id, "")
clean_up_new_note(new_notebook_id)
service_exec.notebook_id = new_notebook_id
service_exec.save()
print 'Notebook ID: {0}'.format(new_notebook_id)
# customise the respective queries in the code
if queries is not None:
for name, info in queries.iteritems():
for original_paragraph_id in info['paragraphs']:
raw_query = TempQuery.objects.get(
pk=int(info['temp_q'])).raw_query
new_query_paragraph_id = create_zep__query_paragraph(
new_notebook_id,
'',
raw_query,
index=2,
df_name="df_" + name,
livy=True)
if settings.TEST_SERVICES:
excluded_paragraphs.append(original_paragraph_id)
new_created_paragraphs.append(new_query_paragraph_id)
else:
print 'deleting paragraph: {0}'.format(
original_paragraph_id)
delete_zep_paragraph(
notebook_id=str(new_notebook_id),
paragraph_id=str(original_paragraph_id))
new_arguments_paragraph = create_zep_arguments_paragraph(
notebook_id=new_notebook_id,
title='',
args_json_string=json.dumps(args_to_note))
if service.arguments_paragraph_id is not None:
delete_zep_paragraph(new_notebook_id, service.arguments_paragraph_id)
if settings.TEST_SERVICES:
service.arguments_paragraph_id = new_arguments_paragraph
service.save()
output_html = service.output_html
output_html = output_html.replace(original_notebook_id, new_notebook_id)
soup = BeautifulSoup(output_html, 'html.parser')
visualizations = []
for f in soup.findAll('iframe'):
visualizations.append(f.get("src"))
import urlparse
dataframe_viz = []
for url in visualizations:
parsed = urlparse.urlparse(url)
if 'notebook_id' in urlparse.parse_qs(parsed.query).keys():
if 'df' in urlparse.parse_qs(parsed.query).keys():
df = urlparse.parse_qs(parsed.query)['df'][0]
elif 'df1' in urlparse.parse_qs(parsed.query).keys():
df = urlparse.parse_qs(parsed.query)['df1'][0]
elif 'df2' in urlparse.parse_qs(parsed.query).keys():
df = urlparse.parse_qs(parsed.query)['df2'][0]
dataframe_viz.append({
'notebook_id':
urlparse.parse_qs(parsed.query)['notebook_id'][0],
'df':
df,
'url':
url,
'done':
False
})
service_exec.dataframe_visualizations = dataframe_viz
service_exec.save()
# 4.RUN THE SERVICE CODE (one by one paragraph, or all together. CHOOSE..)
try:
if livy:
livy_session = run_zep_note(notebook_id=new_notebook_id,
exclude=excluded_paragraphs,
mode='livy')
service_exec.livy_session = livy_session
service_exec.save()
else:
run_zep_note(notebook_id=new_notebook_id,
exclude=excluded_paragraphs,
mode='zeppelin')
# data = create_livy_toJSON_paragraph()
# with open('df_json_{0}___{1}.json'.format(new_notebook_id, df_name), 'w') as outfile:
# json.dump(data, outfile)
if settings.TEST_SERVICES:
for p in new_created_paragraphs:
print 'deleting paragraph: {0}'.format(p)
delete_zep_paragraph(notebook_id=str(new_notebook_id),
paragraph_id=str(p))
# 5. GET THE SERVICE RESULTS
if livy:
result = get_result_dict_from_livy(livy_session, 'result')
else:
result_paragraph_id = create_zep_getDict_paragraph(
notebook_id=new_notebook_id, title='', dict_name='result')
run_zep_paragraph(notebook_id=new_notebook_id,
paragraph_id=result_paragraph_id,
livy_session_id=0,
mode='zeppelin')
result = get_zep_getDict_paragraph_response(
notebook_id=new_notebook_id, paragraph_id=result_paragraph_id)
delete_zep_paragraph(notebook_id=new_notebook_id,
paragraph_id=result_paragraph_id)
print 'result: ' + str(result)
# result = json.loads(str(result))
# 5. BRING THE CUSTOMISED QUERIES TO THE SERVICE OUTPUT
# print 'Change queries:'
# output_html = service.output_html
# for name, info in queries.iteritems():
# print name, info
# q = info['query_id']
# temp_q = info['temp_q']
# print output_html
# output_html = output_html.replace('query='+str(q), 'query='+str(temp_q))
# print output_html
# output_css = service.output_css
# output_js = service.output_js
# print output_html
# return render(request, 'service_builder/load_service.html', {
# 'output_html': output_html,
# 'output_css': output_css,
# 'output_js': output_js,
# 'arguments': json.dumps(service_args),
# 'service_id': service_id})
# result = dict()
# result['query_mapping'] = query_mapping
# return HttpResponse(json.dumps(result), content_type="application/json")
# template = Template("Hello {{ name }}! ")
# result = dict()
# result['result1'] = "John"
output_html = service.output_html
output_css = service.output_css
output_js = service.output_js
# if settings.TEST_SERVICES:
# import os
# with open(os.path.join(settings.BASE_DIR, 'service_builder\\templates\\service_builder\\service_template_1.html')) as f:
# output_html = f.read()
if queries is not None:
for name, info in queries.iteritems():
query = info['query_id']
new_query = info['temp_q']
# print output_html
output_html = re.sub(r"query=" + str(query) + "&",
"query=" + str(new_query) + "&",
output_html)
output_html = re.sub(r"query=" + str(query) + "\"",
"query=" + str(new_query) + "\"",
output_html)
output_html = output_html.replace(original_notebook_id,
new_notebook_id)
soup = BeautifulSoup(str(output_html), 'html.parser')
service_result_container = soup.find(id="service_result_container")
innerHTML = ''
for c in service_result_container.contents:
innerHTML += str(c)
context = Context({"result": result})
template = Template(innerHTML)
return HttpResponse(template.render(context))
except Exception as e:
print '%s (%s)' % (e.message, type(e))
if livy:
close_livy_session(service_exec.livy_session)
return HttpResponse(status=500)
def execute_query(request, pk=None):
user = request.user
if user.is_authenticated:
doc_str = request.POST.get('query', '')
# get or fake query object
if not pk:
q = AbstractQuery(document=json.loads(doc_str))
else:
q = AbstractQuery.objects.get(pk=pk)
try:
q.document = json.loads(doc_str)
except ValueError:
pass
# print q.document
# print q.raw_query
# get POST params
dimension_values = request.POST.get('dimension_values', '')
variable = request.POST.get('variable', '')
only_headers = request.POST.get('only_headers', '').lower() == 'true'
# check for the access
try:
dataset_list = []
doc = q.document
print q.document
for el in doc['from']:
dataset = Variable.objects.get(id=int(el['type'])).dataset
if dataset.id not in dataset_list:
dataset_list.append(dataset.id)
print dataset_list
for dataset_id in dataset_list:
access_decision = PEP.access_to_dataset_for_query(
request, dataset_id)
if access_decision is False:
raise PermissionDenied
except:
return HttpResponseForbidden()
try:
# execute
result = q.execute(dimension_values=dimension_values,
variable=variable,
only_headers=only_headers,
with_encoder=True)
except ValueError as ve:
return JsonResponse({"error_message": str(ve.message)})
response, encoder = result
# import pdb
# pdb.set_trace()
# send results
for dataset_list_el_id in dataset_list:
try:
dataset_obj = Dataset.objects.get(id=dataset_list_el_id)
dataset_exploration(dataset_obj)
if len(dataset_list) > 1:
dataset_join(dataset_obj)
except:
pass
return JsonResponse(response, encoder=encoder)
else:
return HttpResponseForbidden()
def test_access_to_dashboard(self):
dashboardTitle = "A title 2"
dashboardPrivate = True
tempVis2 = [
"", "1", "1", "1", "1", "temptitle",
"<div>Hello this is a test</div>"
]
payload = {
"title": dashboardTitle,
"private": dashboardPrivate,
"0": tempVis2,
"can_be_shared": True
}
payload = json.dumps(payload)
payloadContainer = {payload: ''}
another_user = User(username='******')
another_user.save()
user = User(username='******')
user.save()
cl = Client()
cl.force_login(user)
url = "http://localhost:8000/dashboards/save/"
pk = cl.post(url, data=payloadContainer)
pk = pk.content
pk = json.loads(pk)
accessible_dashboard_pk = pk["pk"]
den_dashboard = Dashboard(user=another_user,
viz_components={},
private=True,
can_be_shared=True)
den_dashboard.save()
non_accessible_dashboard_pk = den_dashboard.id
from django.http import HttpRequest
request1 = HttpRequest()
request1.user = user
decision = PEP.access_to_dashboard(request1, accessible_dashboard_pk)
self.assertEqual(str(decision), str(True))
decision = PEP.access_to_dashboard(request1,
non_accessible_dashboard_pk)
self.assertEqual(str(decision), str(False))
decision = PEP.access_to_edit_dashboard(request1,
accessible_dashboard_pk)
self.assertEqual(str(decision), str(True))
decision = PEP.access_to_edit_dashboard(request1,
non_accessible_dashboard_pk)
self.assertEqual(str(decision), str(False))
# Log out user. Log in the other user. Create request. Accept request to view dashboard (not to edit). Check
# again.
cl.logout()
cl = Client()
cl.force_login(another_user)
resource = den_dashboard
new_access_request = DashboardAccessRequest(user=user,
resource=resource)
new_access_request.save()
request_id = new_access_request.id
url = "http://localhost:8000/access_control/share_access_to_resource/dashboard/"
payload = {"request_id": request_id}
cl.post(url, data=payload)
decision = PEP.access_to_dashboard(request1,
non_accessible_dashboard_pk)
self.assertEqual(str(decision), str(True))
def check_access(request, service):
access_decision = PEP.access_to_service(request, service.id)
if access_decision is False:
raise PermissionDenied