def update_service_arguments(request):
service_id = int(request.POST.get('service_id', '0'))
service = Service.objects.get(pk=int(service_id))
arguments = convert_unicode_json(
json.loads(str(request.POST.get('exposed_args'))))
print arguments
for arg in arguments['filter-arguments']:
if arg['type'] == "SPATIAL_COV":
arg['default_lat_min'] = arg['default'].split('<<')[1].split(
',')[0]
arg['default_lon_min'] = arg['default'].split('<<')[1].split(
',')[1].split('>')[0]
arg['default_lat_max'] = arg['default'].split('>,<')[1].split(
',')[0]
arg['default_lon_max'] = arg['default'].split('>,<')[1].split(
',')[1].split('>')[0]
args_to_note = dict()
for arg in arguments['algorithm-arguments']:
args_to_note[arg['name']] = arg['default']
new_arguments_paragraph = create_zep_arguments_paragraph(
notebook_id=service.notebook_id,
title='',
args_json_string=json.dumps(args_to_note))
run_zep_paragraph(notebook_id=service.notebook_id,
paragraph_id=new_arguments_paragraph,
livy_session_id=0,
mode='zeppelin')
if service.arguments_paragraph_id is not None:
delete_zep_paragraph(service.notebook_id,
service.arguments_paragraph_id)
service.arguments_paragraph_id = new_arguments_paragraph
service.arguments = arguments
service.save()
result = {}
return HttpResponse(json.dumps(result), content_type="application/json")
def load_query(request):
notebook_id = str(request.POST.get('notebook_id'))
query_id = int(request.POST.get('query_id'))
query_name = str(request.POST.get('query_name'))
exposed_args = convert_unicode_json(
json.loads(str(request.POST.get('exposed_args'))))
print "query name: " + query_name
doc = Query.objects.get(pk=query_id).document
for arg in exposed_args:
filters = convert_unicode_json(doc['filters'])
doc['filters'] = update_filter(filters, arg)
raw_query = Query(document=doc).raw_query
query_paragraph_id = create_zep__query_paragraph(notebook_id,
'query paragraph',
raw_query,
index=2,
df_name="df_" +
query_name)
run_zep_paragraph(notebook_id,
query_paragraph_id,
livy_session_id=0,
mode='zeppelin')
result = {
query_name: {
"dataframe": "df_" + query_name,
"paragraph": query_paragraph_id
}
}
return HttpResponse(json.dumps(result), content_type="application/json")
def load_dataframe(request):
notebook_id = str(request.POST.get('notebook_id'))
dataframe_name = str(request.POST.get('dataframe_name'))
df = SavedDataframes.objects.get(user=request.user,
filename=dataframe_name)
df_id = df.pk
username = df.user.username
print "dataframe name: " + dataframe_name
dataframe_paragraph_id = create_zep__query_load_dataframe_paragraph(
notebook_id,
'dataframe paragraph',
username + '/' + dataframe_name + '.parquet',
index=0,
df_name="df_" + str(df_id),
livy=True)
run_zep_paragraph(notebook_id,
dataframe_paragraph_id,
livy_session_id=0,
mode='zeppelin')
result = {
dataframe_name: {
"dataframe": "df_" + str(df_id),
"paragraph": dataframe_paragraph_id
}
}
return HttpResponse(json.dumps(result), content_type="application/json")
def run_initial_zep_paragraph(request):
service_id = request.POST.get('service_id')
service = Service.objects.get(pk=service_id)
run_zep_paragraph(service.notebook_id,
paragraph_id=settings.BASE_NOTE_LOADER_PARAGRAPH,
livy_session_id=0,
mode='zeppelin')
run_zep_paragraph(service.notebook_id,
paragraph_id=settings.BASE_NOTE_ARG_PARAGRAPH,
livy_session_id=0,
mode='zeppelin')
return HttpResponse("OK")
def save_dataframe(request):
dataframe_name = str(request.POST.get('dataframe_name'))
notebook_id = str(request.POST.get('notebook_id'))
username = request.user.username
df = SavedDataframes(user=request.user, filename=dataframe_name)
dataframe_paragraph_id = create_zep__query_save_dataframe_paragraph(
notebook_id,
'dataframe save paragraph',
username + '/' + dataframe_name + '.parquet',
index=0,
df_name=dataframe_name,
livy=True)
result = run_zep_paragraph(notebook_id,
dataframe_paragraph_id,
livy_session_id=0,
mode='zeppelin')
delete_zep_paragraph(notebook_id, dataframe_paragraph_id)
if result == 0:
df.save()
result = {
dataframe_name: {
"dataframe": "df_" + dataframe_name,
"paragraph": dataframe_paragraph_id
}
}
return HttpResponse(json.dumps(result),
content_type="application/json")
else:
return HttpResponse(status=500)
def submit_service_args(request, service_id):
service = Service.objects.get(pk=int(service_id))
if service.title == 'Oil Spill Simulation Service':
output_html = service.output_html
soup = BeautifulSoup(str(output_html), 'html.parser')
service_result_container = soup.find(id="service_result_container")
innerHTML = ''
for c in service_result_container.contents:
innerHTML += str(c)
context = Context({"result": ''})
template = Template(innerHTML)
return HttpResponse(template.render(context))
livy = service.through_livy
service_exec = ServiceInstance(service=service,
user=request.user,
time=datetime.now())
service_exec.save()
# 1.GATHER THE SERVICE ARGUMENTS
service_args = convert_unicode_json(service.arguments)
service_filter_args = service_args["filter-arguments"]
print 'original args:'
print service_filter_args
for filter_arg in service_filter_args:
filter_arg['filter_b'] = request.GET.get(filter_arg['name'],
filter_arg['default'])
print 'user args:'
print service_filter_args
service_algorithm_args = service_args["algorithm-arguments"]
print 'original algorithm args:'
print service_algorithm_args
args_to_note = dict()
for algorithm_arg in service_algorithm_args:
args_to_note[algorithm_arg['name']] = request.GET.get(
algorithm_arg['name'], algorithm_arg['default'])
print 'user algorithm args:'
print args_to_note
service_exec.arguments = {
'filter-arguments': service_filter_args,
'algorithm-arguments': service_algorithm_args
}
service_exec.save()
# 2.CUSTOMIZE THE QUERIES BASED ON THE GATHERED USER ARGUMENTS (AND CREATE TEMPQUERIES)
# query_mapping is a dict that maps the original queries of the template service to the tempQueries created after the user customisation
query_mapping = dict()
queries = convert_unicode_json(service.queries)
if queries is not None:
for name, q in queries.iteritems():
query_id = int(q['query_id'])
query = Query.objects.get(pk=query_id)
doc = query.document
for arg in service_filter_args:
arg_query_id = int(queries[arg['query']]['query_id'])
if arg_query_id == query_id:
filters = convert_unicode_json(doc['filters'])
doc['filters'] = update_filter(filters, arg)
q_temp = TempQuery(original=query,
document=doc,
user_id=query.user_id)
q_temp.save()
q['temp_q'] = q_temp.id
query_mapping[query_id] = q_temp.id
print queries
# 3.BRING THE CUSTOMISED QUERIES TO THE SERVICE CODE
original_notebook_id = service.notebook_id
if settings.TEST_SERVICES:
new_notebook_id = original_notebook_id
excluded_paragraphs = []
new_created_paragraphs = []
else:
excluded_paragraphs = []
new_notebook_id = clone_zep_note(original_notebook_id, "")
clean_up_new_note(new_notebook_id)
service_exec.notebook_id = new_notebook_id
service_exec.save()
print 'Notebook ID: {0}'.format(new_notebook_id)
# customise the respective queries in the code
if queries is not None:
for name, info in queries.iteritems():
for original_paragraph_id in info['paragraphs']:
raw_query = TempQuery.objects.get(
pk=int(info['temp_q'])).raw_query
new_query_paragraph_id = create_zep__query_paragraph(
new_notebook_id,
'',
raw_query,
index=2,
df_name="df_" + name)
if settings.TEST_SERVICES:
excluded_paragraphs.append(original_paragraph_id)
new_created_paragraphs.append(new_query_paragraph_id)
else:
print 'deleting paragraph: {0}'.format(
original_paragraph_id)
delete_zep_paragraph(
notebook_id=str(new_notebook_id),
paragraph_id=str(original_paragraph_id))
new_arguments_paragraph = create_zep_arguments_paragraph(
notebook_id=new_notebook_id,
title='',
args_json_string=json.dumps(args_to_note))
if service.arguments_paragraph_id is not None:
delete_zep_paragraph(new_notebook_id, service.arguments_paragraph_id)
if settings.TEST_SERVICES:
service.arguments_paragraph_id = new_arguments_paragraph
service.save()
output_html = service.output_html
output_html = output_html.replace(original_notebook_id, new_notebook_id)
soup = BeautifulSoup(output_html, 'html.parser')
visualizations = []
for f in soup.findAll('iframe'):
visualizations.append(f.get("src"))
import urlparse
dataframe_viz = []
for url in visualizations:
parsed = urlparse.urlparse(url)
if 'notebook_id' in urlparse.parse_qs(parsed.query).keys():
if 'df' in urlparse.parse_qs(parsed.query).keys():
df = urlparse.parse_qs(parsed.query)['df'][0]
elif 'df1' in urlparse.parse_qs(parsed.query).keys():
df = urlparse.parse_qs(parsed.query)['df1'][0]
elif 'df2' in urlparse.parse_qs(parsed.query).keys():
df = urlparse.parse_qs(parsed.query)['df2'][0]
dataframe_viz.append({
'notebook_id':
urlparse.parse_qs(parsed.query)['notebook_id'][0],
'df':
df,
'url':
url,
'done':
False
})
service_exec.dataframe_visualizations = dataframe_viz
service_exec.save()
# 4.RUN THE SERVICE CODE (one by one paragraph, or all together. CHOOSE..)
try:
if livy:
livy_session = run_zep_note(notebook_id=new_notebook_id,
exclude=excluded_paragraphs,
mode='livy')
service_exec.livy_session = livy_session
service_exec.save()
else:
run_zep_note(notebook_id=new_notebook_id,
exclude=excluded_paragraphs,
mode='zeppelin')
# data = create_livy_toJSON_paragraph()
# with open('df_json_{0}___{1}.json'.format(new_notebook_id, df_name), 'w') as outfile:
# json.dump(data, outfile)
if settings.TEST_SERVICES:
for p in new_created_paragraphs:
print 'deleting paragraph: {0}'.format(p)
delete_zep_paragraph(notebook_id=str(new_notebook_id),
paragraph_id=str(p))
# 5. GET THE SERVICE RESULTS
if livy:
result = get_result_dict_from_livy(livy_session, 'result')
else:
result_paragraph_id = create_zep_getDict_paragraph(
notebook_id=new_notebook_id, title='', dict_name='result')
run_zep_paragraph(notebook_id=new_notebook_id,
paragraph_id=result_paragraph_id,
livy_session_id=0,
mode='zeppelin')
result = get_zep_getDict_paragraph_response(
notebook_id=new_notebook_id, paragraph_id=result_paragraph_id)
delete_zep_paragraph(notebook_id=new_notebook_id,
paragraph_id=result_paragraph_id)
print 'result: ' + str(result)
# result = json.loads(str(result))
# 5. BRING THE CUSTOMISED QUERIES TO THE SERVICE OUTPUT
# print 'Change queries:'
# output_html = service.output_html
# for name, info in queries.iteritems():
# print name, info
# q = info['query_id']
# temp_q = info['temp_q']
# print output_html
# output_html = output_html.replace('query='+str(q), 'query='+str(temp_q))
# print output_html
# output_css = service.output_css
# output_js = service.output_js
# print output_html
# return render(request, 'service_builder/load_service.html', {
# 'output_html': output_html,
# 'output_css': output_css,
# 'output_js': output_js,
# 'arguments': json.dumps(service_args),
# 'service_id': service_id})
# result = dict()
# result['query_mapping'] = query_mapping
# return HttpResponse(json.dumps(result), content_type="application/json")
# template = Template("Hello {{ name }}! ")
# result = dict()
# result['result1'] = "John"
output_html = service.output_html
output_css = service.output_css
output_js = service.output_js
# if settings.TEST_SERVICES:
# import os
# with open(os.path.join(settings.BASE_DIR, 'service_builder\\templates\\service_builder\\service_template_1.html')) as f:
# output_html = f.read()
if queries is not None:
for name, info in queries.iteritems():
query = info['query_id']
new_query = info['temp_q']
# print output_html
output_html = re.sub(r"query=" + str(query) + "&",
"query=" + str(new_query) + "&",
output_html)
output_html = re.sub(r"query=" + str(query) + "\"",
"query=" + str(new_query) + "\"",
output_html)
output_html = output_html.replace(original_notebook_id,
new_notebook_id)
soup = BeautifulSoup(str(output_html), 'html.parser')
service_result_container = soup.find(id="service_result_container")
innerHTML = ''
for c in service_result_container.contents:
innerHTML += str(c)
context = Context({"result": result})
template = Template(innerHTML)
return HttpResponse(template.render(context))
except Exception as e:
print '%s (%s)' % (e.message, type(e))
if livy:
close_livy_session(service_exec.livy_session)
return HttpResponse(status=500)
