def update_service_arguments(request): service_id = int(request.POST.get('service_id', '0')) service = Service.objects.get(pk=int(service_id)) arguments = convert_unicode_json( json.loads(str(request.POST.get('exposed_args')))) print arguments for arg in arguments['filter-arguments']: if arg['type'] == "SPATIAL_COV": arg['default_lat_min'] = arg['default'].split('<<')[1].split( ',')[0] arg['default_lon_min'] = arg['default'].split('<<')[1].split( ',')[1].split('>')[0] arg['default_lat_max'] = arg['default'].split('>,<')[1].split( ',')[0] arg['default_lon_max'] = arg['default'].split('>,<')[1].split( ',')[1].split('>')[0] args_to_note = dict() for arg in arguments['algorithm-arguments']: args_to_note[arg['name']] = arg['default'] new_arguments_paragraph = create_zep_arguments_paragraph( notebook_id=service.notebook_id, title='', args_json_string=json.dumps(args_to_note)) run_zep_paragraph(notebook_id=service.notebook_id, paragraph_id=new_arguments_paragraph, livy_session_id=0, mode='zeppelin') if service.arguments_paragraph_id is not None: delete_zep_paragraph(service.notebook_id, service.arguments_paragraph_id) service.arguments_paragraph_id = new_arguments_paragraph service.arguments = arguments service.save() result = {} return HttpResponse(json.dumps(result), content_type="application/json")
def load_query(request): notebook_id = str(request.POST.get('notebook_id')) query_id = int(request.POST.get('query_id')) query_name = str(request.POST.get('query_name')) exposed_args = convert_unicode_json( json.loads(str(request.POST.get('exposed_args')))) print "query name: " + query_name doc = Query.objects.get(pk=query_id).document for arg in exposed_args: filters = convert_unicode_json(doc['filters']) doc['filters'] = update_filter(filters, arg) raw_query = Query(document=doc).raw_query query_paragraph_id = create_zep__query_paragraph(notebook_id, 'query paragraph', raw_query, index=2, df_name="df_" + query_name) run_zep_paragraph(notebook_id, query_paragraph_id, livy_session_id=0, mode='zeppelin') result = { query_name: { "dataframe": "df_" + query_name, "paragraph": query_paragraph_id } } return HttpResponse(json.dumps(result), content_type="application/json")
def load_dataframe(request): notebook_id = str(request.POST.get('notebook_id')) dataframe_name = str(request.POST.get('dataframe_name')) df = SavedDataframes.objects.get(user=request.user, filename=dataframe_name) df_id = df.pk username = df.user.username print "dataframe name: " + dataframe_name dataframe_paragraph_id = create_zep__query_load_dataframe_paragraph( notebook_id, 'dataframe paragraph', username + '/' + dataframe_name + '.parquet', index=0, df_name="df_" + str(df_id), livy=True) run_zep_paragraph(notebook_id, dataframe_paragraph_id, livy_session_id=0, mode='zeppelin') result = { dataframe_name: { "dataframe": "df_" + str(df_id), "paragraph": dataframe_paragraph_id } } return HttpResponse(json.dumps(result), content_type="application/json")
def run_initial_zep_paragraph(request): service_id = request.POST.get('service_id') service = Service.objects.get(pk=service_id) run_zep_paragraph(service.notebook_id, paragraph_id=settings.BASE_NOTE_LOADER_PARAGRAPH, livy_session_id=0, mode='zeppelin') run_zep_paragraph(service.notebook_id, paragraph_id=settings.BASE_NOTE_ARG_PARAGRAPH, livy_session_id=0, mode='zeppelin') return HttpResponse("OK")
def save_dataframe(request): dataframe_name = str(request.POST.get('dataframe_name')) notebook_id = str(request.POST.get('notebook_id')) username = request.user.username df = SavedDataframes(user=request.user, filename=dataframe_name) dataframe_paragraph_id = create_zep__query_save_dataframe_paragraph( notebook_id, 'dataframe save paragraph', username + '/' + dataframe_name + '.parquet', index=0, df_name=dataframe_name, livy=True) result = run_zep_paragraph(notebook_id, dataframe_paragraph_id, livy_session_id=0, mode='zeppelin') delete_zep_paragraph(notebook_id, dataframe_paragraph_id) if result == 0: df.save() result = { dataframe_name: { "dataframe": "df_" + dataframe_name, "paragraph": dataframe_paragraph_id } } return HttpResponse(json.dumps(result), content_type="application/json") else: return HttpResponse(status=500)
def submit_service_args(request, service_id): service = Service.objects.get(pk=int(service_id)) if service.title == 'Oil Spill Simulation Service': output_html = service.output_html soup = BeautifulSoup(str(output_html), 'html.parser') service_result_container = soup.find(id="service_result_container") innerHTML = '' for c in service_result_container.contents: innerHTML += str(c) context = Context({"result": ''}) template = Template(innerHTML) return HttpResponse(template.render(context)) livy = service.through_livy service_exec = ServiceInstance(service=service, user=request.user, time=datetime.now()) service_exec.save() # 1.GATHER THE SERVICE ARGUMENTS service_args = convert_unicode_json(service.arguments) service_filter_args = service_args["filter-arguments"] print 'original args:' print service_filter_args for filter_arg in service_filter_args: filter_arg['filter_b'] = request.GET.get(filter_arg['name'], filter_arg['default']) print 'user args:' print service_filter_args service_algorithm_args = service_args["algorithm-arguments"] print 'original algorithm args:' print service_algorithm_args args_to_note = dict() for algorithm_arg in service_algorithm_args: args_to_note[algorithm_arg['name']] = request.GET.get( algorithm_arg['name'], algorithm_arg['default']) print 'user algorithm args:' print args_to_note service_exec.arguments = { 'filter-arguments': service_filter_args, 'algorithm-arguments': service_algorithm_args } service_exec.save() # 2.CUSTOMIZE THE QUERIES BASED ON THE GATHERED USER ARGUMENTS (AND CREATE TEMPQUERIES) # query_mapping is a dict that maps the original queries of the template service to the tempQueries created after the user customisation query_mapping = dict() queries = convert_unicode_json(service.queries) if queries is not None: for name, q in queries.iteritems(): query_id = int(q['query_id']) query = Query.objects.get(pk=query_id) doc = query.document for arg in service_filter_args: arg_query_id = int(queries[arg['query']]['query_id']) if arg_query_id == query_id: filters = convert_unicode_json(doc['filters']) doc['filters'] = update_filter(filters, arg) q_temp = TempQuery(original=query, document=doc, user_id=query.user_id) q_temp.save() q['temp_q'] = q_temp.id query_mapping[query_id] = q_temp.id print queries # 3.BRING THE CUSTOMISED QUERIES TO THE SERVICE CODE original_notebook_id = service.notebook_id if settings.TEST_SERVICES: new_notebook_id = original_notebook_id excluded_paragraphs = [] new_created_paragraphs = [] else: excluded_paragraphs = [] new_notebook_id = clone_zep_note(original_notebook_id, "") clean_up_new_note(new_notebook_id) service_exec.notebook_id = new_notebook_id service_exec.save() print 'Notebook ID: {0}'.format(new_notebook_id) # customise the respective queries in the code if queries is not None: for name, info in queries.iteritems(): for original_paragraph_id in info['paragraphs']: raw_query = TempQuery.objects.get( pk=int(info['temp_q'])).raw_query new_query_paragraph_id = create_zep__query_paragraph( new_notebook_id, '', raw_query, index=2, df_name="df_" + name) if settings.TEST_SERVICES: excluded_paragraphs.append(original_paragraph_id) new_created_paragraphs.append(new_query_paragraph_id) else: print 'deleting paragraph: {0}'.format( original_paragraph_id) delete_zep_paragraph( notebook_id=str(new_notebook_id), paragraph_id=str(original_paragraph_id)) new_arguments_paragraph = create_zep_arguments_paragraph( notebook_id=new_notebook_id, title='', args_json_string=json.dumps(args_to_note)) if service.arguments_paragraph_id is not None: delete_zep_paragraph(new_notebook_id, service.arguments_paragraph_id) if settings.TEST_SERVICES: service.arguments_paragraph_id = new_arguments_paragraph service.save() output_html = service.output_html output_html = output_html.replace(original_notebook_id, new_notebook_id) soup = BeautifulSoup(output_html, 'html.parser') visualizations = [] for f in soup.findAll('iframe'): visualizations.append(f.get("src")) import urlparse dataframe_viz = [] for url in visualizations: parsed = urlparse.urlparse(url) if 'notebook_id' in urlparse.parse_qs(parsed.query).keys(): if 'df' in urlparse.parse_qs(parsed.query).keys(): df = urlparse.parse_qs(parsed.query)['df'][0] elif 'df1' in urlparse.parse_qs(parsed.query).keys(): df = urlparse.parse_qs(parsed.query)['df1'][0] elif 'df2' in urlparse.parse_qs(parsed.query).keys(): df = urlparse.parse_qs(parsed.query)['df2'][0] dataframe_viz.append({ 'notebook_id': urlparse.parse_qs(parsed.query)['notebook_id'][0], 'df': df, 'url': url, 'done': False }) service_exec.dataframe_visualizations = dataframe_viz service_exec.save() # 4.RUN THE SERVICE CODE (one by one paragraph, or all together. CHOOSE..) try: if livy: livy_session = run_zep_note(notebook_id=new_notebook_id, exclude=excluded_paragraphs, mode='livy') service_exec.livy_session = livy_session service_exec.save() else: run_zep_note(notebook_id=new_notebook_id, exclude=excluded_paragraphs, mode='zeppelin') # data = create_livy_toJSON_paragraph() # with open('df_json_{0}___{1}.json'.format(new_notebook_id, df_name), 'w') as outfile: # json.dump(data, outfile) if settings.TEST_SERVICES: for p in new_created_paragraphs: print 'deleting paragraph: {0}'.format(p) delete_zep_paragraph(notebook_id=str(new_notebook_id), paragraph_id=str(p)) # 5. GET THE SERVICE RESULTS if livy: result = get_result_dict_from_livy(livy_session, 'result') else: result_paragraph_id = create_zep_getDict_paragraph( notebook_id=new_notebook_id, title='', dict_name='result') run_zep_paragraph(notebook_id=new_notebook_id, paragraph_id=result_paragraph_id, livy_session_id=0, mode='zeppelin') result = get_zep_getDict_paragraph_response( notebook_id=new_notebook_id, paragraph_id=result_paragraph_id) delete_zep_paragraph(notebook_id=new_notebook_id, paragraph_id=result_paragraph_id) print 'result: ' + str(result) # result = json.loads(str(result)) # 5. BRING THE CUSTOMISED QUERIES TO THE SERVICE OUTPUT # print 'Change queries:' # output_html = service.output_html # for name, info in queries.iteritems(): # print name, info # q = info['query_id'] # temp_q = info['temp_q'] # print output_html # output_html = output_html.replace('query='+str(q), 'query='+str(temp_q)) # print output_html # output_css = service.output_css # output_js = service.output_js # print output_html # return render(request, 'service_builder/load_service.html', { # 'output_html': output_html, # 'output_css': output_css, # 'output_js': output_js, # 'arguments': json.dumps(service_args), # 'service_id': service_id}) # result = dict() # result['query_mapping'] = query_mapping # return HttpResponse(json.dumps(result), content_type="application/json") # template = Template("Hello {{ name }}! ") # result = dict() # result['result1'] = "John" output_html = service.output_html output_css = service.output_css output_js = service.output_js # if settings.TEST_SERVICES: # import os # with open(os.path.join(settings.BASE_DIR, 'service_builder\\templates\\service_builder\\service_template_1.html')) as f: # output_html = f.read() if queries is not None: for name, info in queries.iteritems(): query = info['query_id'] new_query = info['temp_q'] # print output_html output_html = re.sub(r"query=" + str(query) + "&", "query=" + str(new_query) + "&", output_html) output_html = re.sub(r"query=" + str(query) + "\"", "query=" + str(new_query) + "\"", output_html) output_html = output_html.replace(original_notebook_id, new_notebook_id) soup = BeautifulSoup(str(output_html), 'html.parser') service_result_container = soup.find(id="service_result_container") innerHTML = '' for c in service_result_container.contents: innerHTML += str(c) context = Context({"result": result}) template = Template(innerHTML) return HttpResponse(template.render(context)) except Exception as e: print '%s (%s)' % (e.message, type(e)) if livy: close_livy_session(service_exec.livy_session) return HttpResponse(status=500)