def add_pyspark(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = AddPySparkForm(request.POST, flow=get_flow())
if form.is_valid():
step_name = form.cleaned_data['step_name']
step_description = form.cleaned_data['step_description']
depends_ons = form.cleaned_data['depends_ons']
pyspark_file_path = form.cleaned_data['pyspark_file_path']
create_file(pyspark_file_path, f'return {depends_ons[0]}_df')
updated_flow = api.add_pyspark_step(
flow=get_flow(),
name=step_name,
depends_on=depends_ons,
description=step_description,
pyspark_file_path=pyspark_file_path)
set_flow(updated_flow)
return HttpResponseRedirect('/dag/view')
# if a GET (or any other method) we'll create a blank form
else:
form = AddPySparkForm(flow=get_flow())
return render(request, 'dag/add_pyspark.html', {'form': form})
def add_sql(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = AddSqlForm(request.POST, flow=get_flow())
if form.is_valid():
step_name = form.cleaned_data['step_name']
step_description = form.cleaned_data['step_description']
depends_ons = form.cleaned_data['depends_ons']
sql_table_names = ', '.join(
[f'{table} {table}' for table in depends_ons])
sql_select = ', '.join([f'{table}.*' for table in depends_ons])
sql_query = f'select {sql_select} from {sql_table_names}'
sql_file_path = form.cleaned_data['sql_file_path']
create_file(sql_file_path, sql_query)
updated_flow = api.add_sql_step(flow=get_flow(),
name=step_name,
depends_on=depends_ons,
description=step_description,
sql_file_path=sql_file_path)
set_flow(updated_flow)
return HttpResponseRedirect('/dag/view')
# if a GET (or any other method) we'll create a blank form
else:
form = AddSqlForm(flow=get_flow())
return render(request, 'dag/add_sql.html', {'form': form})
def add_sink(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = AddSinkForm(request.POST, flow=get_flow())
if form.is_valid():
step_name = form.cleaned_data['step_name']
step_description = form.cleaned_data['step_description']
depends_ons = form.cleaned_data['depends_ons']
uri = form.cleaned_data['uri']
format_type = form.cleaned_data['format_type']
mode = form.cleaned_data['mode']
options = parse_options(form.cleaned_data['options_text'])
updated_flow = api.add_sink_step(flow=get_flow(),
name=step_name,
depends_on=[depends_ons],
description=step_description,
uri=uri,
format_type=format_type,
mode=mode,
options=options)
set_flow(updated_flow)
return HttpResponseRedirect('/dag/view')
# if a GET (or any other method) we'll create a blank form
else:
form = AddSinkForm(flow=get_flow())
return render(request, 'dag/add_sink.html', {'form': form})
def delete_step(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = SelectedStepForm(request.POST)
if form.is_valid():
step_name = form.cleaned_data['step_name']
updated_flow = api.remove_step(flow=get_flow(), name=step_name)
set_flow(updated_flow)
return HttpResponseRedirect('/dag/view')
else:
return HttpResponseRedirect('/dag/view')
def open(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = ProjectOpenForm(request.POST)
# check whether it's valid:
if form.is_valid():
flow = cli.open_flow(dir=form.cleaned_data['project_location'],
env_name='local')
set_flow(flow)
return HttpResponseRedirect('/dag/view')
# if a GET (or any other method) we'll create a blank form
else:
form = ProjectOpenForm()
return render(request, 'project/open.html', {'form': form})
def add_source(request):
# if this is a POST request we need to process the form data
if request.method == 'POST':
# create a form instance and populate it with data from the request:
form = AddSourceForm(request.POST)
# check whether it's valid:
if form.is_valid():
options_dict = parse_options(form.cleaned_data['options_text'])
updated_flow = api.add_source_step(
flow=get_flow(),
name=form.cleaned_data['step_name'],
description=form.cleaned_data['step_description'],
uri=form.cleaned_data['uri'],
format_type=form.cleaned_data['format_type'],
options=options_dict)
set_flow(updated_flow)
return HttpResponseRedirect('/dag/view')
# if a GET (or any other method) we'll create a blank form
else:
form = AddSourceForm()
return render(request, 'dag/add_source.html', {'form': form})