def do_clone_workflow(user, workflow: models.Workflow) -> models.Workflow:
"""Clone a workflow.
:param user: User performing the operation
:param workflow: source workflow
:return: Cloned object
"""
new_workflow = models.Workflow(
user=workflow.user,
name=create_new_name(
workflow.name,
models.Workflow.objects.filter(
Q(user=workflow.user) | Q(shared=workflow.user)),
),
description_text=workflow.description_text,
nrows=workflow.nrows,
ncols=workflow.ncols,
attributes=copy.deepcopy(workflow.attributes),
query_builder_ops=copy.deepcopy(workflow.query_builder_ops),
luser_email_column_md5=workflow.luser_email_column_md5,
lusers_is_outdated=workflow.lusers_is_outdated)
new_workflow.save()
try:
new_workflow.shared.set(list(workflow.shared.all()))
new_workflow.lusers.set(list(workflow.lusers.all()))
# Clone the columns
for item_obj in workflow.columns.all():
do_clone_column_only(item_obj, new_workflow=new_workflow)
# Update the luser_email_column if needed:
if workflow.luser_email_column:
new_workflow.luser_email_column = new_workflow.columns.get(
name=workflow.luser_email_column.name, )
# Clone the DB table
sql.clone_table(workflow.get_data_frame_table_name(),
new_workflow.get_data_frame_table_name())
# Clone actions
for item_obj in workflow.actions.all():
do_clone_action(user, item_obj, new_workflow)
for item_obj in workflow.views.all():
do_clone_view(user, item_obj, new_workflow)
# Done!
new_workflow.save()
except Exception as exc:
raise OnTaskServiceException(
message=_('Error while cloning workflow: {0}').format(exc),
to_delete=[new_workflow])
workflow.log(user,
models.Log.WORKFLOW_CLONE,
id_old=workflow.id,
name_old=workflow.name)
return new_workflow
def upload_step_two(
request: http.HttpRequest,
workflow: models.Workflow,
select_column_data: Dict,
upload_data: Dict,
) -> http.HttpResponse:
"""Process the received dataframe and either store or continue merge.
:param request: Http request received.
:param workflow: workflow being processed.
:param select_column_data: Dictionary with the upload/merge information
(new column names, keep the keys)
:param upload_data: Dictionary with the upload information.
:return: Http Response
"""
# We need to modify upload_data with the information received in the
# post
initial_columns = upload_data.get('initial_column_names')
src_is_key_column = upload_data.get('src_is_key_column')
keep_key_column = upload_data.get('keep_key_column')
for idx in range(len(initial_columns)):
new_name = select_column_data['new_name_%s' % idx]
upload_data['rename_column_names'][idx] = new_name
upload = select_column_data['upload_%s' % idx]
upload_data['columns_to_upload'][idx] = upload
if src_is_key_column[idx]:
# If the column is key, check if the user wants to keep it
keep_key_column[idx] = select_column_data['make_key_%s' % idx]
if workflow.has_data_frame():
# A Merge operation is required
# Update the dictionary with the session information so that it is
# available in the next step
request.session['upload_data'] = upload_data
# This is a merge operation, so move to Step 3
return redirect('dataops:upload_s3')
# This is the first data to be stored in the workflow. Save the uploaded
# dataframe in the DB and finish.
pandas.store_workflow_table(workflow, upload_data)
# Update the session information
store_workflow_in_session(request.session, workflow)
col_info = workflow.get_column_info()
workflow.log(request.user,
upload_data['log_upload'],
column_names=col_info[0],
column_types=col_info[1],
column_unique=col_info[2])
return redirect(reverse('table:display'))
def do_flush(request: http.HttpRequest, workflow: models.Workflow):
"""Perform the workflow data flush.
:param request: Request received
:param workflow: Workflow being manipulated.
:return: Side effect, data is removed.
"""
# Delete the table
workflow.flush()
workflow.refresh_from_db()
# update the request object with the new number of rows
core.store_workflow_in_session(request.session, workflow)
workflow.log(request.user, models.Log.WORKFLOW_DATA_FLUSH)
def save_attribute_form(request: HttpRequest, workflow: Workflow,
template: str, form: forms.Form,
attr_idx: int) -> JsonResponse:
"""Process the AJAX request to create or update an attribute.
:param request: Request object received
:param workflow: current workflow being manipulated
:param template: Template to render in the response
:param form: Form used to ask for data
:param attr_idx: Index of the attribute being manipulated
:return: AJAX reponse
"""
if request.method == 'POST' and form.is_valid():
# Correct form submitted
if not form.has_changed():
return JsonResponse({'html_redirect': None})
# proceed with updating the attributes.
wf_attributes = workflow.attributes
# If key_idx is not -1, this means we are editing an existing pair
if attr_idx != -1:
key = sorted(wf_attributes.keys())[attr_idx]
wf_attributes.pop(key)
# Rename the appearances of the variable in all actions
for action_item in workflow.actions.all():
action_item.rename_variable(key, form.cleaned_data['key'])
# Update value
wf_attributes[
form.cleaned_data['key']] = form.cleaned_data['attr_value']
workflow.attributes = wf_attributes
workflow.save()
workflow.log(request.user, Log.WORKFLOW_ATTRIBUTE_CREATE,
**wf_attributes)
return JsonResponse({'html_redirect': ''})
return JsonResponse({
'html_form':
render_to_string(template, {
'form': form,
'id': attr_idx
},
request=request),
})
def upload_step_four(
request: http.HttpRequest,
workflow: models.Workflow,
upload_data: Dict,
) -> http.HttpResponse:
"""Perform the merge operation.
:param request: Received request
:param workflow: Workflow being processed
:param upload_data: Dictionary with all the information about the merge.
:return: HttpResponse
"""
# Get the dataframes to merge
try:
dst_df = pandas.load_table(workflow.get_data_frame_table_name())
src_df = pandas.load_table(workflow.get_upload_table_name())
except Exception:
return render(request, 'error.html',
{'message': _('Exception while loading data frame')})
try:
pandas.perform_dataframe_upload_merge(workflow, dst_df, src_df,
upload_data)
except Exception as exc:
# Nuke the temporary table
sql.delete_table(workflow.get_upload_table_name())
col_info = workflow.get_column_info()
workflow.log(request.user,
models.Log.WORKFLOW_DATA_FAILEDMERGE,
column_names=col_info[0],
column_types=col_info[1],
column_unique=col_info[2],
error_message=str(exc))
messages.error(request, _('Merge operation failed. ') + str(exc))
return redirect(reverse('table:display'))
col_info = workflow.get_column_info()
workflow.log(request.user,
upload_data['log_upload'],
column_names=col_info[0],
column_types=col_info[1],
column_unique=col_info[2])
store_workflow_in_session(request.session, workflow)
request.session.pop('upload_data', None)
return redirect(reverse('table:display'))
def update_luser_email_column(
user,
pk: int,
workflow: models.Workflow,
column: models.Column,
):
"""Update the field luser_email in the workflow.
:param user: User making the request
:param pk: Column ID to obtain the user id
:param workflow: Workflow being manipulated.
:param column: Column being used to update the luser field.
:return:
"""
if not pk:
# Empty pk, means reset the field.
workflow.luser_email_column = None
workflow.luser_email_column_md5 = ''
workflow.lusers.set([])
workflow.save(
update_fields=['luser_email_column', 'luser_email_column_md5'])
return
table_name = workflow.get_data_frame_table_name()
# Get the column content
emails = sql.get_rows(table_name, column_names=[column.name])
# Verify that the column as a valid set of emails
incorrect_email = get_incorrect_email([row[column.name] for row in emails])
if incorrect_email:
raise services.OnTaskWorkflowEmailError(message=_(
'Incorrect email addresses "{0}".').format(incorrect_email))
# Update the column
workflow.luser_email_column = column
workflow.save(update_fields=['luser_email_column'])
# Calculate the MD5 value
md5_hash = sql.get_text_column_hash(table_name, column.name)
if workflow.luser_email_column_md5 == md5_hash:
return
# Change detected, run the update in batch mode
workflow.luser_email_column_md5 = md5_hash
workflow.save(update_fields=['luser_email_column_md5'])
# Log the event with the status "preparing updating"
log_item = workflow.log(user, models.Log.WORKFLOW_UPDATE_LUSERS)
# Push the update of lusers to batch processing
tasks.execute_operation.delay(
operation_type=models.Log.WORKFLOW_UPDATE_LUSERS,
user_id=user.id,
workflow_id=workflow.id,
log_id=log_item.id)