def clone_column(column, new_workflow=None, new_name=None):
"""
Function that given a column clones it and changes workflow and name
:param column: Object to clone
:param new_workflow: New workflow object to point
:param new_name: New name
:return: Cloned object
"""
# Store the old object name before squashing it
old_id = column.id
old_name = column.name
# Clone
column.id = None
# Update some of the fields
if new_name:
column.name = new_name
if new_workflow:
column.workflow = new_workflow
# Update
column.save()
# Add the column to the table and update it.
data_frame = pandas_db.load_from_db(column.workflow.id)
data_frame[new_name] = data_frame[old_name]
ops.store_dataframe_in_db(data_frame, column.workflow.id)
return column
def reposition_column_and_update_df(workflow, column, to_idx):
"""
:param workflow: Workflow object for which the repositioning is done
:param column: column object to relocate
:param to_idx: Destination index of the given column
:return: Content reflected in the DB
"""
df = pandas_db.load_from_db(workflow.id)
reposition_columns(workflow, column.position, to_idx)
column.position = to_idx
column.save()
ops.store_dataframe_in_db(df, workflow.id)
def clone_column(column, new_workflow=None, new_name=None):
"""
Function that given a column clones it and changes workflow and name
:param column: Object to clone
:param new_workflow: New workflow object to point
:param new_name: New name
:return: Cloned object
"""
# Store the old object name before squashing it
old_name = column.name
old_position = column.position
# Clone
column.id = None
# Update some of the fields
if new_name:
column.name = new_name
if new_workflow:
column.workflow = new_workflow
# Set column at the end
column.position = column.workflow.ncols + 1
column.save()
# Update the number of columns in the workflow
column.workflow.ncols += 1
column.workflow.save()
# Reposition the columns above the one being deleted
reposition_columns(column.workflow, column.position, old_position + 1)
# Set the new column in the right location
column.position = old_position + 1
column.save()
# Add the column to the table and update it.
data_frame = pandas_db.load_from_db(column.workflow.id)
data_frame[new_name] = data_frame[old_name]
ops.store_dataframe_in_db(data_frame, column.workflow.id)
return column
def row_create(request):
"""
Receives a POST request to create a new row in the data table
:param request: Request object with all the data.
:return:
"""
# If there is no workflow object, go back to the index
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
# If the workflow has no data, the operation should not be allowed
if workflow.nrows == 0:
return redirect('dataops:list')
# Create the form
form = RowForm(request.POST or None, workflow=workflow)
if request.method == 'GET' or not form.is_valid():
return render(
request, 'dataops/row_create.html', {
'workflow': workflow,
'form': form,
'cancel_url': reverse('table:display')
})
# Create the query to update the row
columns = workflow.get_columns()
column_names = [c.name for c in columns]
field_name = field_prefix + '%s'
row_vals = [
form.cleaned_data[field_name % idx] for idx in range(len(columns))
]
# Load the existing df from the db
df = pandas_db.load_from_db(workflow.id)
# Perform the row addition in the DF first
# df2 = pd.DataFrame([[5, 6], [7, 8]], columns=list('AB'))
# df.append(df2, ignore_index=True)
new_row = pd.DataFrame([row_vals], columns=column_names)
df = df.append(new_row, ignore_index=True)
# Verify that the unique columns remain unique
for ucol in [c for c in columns if c.is_key]:
if not ops.is_unique_column(df[ucol.name]):
form.add_error(
None, 'Repeated value in column ' + ucol.name + '.' +
' It must be different to maintain Key property')
return render(
request, 'dataops/row_create.html', {
'workflow': workflow,
'form': form,
'cancel_url': reverse('table:display')
})
# Restore the dataframe to the DB
ops.store_dataframe_in_db(df, workflow.id)
# Log the event
log_payload = zip(column_names, [str(x) for x in row_vals])
logs.ops.put(request.user, 'tablerow_create', workflow, {
'id': workflow.id,
'name': workflow.name,
'new_values': log_payload
})
# Done. Back to the table view
return redirect('table:display')
def trck(request):
"""
Receive a request with a token from email read tracking
:param request: Request object
:return: Reflects in the DB the reception and (optionally) in the data
table of the workflow
"""
if request.method != 'GET':
raise Http404
# Detected attempt to track event
track_id = request.GET.get('v', None)
if not track_id:
raise Http404
# If the track_id is not correctly signed, out.
try:
track_id = signing.loads(track_id)
except signing.BadSignature:
raise Http404
# The request is legit and the value has been verified. Track_id has now
# the dictionary with the information included in the tracking
# Get the objects related to the ping
try:
user = get_user_model().objects.get(email=track_id['sender'])
action = Action.objects.get(pk=track_id['action'])
except Exception:
raise Http404
# If the track comes with column_dst, the event needs to be reflected
# back in the data frame
column_dst = track_id.get('column_dst', '')
if column_dst:
# Load the dataframe
data_frame = pandas_db.load_from_db(action.workflow.id)
# Extract the relevant fields from the track_id
column_to = track_id['column_to']
msg_to = track_id['to']
track_col_name = track_id['column_dst']
# New val in DF: df.loc[df['a']==1,'b'] = VAL
data_frame.loc[data_frame[column_to] == msg_to, track_col_name] += 1
# Save DF
ops.store_dataframe_in_db(data_frame, action.workflow.id)
# Get the tracking column and update all the conditions in the
# actions that have this column as part of their formulas
track_col = action.workflow.columns.get(name=track_col_name)
for action in action.workflow.actions.all():
action.update_n_rows_selected(track_col)
# Record the event
logs.ops.put(
user,
'action_email_read',
action.workflow,
{
'to': track_id['to'], # The destination of the email
'email_column':
track_id['column_to'], # The column used to get the
'column_dst': column_dst
})
return HttpResponse(settings.PIXEL, content_type='image/png')
def send_messages(user, action, subject, email_column, canvas_id_column,
from_email, send_to_canvas, send_confirmation, track_read):
"""
Performs the submission of the emails for the given action and with the
given subject. The subject will be evaluated also with respect to the
rows, attributes, and conditions.
:param user: User object that executed the action
:param action: Action from where to take the messages
:param subject: Email subject
:param email_column: Name of the column from which to extract emails
:param from_email: Email of the sender
:param send_confirmation: Boolean to send confirmation to sender
:param track_read: Should read tracking be included?
:return: Send the emails
"""
# Evaluate the action string, evaluate the subject, and get the value of
# the email colummn.
result = evaluate_action(action,
extra_string=subject,
column_name=email_column)
# this is for canvas inbox
recipients_list = []
result_2nd = evaluate_action_canvas(action,
extra_string=subject,
column_name=canvas_id_column)
# Check the type of the result to see if it was successful
if not isinstance(result, list):
# Something went wrong. The result contains a message
return result
track_col_name = ''
data_frame = None
if track_read:
data_frame = pandas_db.load_from_db(action.workflow.id)
# Make sure the column name does not collide with an existing one
i = 0 # Suffix to rename
while True:
i += 1
track_col_name = 'EmailRead_{0}'.format(i)
if track_col_name not in data_frame.columns:
break
# CC the message to canvas conversation Api if the box send_to_canvas is ticked
if send_to_canvas:
msgs = []
for msg_body, msg_subject, msg_to in result_2nd:
# Get the plain text content and bundle it together with the HTML in
# a message to be added to the list.
text_content = strip_tags(msg_body)
recipients_list.append(str(msg_to))
#msg.attach_alternative(msg_body + track_str, "text/html")
#msgs.append(msg)
try:
# send a plain text copy to canvas inbox
p = post_to_canvas_api()
p.set_payload(True, '', '', recipients_list, msg_subject,
html2text.html2text(msg_body))
r = p.post_to_conversation(p.access_token, p.payload)
# extracting response text
pastebin_url = r.text
print(r.status_code)
print(pastebin_url)
except Exception as e:
# Something went wrong, notify above
return str(e)
# Update the number of filtered rows if the action has a filter (table
# might have changed)
filter = action.conditions.filter(is_filter=True).first()
if filter and filter.n_rows_selected != len(result):
filter.n_rows_selected = len(result)
filter.save()
# Everything seemed to work to create the messages.
msgs = []
for msg_body, msg_subject, msg_to in result:
# If read tracking is on, add suffix for message (or empty)
if track_read:
# The track id must identify: action & user
track_id = {
'action': action.id,
'sender': user.email,
'to': msg_to,
'column_to': email_column,
'column_dst': track_col_name
}
track_str = \
"""<img src="https://{0}{1}{2}?v={3}" alt=""
style="position:absolute; visibility:hidden"/>""".format(
Site.objects.get_current().domain,
ontask_settings.BASE_URL,
reverse('trck'),
signing.dumps(track_id)
)
else:
track_str = ''
# Get the plain text content and bundle it together with the HTML in
# a message to be added to the list.
text_content = strip_tags(msg_body)
msg = EmailMultiAlternatives(msg_subject, text_content, from_email,
[msg_to])
msg.attach_alternative(msg_body + track_str, "text/html")
msgs.append(msg)
# Mass mail!
try:
connection = mail.get_connection()
connection.send_messages(msgs)
except Exception as e:
# Something went wrong, notify above
return str(e)
# Add the column if needed
if track_read:
# Create the new column and store
column = Column(name=track_col_name,
workflow=action.workflow,
data_type='integer',
is_key=False,
position=action.workflow.ncols + 1)
column.save()
# Increase the number of columns in the workflow
action.workflow.ncols += 1
action.workflow.save()
# Initial value in the data frame and store the table
data_frame[track_col_name] = 0
ops.store_dataframe_in_db(data_frame, action.workflow.id)
# Log the events (one per email)
now = datetime.datetime.now(pytz.timezone(ontask_settings.TIME_ZONE))
context = {
'user': user.id,
'action': action.id,
'email_sent_datetime': str(now),
}
for msg in msgs:
context['subject'] = msg.subject
context['body'] = msg.body
context['from_email'] = msg.from_email
context['to_email'] = msg.to[0]
logs.ops.put(user, 'action_email_sent', action.workflow, context)
# Log the event
logs.ops.put(
user, 'action_email_sent', action.workflow, {
'user': user.id,
'action': action.name,
'num_messages': len(msgs),
'email_sent_datetime': str(now),
'filter_present': filter is not None,
'num_rows': action.workflow.nrows,
'subject': subject,
'from_email': user.email
})
# If no confirmation email is required, done
if not send_confirmation:
return None
# Creating the context for the personal email
context = {
'user': user,
'action': action,
'num_messages': len(msgs),
'email_sent_datetime': now,
'filter_present': filter is not None,
'num_rows': action.workflow.nrows,
'num_selected': filter.n_rows_selected if filter else -1
}
# Create template and render with context
try:
html_content = Template(str(getattr(settings,
'NOTIFICATION_TEMPLATE'))).render(
Context(context))
text_content = strip_tags(html_content)
except TemplateSyntaxError as e:
return 'Syntax error detected in OnTask notification template (' + \
e.message + ')'
# Log the event
logs.ops.put(
user, 'action_email_notify', action.workflow, {
'user': user.id,
'action': action.id,
'num_messages': len(msgs),
'email_sent_datetime': str(now),
'filter_present': filter is not None,
'num_rows': action.workflow.nrows,
'subject': str(getattr(settings, 'NOTIFICATION_SUBJECT')),
'body': text_content,
'from_email': str(getattr(settings, 'NOTIFICATION_SENDER')),
'to_email': [user.email]
})
# Send email out
try:
send_mail(str(getattr(settings, 'NOTIFICATION_SUBJECT')),
text_content,
str(getattr(settings, 'NOTIFICATION_SENDER')), [user.email],
html_message=html_content)
except Exception as e:
return 'An error occurred when sending your notification: ' + e.message
return None
def clone(request, pk):
"""
AJAX view to clone a workflow
:param request: HTTP request
:param pk: Workflow id
:return: JSON data
"""
# JSON response
data = dict()
# Get the current workflow
workflow = get_workflow(request, pk)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
# Initial data in the context
data['form_is_valid'] = False
context = {'pk': pk, 'name': workflow.name}
if request.method == 'GET':
data['html_form'] = render_to_string(
'workflow/includes/partial_workflow_clone.html',
context,
request=request)
return JsonResponse(data)
# POST REQUEST
# Get the new name appending as many times as needed the 'Copy of '
new_name = 'Copy of ' + workflow.name
while Workflow.objects.filter(name=new_name).exists():
new_name = 'Copy of ' + new_name
workflow.id = None
workflow.name = new_name
try:
workflow.save()
except IntegrityError:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:details',
kwargs={'pk': workflow.id})
messages.error(request, 'Unable to clone workflow')
return JsonResponse(data)
# Get the initial object back
workflow_new = workflow
workflow = get_workflow(request, pk)
# Clone the data frame
data_frame = pandas_db.load_from_db(workflow.pk)
ops.store_dataframe_in_db(data_frame, workflow_new.id)
# Clone actions
action.ops.clone_actions([a for a in workflow.actions.all()], workflow_new)
# Done!
workflow_new.save()
# Log event
logs.ops.put(
request.user, 'workflow_clone', workflow_new, {
'id_old': workflow_new.id,
'id_new': workflow.id,
'name_old': workflow_new.name,
'name_new': workflow.name
})
messages.success(request, 'Workflow successfully cloned.')
data['form_is_valid'] = True
data['html_redirect'] = "" # Reload page
return JsonResponse(data)
def column_add(request):
# TODO: Encapsulate operations in a function so that is available for the
# API
# Data to send as JSON response
data = {}
# Get the workflow element
workflow = get_workflow(request)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
if workflow.nrows == 0:
data['form_is_valid'] = True
data['html_redirect'] = ''
messages.error(
request,
_('Cannot add column to a workflow without data')
)
return JsonResponse(data)
# Form to read/process data
form = ColumnAddForm(request.POST or None, workflow=workflow)
# If a GET or incorrect request, render the form again
if request.method == 'GET' or not form.is_valid():
data['html_form'] = render_to_string(
'workflow/includes/partial_column_addedit.html',
{'form': form,
'add': True},
request=request)
return JsonResponse(data)
# Processing now a valid POST request
# Access the updated information
column_initial_value = form.initial_valid_value
# Save the column object attached to the form
column = form.save(commit=False)
# Fill in the remaining fields in the column
column.workflow = workflow
column.is_key = False
# Update the data frame, which must be stored in the form because
# it was loaded when validating it.
df = pandas_db.load_from_db(workflow.id)
# Add the column with the initial value
df = ops.data_frame_add_column(df, column, column_initial_value)
# Update the column type with the value extracted from the data frame
column.data_type = \
pandas_db.pandas_datatype_names[df[column.name].dtype.name]
# Update the positions of the appropriate columns
workflow.reposition_columns(workflow.ncols + 1, column.position)
column.save()
# Store the df to DB
ops.store_dataframe_in_db(df, workflow.id)
# Log the event
logs.ops.put(request.user,
'column_add',
workflow,
{'id': workflow.id,
'name': workflow.name,
'column_name': column.name,
'column_type': column.data_type})
data['form_is_valid'] = True
data['html_redirect'] = ''
return JsonResponse(data)
def column_edit(request, pk):
# TODO: Encapsulate operations in a function so that is available for the
# API
# Data to send as JSON response
data = {}
# Get the workflow element
workflow = get_workflow(request)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
# Get the column object and make sure it belongs to the workflow
try:
column = Column.objects.get(pk=pk,
workflow=workflow)
except ObjectDoesNotExist:
# Something went wrong, redirect to the workflow detail page
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:detail',
kwargs={'pk': workflow.id})
return JsonResponse(data)
# Form to read/process data
form = ColumnRenameForm(request.POST or None,
workflow=workflow,
instance=column)
old_name = column.name
# Keep a copy of the previous position
old_position = column.position
context = {'form': form,
'cname': old_name,
'pk': pk}
if request.method == 'GET' or not form.is_valid():
data['html_form'] = render_to_string(
'workflow/includes/partial_column_addedit.html',
context,
request=request)
return JsonResponse(data)
# Processing a POST request with valid data in the form
# Process further only if any data changed.
if form.changed_data:
# Some field changed value, so save the result, but
# no commit as we need to propagate the info to the df
column = form.save(commit=False)
# Get the data frame from the form (should be
# loaded)
df = form.data_frame
# If there is a new name, rename the data frame columns
if 'name' in form.changed_data:
# Rename the column in the data frame
df = ops.rename_df_column(df,
workflow,
old_name,
column.name)
if 'position' in form.changed_data:
# Update the positions of the appropriate columns
workflow.reposition_columns(old_position, column.position)
# Save the column information
form.save()
# Changes in column require rebuilding the query_builder_ops
workflow.set_query_builder_ops()
# Save the workflow
workflow.save()
# And save the DF in the DB
ops.store_dataframe_in_db(df, workflow.id)
data['form_is_valid'] = True
data['html_redirect'] = ''
# Log the event
logs.ops.put(request.user,
'column_rename',
workflow,
{'id': workflow.id,
'name': workflow.name,
'column_name': old_name,
'new_name': column.name})
# Done processing the correct POST request
return JsonResponse(data)
def formula_column_add(request):
# TODO: Encapsulate operations in a function so that is available for the
# API
# Data to send as JSON response, in principle, assume form is not valid
data = {'form_is_valid': False}
# Get the workflow element
workflow = get_workflow(request)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
if workflow.nrows == 0:
data['form_is_valid'] = True
data['html_redirect'] = ''
messages.error(
request,
_('Cannot add column to a workflow without data')
)
return JsonResponse(data)
# Form to read/process data
form = FormulaColumnAddForm(
data=request.POST or None,
operands=formula_column_operands,
columns=workflow.columns.all()
)
# If a GET or incorrect request, render the form again
if request.method == 'GET' or not form.is_valid():
data['html_form'] = render_to_string(
'workflow/includes/partial_formula_column_add.html',
{'form': form},
request=request
)
return JsonResponse(data)
# Processing now a valid POST request
# Save the column object attached to the form and add additional fields
column = form.save(commit=False)
column.workflow = workflow
column.is_key = False
# Save the instance
try:
column = form.save()
form.save_m2m()
except IntegrityError as e:
form.add_error('name', _('A column with that name already exists'))
data['html_form'] = render_to_string(
'workflow/includes/partial_formula_column_add.html',
{'form': form},
request=request
)
return JsonResponse(data)
# Update the data frame
df = pandas_db.load_from_db(workflow.id)
try:
# Add the column with the appropriate computation
operation = form.cleaned_data['op_type']
cnames = [c.name for c in form.selected_columns]
if operation == 'sum':
df[column.name] = df[cnames].sum(axis=1)
elif operation == 'prod':
df[column.name] = df[cnames].prod(axis=1)
elif operation == 'max':
df[column.name] = df[cnames].max(axis=1)
elif operation == 'min':
df[column.name] = df[cnames].min(axis=1)
elif operation == 'mean':
df[column.name] = df[cnames].mean(axis=1)
elif operation == 'median':
df[column.name] = df[cnames].median(axis=1)
elif operation == 'std':
df[column.name] = df[cnames].std(axis=1)
elif operation == 'all':
df[column.name] = df[cnames].all(axis=1)
elif operation == 'any':
df[column.name] = df[cnames].any(axis=1)
else:
raise Exception(
_('Operand {0} not implemented').format(operation)
)
except Exception as e:
# Something went wrong in pandas, we need to remove the column
column.delete()
# Notify in the form
form.add_error(
None,
_('Unable to perform the requested operation ({0})').format(
e.message
)
)
data['html_form'] = render_to_string(
'workflow/includes/partial_formula_column_add.html',
{'form': form},
request=request
)
return JsonResponse(data)
# Populate the column type
column.data_type = \
pandas_db.pandas_datatype_names[df[column.name].dtype.name]
# Update the positions of the appropriate columns
workflow.reposition_columns(workflow.ncols + 1, column.position)
column.save()
# Store the df to DB
ops.store_dataframe_in_db(df, workflow.id)
# Log the event
logs.ops.put(request.user,
'column_add',
workflow,
{'id': workflow.id,
'name': workflow.name,
'column_name': column.name,
'column_type': column.data_type})
# The form has been successfully processed
data['form_is_valid'] = True
data['html_redirect'] = '' # Refresh the page
return JsonResponse(data)
def column_add(request):
# Data to send as JSON response
data = {}
# Get the workflow element
workflow = get_workflow(request)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
if workflow.nrows == 0:
data['form_is_valid'] = True
data['html_redirect'] = ''
messages.error(request, 'Cannot add column to a workflow without data')
return JsonResponse(data)
# Form to read/process data
form = ColumnAddForm(request.POST or None, workflow=workflow)
# If a GET or incorrect request, render the form again
if request.method == 'GET' or not form.is_valid():
data['html_form'] = render_to_string(
'workflow/includes/partial_column_add.html', {'form': form},
request=request)
return JsonResponse(data)
# Processing now a valid POST request
# Access the updated information
column_initial_value = form.initial_valid_value
# Save the column object attached to the form
column = form.save(commit=False)
# Fill in the remaining fields in the column
column.workflow = workflow
column.is_key = False
column.save()
# Update the data frame, which must be stored in the form because
# it was loaded when validating it.
df = pandas_db.load_from_db(workflow.id)
# Add the column with the initial value
df = ops.data_frame_add_empty_column(df, column.name, column.data_type,
column_initial_value)
# Store the df to DB
ops.store_dataframe_in_db(df, workflow.id)
# Log the event
logs.ops.put(
request.user, 'column_add', workflow, {
'id': workflow.id,
'name': workflow.name,
'column_name': column.name,
'column_type': column.data_type
})
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:detail',
kwargs={'pk': workflow.id})
return JsonResponse(data)
