def clone_column(column, new_workflow=None, new_name=None): """ Function that given a column clones it and changes workflow and name :param column: Object to clone :param new_workflow: New workflow object to point :param new_name: New name :return: Cloned object """ # Store the old object name before squashing it old_id = column.id old_name = column.name # Clone column.id = None # Update some of the fields if new_name: column.name = new_name if new_workflow: column.workflow = new_workflow # Update column.save() # Add the column to the table and update it. data_frame = pandas_db.load_from_db(column.workflow.id) data_frame[new_name] = data_frame[old_name] ops.store_dataframe_in_db(data_frame, column.workflow.id) return column
def reposition_column_and_update_df(workflow, column, to_idx): """ :param workflow: Workflow object for which the repositioning is done :param column: column object to relocate :param to_idx: Destination index of the given column :return: Content reflected in the DB """ df = pandas_db.load_from_db(workflow.id) reposition_columns(workflow, column.position, to_idx) column.position = to_idx column.save() ops.store_dataframe_in_db(df, workflow.id)
def clone_column(column, new_workflow=None, new_name=None): """ Function that given a column clones it and changes workflow and name :param column: Object to clone :param new_workflow: New workflow object to point :param new_name: New name :return: Cloned object """ # Store the old object name before squashing it old_name = column.name old_position = column.position # Clone column.id = None # Update some of the fields if new_name: column.name = new_name if new_workflow: column.workflow = new_workflow # Set column at the end column.position = column.workflow.ncols + 1 column.save() # Update the number of columns in the workflow column.workflow.ncols += 1 column.workflow.save() # Reposition the columns above the one being deleted reposition_columns(column.workflow, column.position, old_position + 1) # Set the new column in the right location column.position = old_position + 1 column.save() # Add the column to the table and update it. data_frame = pandas_db.load_from_db(column.workflow.id) data_frame[new_name] = data_frame[old_name] ops.store_dataframe_in_db(data_frame, column.workflow.id) return column
def row_create(request): """ Receives a POST request to create a new row in the data table :param request: Request object with all the data. :return: """ # If there is no workflow object, go back to the index workflow = get_workflow(request) if not workflow: return redirect('workflow:index') # If the workflow has no data, the operation should not be allowed if workflow.nrows == 0: return redirect('dataops:list') # Create the form form = RowForm(request.POST or None, workflow=workflow) if request.method == 'GET' or not form.is_valid(): return render( request, 'dataops/row_create.html', { 'workflow': workflow, 'form': form, 'cancel_url': reverse('table:display') }) # Create the query to update the row columns = workflow.get_columns() column_names = [c.name for c in columns] field_name = field_prefix + '%s' row_vals = [ form.cleaned_data[field_name % idx] for idx in range(len(columns)) ] # Load the existing df from the db df = pandas_db.load_from_db(workflow.id) # Perform the row addition in the DF first # df2 = pd.DataFrame([[5, 6], [7, 8]], columns=list('AB')) # df.append(df2, ignore_index=True) new_row = pd.DataFrame([row_vals], columns=column_names) df = df.append(new_row, ignore_index=True) # Verify that the unique columns remain unique for ucol in [c for c in columns if c.is_key]: if not ops.is_unique_column(df[ucol.name]): form.add_error( None, 'Repeated value in column ' + ucol.name + '.' + ' It must be different to maintain Key property') return render( request, 'dataops/row_create.html', { 'workflow': workflow, 'form': form, 'cancel_url': reverse('table:display') }) # Restore the dataframe to the DB ops.store_dataframe_in_db(df, workflow.id) # Log the event log_payload = zip(column_names, [str(x) for x in row_vals]) logs.ops.put(request.user, 'tablerow_create', workflow, { 'id': workflow.id, 'name': workflow.name, 'new_values': log_payload }) # Done. Back to the table view return redirect('table:display')
def trck(request): """ Receive a request with a token from email read tracking :param request: Request object :return: Reflects in the DB the reception and (optionally) in the data table of the workflow """ if request.method != 'GET': raise Http404 # Detected attempt to track event track_id = request.GET.get('v', None) if not track_id: raise Http404 # If the track_id is not correctly signed, out. try: track_id = signing.loads(track_id) except signing.BadSignature: raise Http404 # The request is legit and the value has been verified. Track_id has now # the dictionary with the information included in the tracking # Get the objects related to the ping try: user = get_user_model().objects.get(email=track_id['sender']) action = Action.objects.get(pk=track_id['action']) except Exception: raise Http404 # If the track comes with column_dst, the event needs to be reflected # back in the data frame column_dst = track_id.get('column_dst', '') if column_dst: # Load the dataframe data_frame = pandas_db.load_from_db(action.workflow.id) # Extract the relevant fields from the track_id column_to = track_id['column_to'] msg_to = track_id['to'] track_col_name = track_id['column_dst'] # New val in DF: df.loc[df['a']==1,'b'] = VAL data_frame.loc[data_frame[column_to] == msg_to, track_col_name] += 1 # Save DF ops.store_dataframe_in_db(data_frame, action.workflow.id) # Get the tracking column and update all the conditions in the # actions that have this column as part of their formulas track_col = action.workflow.columns.get(name=track_col_name) for action in action.workflow.actions.all(): action.update_n_rows_selected(track_col) # Record the event logs.ops.put( user, 'action_email_read', action.workflow, { 'to': track_id['to'], # The destination of the email 'email_column': track_id['column_to'], # The column used to get the 'column_dst': column_dst }) return HttpResponse(settings.PIXEL, content_type='image/png')
def send_messages(user, action, subject, email_column, canvas_id_column, from_email, send_to_canvas, send_confirmation, track_read): """ Performs the submission of the emails for the given action and with the given subject. The subject will be evaluated also with respect to the rows, attributes, and conditions. :param user: User object that executed the action :param action: Action from where to take the messages :param subject: Email subject :param email_column: Name of the column from which to extract emails :param from_email: Email of the sender :param send_confirmation: Boolean to send confirmation to sender :param track_read: Should read tracking be included? :return: Send the emails """ # Evaluate the action string, evaluate the subject, and get the value of # the email colummn. result = evaluate_action(action, extra_string=subject, column_name=email_column) # this is for canvas inbox recipients_list = [] result_2nd = evaluate_action_canvas(action, extra_string=subject, column_name=canvas_id_column) # Check the type of the result to see if it was successful if not isinstance(result, list): # Something went wrong. The result contains a message return result track_col_name = '' data_frame = None if track_read: data_frame = pandas_db.load_from_db(action.workflow.id) # Make sure the column name does not collide with an existing one i = 0 # Suffix to rename while True: i += 1 track_col_name = 'EmailRead_{0}'.format(i) if track_col_name not in data_frame.columns: break # CC the message to canvas conversation Api if the box send_to_canvas is ticked if send_to_canvas: msgs = [] for msg_body, msg_subject, msg_to in result_2nd: # Get the plain text content and bundle it together with the HTML in # a message to be added to the list. text_content = strip_tags(msg_body) recipients_list.append(str(msg_to)) #msg.attach_alternative(msg_body + track_str, "text/html") #msgs.append(msg) try: # send a plain text copy to canvas inbox p = post_to_canvas_api() p.set_payload(True, '', '', recipients_list, msg_subject, html2text.html2text(msg_body)) r = p.post_to_conversation(p.access_token, p.payload) # extracting response text pastebin_url = r.text print(r.status_code) print(pastebin_url) except Exception as e: # Something went wrong, notify above return str(e) # Update the number of filtered rows if the action has a filter (table # might have changed) filter = action.conditions.filter(is_filter=True).first() if filter and filter.n_rows_selected != len(result): filter.n_rows_selected = len(result) filter.save() # Everything seemed to work to create the messages. msgs = [] for msg_body, msg_subject, msg_to in result: # If read tracking is on, add suffix for message (or empty) if track_read: # The track id must identify: action & user track_id = { 'action': action.id, 'sender': user.email, 'to': msg_to, 'column_to': email_column, 'column_dst': track_col_name } track_str = \ """<img src="https://{0}{1}{2}?v={3}" alt="" style="position:absolute; visibility:hidden"/>""".format( Site.objects.get_current().domain, ontask_settings.BASE_URL, reverse('trck'), signing.dumps(track_id) ) else: track_str = '' # Get the plain text content and bundle it together with the HTML in # a message to be added to the list. text_content = strip_tags(msg_body) msg = EmailMultiAlternatives(msg_subject, text_content, from_email, [msg_to]) msg.attach_alternative(msg_body + track_str, "text/html") msgs.append(msg) # Mass mail! try: connection = mail.get_connection() connection.send_messages(msgs) except Exception as e: # Something went wrong, notify above return str(e) # Add the column if needed if track_read: # Create the new column and store column = Column(name=track_col_name, workflow=action.workflow, data_type='integer', is_key=False, position=action.workflow.ncols + 1) column.save() # Increase the number of columns in the workflow action.workflow.ncols += 1 action.workflow.save() # Initial value in the data frame and store the table data_frame[track_col_name] = 0 ops.store_dataframe_in_db(data_frame, action.workflow.id) # Log the events (one per email) now = datetime.datetime.now(pytz.timezone(ontask_settings.TIME_ZONE)) context = { 'user': user.id, 'action': action.id, 'email_sent_datetime': str(now), } for msg in msgs: context['subject'] = msg.subject context['body'] = msg.body context['from_email'] = msg.from_email context['to_email'] = msg.to[0] logs.ops.put(user, 'action_email_sent', action.workflow, context) # Log the event logs.ops.put( user, 'action_email_sent', action.workflow, { 'user': user.id, 'action': action.name, 'num_messages': len(msgs), 'email_sent_datetime': str(now), 'filter_present': filter is not None, 'num_rows': action.workflow.nrows, 'subject': subject, 'from_email': user.email }) # If no confirmation email is required, done if not send_confirmation: return None # Creating the context for the personal email context = { 'user': user, 'action': action, 'num_messages': len(msgs), 'email_sent_datetime': now, 'filter_present': filter is not None, 'num_rows': action.workflow.nrows, 'num_selected': filter.n_rows_selected if filter else -1 } # Create template and render with context try: html_content = Template(str(getattr(settings, 'NOTIFICATION_TEMPLATE'))).render( Context(context)) text_content = strip_tags(html_content) except TemplateSyntaxError as e: return 'Syntax error detected in OnTask notification template (' + \ e.message + ')' # Log the event logs.ops.put( user, 'action_email_notify', action.workflow, { 'user': user.id, 'action': action.id, 'num_messages': len(msgs), 'email_sent_datetime': str(now), 'filter_present': filter is not None, 'num_rows': action.workflow.nrows, 'subject': str(getattr(settings, 'NOTIFICATION_SUBJECT')), 'body': text_content, 'from_email': str(getattr(settings, 'NOTIFICATION_SENDER')), 'to_email': [user.email] }) # Send email out try: send_mail(str(getattr(settings, 'NOTIFICATION_SUBJECT')), text_content, str(getattr(settings, 'NOTIFICATION_SENDER')), [user.email], html_message=html_content) except Exception as e: return 'An error occurred when sending your notification: ' + e.message return None
def clone(request, pk): """ AJAX view to clone a workflow :param request: HTTP request :param pk: Workflow id :return: JSON data """ # JSON response data = dict() # Get the current workflow workflow = get_workflow(request, pk) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) # Initial data in the context data['form_is_valid'] = False context = {'pk': pk, 'name': workflow.name} if request.method == 'GET': data['html_form'] = render_to_string( 'workflow/includes/partial_workflow_clone.html', context, request=request) return JsonResponse(data) # POST REQUEST # Get the new name appending as many times as needed the 'Copy of ' new_name = 'Copy of ' + workflow.name while Workflow.objects.filter(name=new_name).exists(): new_name = 'Copy of ' + new_name workflow.id = None workflow.name = new_name try: workflow.save() except IntegrityError: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:details', kwargs={'pk': workflow.id}) messages.error(request, 'Unable to clone workflow') return JsonResponse(data) # Get the initial object back workflow_new = workflow workflow = get_workflow(request, pk) # Clone the data frame data_frame = pandas_db.load_from_db(workflow.pk) ops.store_dataframe_in_db(data_frame, workflow_new.id) # Clone actions action.ops.clone_actions([a for a in workflow.actions.all()], workflow_new) # Done! workflow_new.save() # Log event logs.ops.put( request.user, 'workflow_clone', workflow_new, { 'id_old': workflow_new.id, 'id_new': workflow.id, 'name_old': workflow_new.name, 'name_new': workflow.name }) messages.success(request, 'Workflow successfully cloned.') data['form_is_valid'] = True data['html_redirect'] = "" # Reload page return JsonResponse(data)
def column_add(request): # TODO: Encapsulate operations in a function so that is available for the # API # Data to send as JSON response data = {} # Get the workflow element workflow = get_workflow(request) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) if workflow.nrows == 0: data['form_is_valid'] = True data['html_redirect'] = '' messages.error( request, _('Cannot add column to a workflow without data') ) return JsonResponse(data) # Form to read/process data form = ColumnAddForm(request.POST or None, workflow=workflow) # If a GET or incorrect request, render the form again if request.method == 'GET' or not form.is_valid(): data['html_form'] = render_to_string( 'workflow/includes/partial_column_addedit.html', {'form': form, 'add': True}, request=request) return JsonResponse(data) # Processing now a valid POST request # Access the updated information column_initial_value = form.initial_valid_value # Save the column object attached to the form column = form.save(commit=False) # Fill in the remaining fields in the column column.workflow = workflow column.is_key = False # Update the data frame, which must be stored in the form because # it was loaded when validating it. df = pandas_db.load_from_db(workflow.id) # Add the column with the initial value df = ops.data_frame_add_column(df, column, column_initial_value) # Update the column type with the value extracted from the data frame column.data_type = \ pandas_db.pandas_datatype_names[df[column.name].dtype.name] # Update the positions of the appropriate columns workflow.reposition_columns(workflow.ncols + 1, column.position) column.save() # Store the df to DB ops.store_dataframe_in_db(df, workflow.id) # Log the event logs.ops.put(request.user, 'column_add', workflow, {'id': workflow.id, 'name': workflow.name, 'column_name': column.name, 'column_type': column.data_type}) data['form_is_valid'] = True data['html_redirect'] = '' return JsonResponse(data)
def column_edit(request, pk): # TODO: Encapsulate operations in a function so that is available for the # API # Data to send as JSON response data = {} # Get the workflow element workflow = get_workflow(request) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) # Get the column object and make sure it belongs to the workflow try: column = Column.objects.get(pk=pk, workflow=workflow) except ObjectDoesNotExist: # Something went wrong, redirect to the workflow detail page data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:detail', kwargs={'pk': workflow.id}) return JsonResponse(data) # Form to read/process data form = ColumnRenameForm(request.POST or None, workflow=workflow, instance=column) old_name = column.name # Keep a copy of the previous position old_position = column.position context = {'form': form, 'cname': old_name, 'pk': pk} if request.method == 'GET' or not form.is_valid(): data['html_form'] = render_to_string( 'workflow/includes/partial_column_addedit.html', context, request=request) return JsonResponse(data) # Processing a POST request with valid data in the form # Process further only if any data changed. if form.changed_data: # Some field changed value, so save the result, but # no commit as we need to propagate the info to the df column = form.save(commit=False) # Get the data frame from the form (should be # loaded) df = form.data_frame # If there is a new name, rename the data frame columns if 'name' in form.changed_data: # Rename the column in the data frame df = ops.rename_df_column(df, workflow, old_name, column.name) if 'position' in form.changed_data: # Update the positions of the appropriate columns workflow.reposition_columns(old_position, column.position) # Save the column information form.save() # Changes in column require rebuilding the query_builder_ops workflow.set_query_builder_ops() # Save the workflow workflow.save() # And save the DF in the DB ops.store_dataframe_in_db(df, workflow.id) data['form_is_valid'] = True data['html_redirect'] = '' # Log the event logs.ops.put(request.user, 'column_rename', workflow, {'id': workflow.id, 'name': workflow.name, 'column_name': old_name, 'new_name': column.name}) # Done processing the correct POST request return JsonResponse(data)
def formula_column_add(request): # TODO: Encapsulate operations in a function so that is available for the # API # Data to send as JSON response, in principle, assume form is not valid data = {'form_is_valid': False} # Get the workflow element workflow = get_workflow(request) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) if workflow.nrows == 0: data['form_is_valid'] = True data['html_redirect'] = '' messages.error( request, _('Cannot add column to a workflow without data') ) return JsonResponse(data) # Form to read/process data form = FormulaColumnAddForm( data=request.POST or None, operands=formula_column_operands, columns=workflow.columns.all() ) # If a GET or incorrect request, render the form again if request.method == 'GET' or not form.is_valid(): data['html_form'] = render_to_string( 'workflow/includes/partial_formula_column_add.html', {'form': form}, request=request ) return JsonResponse(data) # Processing now a valid POST request # Save the column object attached to the form and add additional fields column = form.save(commit=False) column.workflow = workflow column.is_key = False # Save the instance try: column = form.save() form.save_m2m() except IntegrityError as e: form.add_error('name', _('A column with that name already exists')) data['html_form'] = render_to_string( 'workflow/includes/partial_formula_column_add.html', {'form': form}, request=request ) return JsonResponse(data) # Update the data frame df = pandas_db.load_from_db(workflow.id) try: # Add the column with the appropriate computation operation = form.cleaned_data['op_type'] cnames = [c.name for c in form.selected_columns] if operation == 'sum': df[column.name] = df[cnames].sum(axis=1) elif operation == 'prod': df[column.name] = df[cnames].prod(axis=1) elif operation == 'max': df[column.name] = df[cnames].max(axis=1) elif operation == 'min': df[column.name] = df[cnames].min(axis=1) elif operation == 'mean': df[column.name] = df[cnames].mean(axis=1) elif operation == 'median': df[column.name] = df[cnames].median(axis=1) elif operation == 'std': df[column.name] = df[cnames].std(axis=1) elif operation == 'all': df[column.name] = df[cnames].all(axis=1) elif operation == 'any': df[column.name] = df[cnames].any(axis=1) else: raise Exception( _('Operand {0} not implemented').format(operation) ) except Exception as e: # Something went wrong in pandas, we need to remove the column column.delete() # Notify in the form form.add_error( None, _('Unable to perform the requested operation ({0})').format( e.message ) ) data['html_form'] = render_to_string( 'workflow/includes/partial_formula_column_add.html', {'form': form}, request=request ) return JsonResponse(data) # Populate the column type column.data_type = \ pandas_db.pandas_datatype_names[df[column.name].dtype.name] # Update the positions of the appropriate columns workflow.reposition_columns(workflow.ncols + 1, column.position) column.save() # Store the df to DB ops.store_dataframe_in_db(df, workflow.id) # Log the event logs.ops.put(request.user, 'column_add', workflow, {'id': workflow.id, 'name': workflow.name, 'column_name': column.name, 'column_type': column.data_type}) # The form has been successfully processed data['form_is_valid'] = True data['html_redirect'] = '' # Refresh the page return JsonResponse(data)
def column_add(request): # Data to send as JSON response data = {} # Get the workflow element workflow = get_workflow(request) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) if workflow.nrows == 0: data['form_is_valid'] = True data['html_redirect'] = '' messages.error(request, 'Cannot add column to a workflow without data') return JsonResponse(data) # Form to read/process data form = ColumnAddForm(request.POST or None, workflow=workflow) # If a GET or incorrect request, render the form again if request.method == 'GET' or not form.is_valid(): data['html_form'] = render_to_string( 'workflow/includes/partial_column_add.html', {'form': form}, request=request) return JsonResponse(data) # Processing now a valid POST request # Access the updated information column_initial_value = form.initial_valid_value # Save the column object attached to the form column = form.save(commit=False) # Fill in the remaining fields in the column column.workflow = workflow column.is_key = False column.save() # Update the data frame, which must be stored in the form because # it was loaded when validating it. df = pandas_db.load_from_db(workflow.id) # Add the column with the initial value df = ops.data_frame_add_empty_column(df, column.name, column.data_type, column_initial_value) # Store the df to DB ops.store_dataframe_in_db(df, workflow.id) # Log the event logs.ops.put( request.user, 'column_add', workflow, { 'id': workflow.id, 'name': workflow.name, 'column_name': column.name, 'column_type': column.data_type }) data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:detail', kwargs={'pk': workflow.id}) return JsonResponse(data)