def override(self, request, pk, format=None): """ Method to override the content in the workflow :param request: Received request object :param pk: Workflow ID :param format: format for the response :return: """ # Try to retrieve the wflow to check for permissions wflow = self.get_object(pk) serializer = self.serializer_class(data=request.data) if not serializer.is_valid(): # Flag the error return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) # Data received is a correct data frame. df = serializer.validated_data['data_frame'] # Store the content in the db and... ops.store_dataframe_in_db(df, pk) # Update all the counters in the conditions for action in wflow.actions.all(): action.update_n_rows_selected() return Response(None, status=status.HTTP_201_CREATED)
def clone_column(column, new_workflow=None, new_name=None): """ Function that given a column clones it and changes workflow and name :param column: Object to clone :param new_workflow: New workflow object to point :param new_name: New name :return: Cloned object """ # Store the old object name before squashing it old_id = column.id old_name = column.name # Clone column.id = None # Update some of the fields if new_name: column.name = new_name if new_workflow: column.workflow = new_workflow # Update column.save() # Add the column to the table and update it. data_frame = pandas_db.load_from_db(column.workflow.id) data_frame[new_name] = data_frame[old_name] ops.store_dataframe_in_db(data_frame, column.workflow.id) return column
def override(self, request, pk, format=None): # Try to retrieve the wflow to check for permissions self.get_object(pk, user=self.request.user) serializer = self.serializer_class(data=request.data) if serializer.is_valid(): df = serializer.validated_data['data_frame'] ops.store_dataframe_in_db(df, pk) return Response(None, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def reposition_column_and_update_df(workflow, column, to_idx): """ :param workflow: Workflow object for which the repositioning is done :param column: column object to relocate :param to_idx: Destination index of the given column :return: Content reflected in the DB """ df = pandas_db.load_from_db(workflow.id) workflow.reposition_columns(column.position, to_idx) column.position = to_idx column.save() ops.store_dataframe_in_db(df, workflow.id)
def clone_column(column, new_workflow=None, new_name=None): """ Function that given a column clones it and changes workflow and name :param column: Object to clone :param new_workflow: New workflow object to point :param new_name: New name :return: Cloned object """ # Store the old object name before squashing it old_name = column.name old_position = column.position # Clone column.id = None # Update some of the fields if new_name: column.name = new_name if new_workflow: column.workflow = new_workflow # Set column at the end column.position = column.workflow.ncols + 1 column.save() # Update the number of columns in the workflow column.workflow.ncols += 1 column.workflow.save() # Reposition the columns above the one being deleted column.workflow.reposition_columns(column.position, old_position + 1) # Add the column to the table and update it. data_frame = pandas_db.load_from_db(column.workflow.id) data_frame[new_name] = data_frame[old_name] ops.store_dataframe_in_db(data_frame, column.workflow.id) return column
def create(self, validated_data, **kwargs): # Process first the used_columns field to get a sense of how many # columns, their type how many of them new, etc. etc. new_columns = [] for citem in validated_data['used_columns']: cname = citem.get('name', None) if not cname: raise Exception('Incorrect column name {0}.'.format(cname)) col = Column.objects.filter(workflow=self.context['workflow'], name=cname).first() if not col: # new column if citem['is_key']: raise Exception('New action cannot have non-existing key ' 'column {0}'.format(cname)) # Accummulate the new columns just in case we have to undo # the changes new_columns.append(citem) continue # existing column if col.data_type != citem.get('data_type', None) or \ col.is_key != citem['is_key'] or \ set(col.categories) != set(citem['categories']): # The two columns are different raise Exception( 'Imported column {0} is different from existing ' 'one.'.format(cname)) new_column_names = [x['name'] for x in new_columns] action_obj = None try: # used_columns has been verified. action_obj = Action( workflow=self.context['workflow'], name=validated_data['name'], description_text=validated_data['description_text'], is_out=validated_data['is_out'], serve_enabled=validated_data['serve_enabled'], active_from=validated_data['active_from'], active_to=validated_data['active_to'], content=validated_data.get('content', '')) action_obj.save() if new_columns: # There are some new columns that need to be created column_data = ColumnSerializer(data=new_columns, many=True, context=self.context) # And save its content if column_data.is_valid(): column_data.save() workflow = self.context['workflow'] df = pandas_db.load_from_db(self.context['workflow'].id) if df is None: # If there is no data frame, there is no point on # adding columns. Column.objects.filter( workflow=self.context['workflow'], name__in=new_column_names).delete() action_obj.delete() raise Exception('Action cannot be imported with and ' 'empty data table') for col in Column.objects.filter( workflow=workflow, name__in=new_column_names): # Add the column with the initial value df = ops.data_frame_add_column(df, col, None) # Update the column position col.position = len(df.columns) col.save() # Store the df to DB ops.store_dataframe_in_db(df, workflow.id) else: raise Exception('Unable to create column data') # Load the conditions pointing to the action condition_data = ConditionSerializer( data=validated_data.get('conditions', []), many=True, context={'action': action_obj}) if condition_data.is_valid(): condition_data.save() else: raise Exception('Unable to create condition information') # Update the condition variables for each formula if not present for condition in action_obj.conditions.all(): if condition.columns.all().count() == 0: col_names = get_variables(condition.formula) # Add the corresponding columns to the condition condition.columns.set( self.context['workflow'].columns.filter( name__in=col_names)) # Load the columns field columns = ColumnNameSerializer(data=validated_data['columns'], many=True, required=False, context=self.context) if columns.is_valid(): for citem in columns.data: column = action_obj.workflow.columns.get( name=citem['name']) action_obj.columns.add(column) columns.save() else: raise Exception('Unable to create columns field') except Exception: if action_obj and action_obj.id: action_obj.delete() Column.objects.filter(workflow=self.context['workflow'], name__in=new_column_names).delete() raise return action_obj
def create(self, validated_data, **kwargs): # Initial values workflow_obj = None try: workflow_obj = Workflow( user=self.context['user'], name=self.context['name'], description_text=validated_data['description_text'], nrows=0, ncols=0, attributes=validated_data['attributes'], query_builder_ops=validated_data.get('query_builder_ops', {})) workflow_obj.save() # Create the columns column_data = ColumnSerializer(data=validated_data.get( 'columns', []), many=True, context={'workflow': workflow_obj}) # And save its content if column_data.is_valid(): column_data.save() else: raise Exception('Unable to save column information') # If there is any column with position = 0, recompute (this is to # guarantee backward compatibility. if workflow_obj.columns.filter(position=0).exists(): for idx, c in enumerate(workflow_obj.columns.all()): c.position = idx + 1 c.save() # Load the data frame data_frame = validated_data.get('data_frame', None) if data_frame is not None: ops.store_dataframe_in_db(data_frame, workflow_obj.id) # Reconcile now the information in workflow and columns with the # one loaded workflow_obj.data_frame_table_name = \ pandas_db.create_table_name(workflow_obj.pk) workflow_obj.ncols = validated_data['ncols'] workflow_obj.nrows = validated_data['nrows'] workflow_obj.save() # Create the actions pointing to the workflow action_data = ActionSerializer(data=validated_data.get( 'actions', []), many=True, context={'workflow': workflow_obj}) if action_data.is_valid(): action_data.save() else: raise Exception('Unable to save column information') # Create the views pointing to the workflow view_data = ViewSerializer(data=validated_data.get('views', []), many=True, context={'workflow': workflow_obj}) if view_data.is_valid(): view_data.save() else: raise Exception('Unable to save column information') except Exception: # Get rid of the objects created if workflow_obj: if workflow_obj.has_data_frame(): pandas_db.delete_table(workflow_obj.id) if workflow_obj.id: workflow_obj.delete() raise return workflow_obj
def create(self, validated_data, **kwargs): workflow_obj = Workflow( user=self.context['user'], name=self.context['name'], description_text=validated_data['description_text'], nrows=0, ncols=0, attributes=validated_data['attributes'], query_builder_ops=validated_data.get('query_builder_ops', {}) ) workflow_obj.save() # Create the columns column_data = ColumnSerializer( data=validated_data.get('columns', []), many=True, context={'workflow': workflow_obj}) # And save its content if column_data.is_valid(): column_data.save() else: workflow_obj.delete() return None # Create the actions pointing to the workflow action_data = ActionSerializer( data=validated_data.get('actions', []), many=True, context={'workflow': workflow_obj} ) if action_data.is_valid(): action_data.save() else: workflow_obj.delete() return None # Create the views pointing to the workflow view_data = ViewSerializer( data=validated_data.get('views', []), many=True, context={'workflow': workflow_obj} ) if view_data.is_valid(): view_data.save() else: workflow_obj.delete() return None # Load the data frame data_frame = validated_data.get('data_frame', None) if data_frame is not None: ops.store_dataframe_in_db(data_frame, workflow_obj.id) # Reconcile now the information in workflow and columns with the # one loaded workflow_obj.data_frame_table_name = \ pandas_db.create_table_name(workflow_obj.pk) workflow_obj.ncols = validated_data['ncols'] workflow_obj.nrows = validated_data['nrows'] workflow_obj.save() return workflow_obj
def row_create(request): """ Receives a POST request to create a new row in the data table :param request: Request object with all the data. :return: """ # If there is no workflow object, go back to the index workflow = get_workflow(request) if not workflow: return redirect('workflow:index') # If the workflow has no data, the operation should not be allowed if workflow.nrows == 0: return redirect('dataops:uploadmerge') # Create the form form = RowForm(request.POST or None, workflow=workflow) if request.method == 'GET' or not form.is_valid(): return render( request, 'dataops/row_create.html', { 'workflow': workflow, 'form': form, 'cancel_url': reverse('table:display') }) # Create the query to update the row columns = workflow.get_columns() column_names = [c.name for c in columns] field_name = field_prefix + '%s' row_vals = [ form.cleaned_data[field_name % idx] for idx in range(len(columns)) ] # Load the existing df from the db df = pandas_db.load_from_db(workflow.id) # Perform the row addition in the DF first # df2 = pd.DataFrame([[5, 6], [7, 8]], columns=list('AB')) # df.append(df2, ignore_index=True) new_row = pd.DataFrame([row_vals], columns=column_names) df = df.append(new_row, ignore_index=True) # Verify that the unique columns remain unique for ucol in [c for c in columns if c.is_key]: if not ops.is_unique_column(df[ucol.name]): form.add_error( None, 'Repeated value in column ' + ucol.name + '.' + ' It must be different to maintain Key property') return render( request, 'dataops/row_create.html', { 'workflow': workflow, 'form': form, 'cancel_url': reverse('table:display') }) # Restore the dataframe to the DB ops.store_dataframe_in_db(df, workflow.id) # Recompute all the values of the conditions in each of the actions for act in workflow.actions.all(): act.update_n_rows_selected() # Log the event log_payload = zip(column_names, [str(x) for x in row_vals]) logs.ops.put(request.user, 'tablerow_create', workflow, { 'id': workflow.id, 'name': workflow.name, 'new_values': log_payload }) # Done. Back to the table view return redirect('table:display')
def clone(request, pk): """ AJAX view to clone a workflow :param request: HTTP request :param pk: Workflow id :return: JSON data """ # JSON response data = dict() # Get the current workflow workflow = get_workflow(request, pk) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) # Initial data in the context data['form_is_valid'] = False context = {'pk': pk, 'name': workflow.name} if request.method == 'GET': data['html_form'] = render_to_string( 'workflow/includes/partial_workflow_clone.html', context, request=request) return JsonResponse(data) # POST REQUEST # Get the new name appending as many times as needed the 'Copy of ' new_name = 'Copy of ' + workflow.name while Workflow.objects.filter(name=new_name).exists(): new_name = 'Copy of ' + new_name workflow.id = None workflow.name = new_name try: workflow.save() except IntegrityError: data['form_is_valid'] = True data['html_redirect'] = '' messages.error(request, _('Unable to clone workflow')) return JsonResponse(data) # Get the initial object back workflow_new = workflow workflow = get_workflow(request, pk) # Clone the data frame data_frame = pandas_db.load_from_db(workflow.pk) ops.store_dataframe_in_db(data_frame, workflow_new.id) # Clone actions action.ops.clone_actions([a for a in workflow.actions.all()], workflow_new) # Done! workflow_new.save() # Log event Log.objects.register( request.user, Log.WORKFLOW_CLONE, workflow_new, { 'id_old': workflow_new.id, 'id_new': workflow.id, 'name_old': workflow_new.name, 'name_new': workflow.name }) messages.success(request, _('Workflow successfully cloned.')) data['form_is_valid'] = True data['html_redirect'] = "" # Reload page return JsonResponse(data)
def send_messages(user, action, subject, email_column, from_email, cc_email_list, bcc_email_list, send_confirmation, track_read, exclude_values, log_item): """ Sends the emails for the given action and with the given subject. The subject will be evaluated also with respect to the rows, attributes, and conditions. The messages are sent in bursts with a pause in seconds as specified by the configuration variables EMAIL_BURST and EMAIL_BURST_PAUSE :param user: User object that executed the action :param action: Action from where to take the messages :param subject: Email subject :param email_column: Name of the column from which to extract emails :param from_email: Email of the sender :param cc_email_list: List of emails to include in the CC :param bcc_email_list: List of emails to include in the BCC :param send_confirmation: Boolean to send confirmation to sender :param track_read: Should read tracking be included? :param exclude_values: List of values to exclude from the mailing :param log_item: Log object to store results :return: Send the emails """ # Evaluate the action string, evaluate the subject, and get the value of # the email column. result = evaluate_action(action, extra_string=subject, column_name=email_column, exclude_values=exclude_values) # Check the type of the result to see if it was successful if not isinstance(result, list): # Something went wrong. The result contains a message return result track_col_name = '' data_frame = None if track_read: data_frame = pandas_db.load_from_db(action.workflow.id) # Make sure the column name does not collide with an existing one i = 0 # Suffix to rename while True: i += 1 track_col_name = 'EmailRead_{0}'.format(i) if track_col_name not in data_frame.columns: break # Get the log item payload to store the tracking column log_item.payload['track_column'] = track_col_name log_item.save() # Update the number of filtered rows if the action has a filter (table # might have changed) cfilter = action.get_filter() if cfilter and cfilter.n_rows_selected != len(result): cfilter.n_rows_selected = len(result) cfilter.save() # Set the cc_email_list and bcc_email_list to the right values if not cc_email_list: cc_email_list = [] if not bcc_email_list: bcc_email_list = [] # Check that cc and bcc contain list of valid email addresses if not all([validate_email(x) for x in cc_email_list]): return _('Invalid email address in cc email') if not all([validate_email(x) for x in bcc_email_list]): return _('Invalid email address in bcc email') # Everything seemed to work to create the messages. msgs = [] track_ids = [] for msg_body, msg_subject, msg_to in result: # If read tracking is on, add suffix for message (or empty) if track_read: # The track id must identify: action & user track_id = { 'action': action.id, 'sender': user.email, 'to': msg_to, 'column_to': email_column, 'column_dst': track_col_name } track_str = \ """<img src="https://{0}{1}{2}?v={3}" alt="" style="position:absolute; visibility:hidden"/>""".format( Site.objects.get_current().domain, ontask_settings.BASE_URL, reverse('trck'), signing.dumps(track_id) ) else: track_str = '' # Get the plain text content and bundle it together with the HTML in # a message to be added to the list. text_content = html2text.html2text(msg_body) msg = EmailMultiAlternatives(msg_subject, text_content, from_email, [msg_to], bcc=bcc_email_list, cc=cc_email_list) msg.attach_alternative(msg_body + track_str, "text/html") msgs.append(msg) track_ids.append(track_str) # Add the column if needed (before the mass email to avoid overload if track_read: # Create the new column and store column = Column( name=track_col_name, description_text='Emails sent with action {0} on {1}'.format( action.name, str(timezone.now())), workflow=action.workflow, data_type='integer', is_key=False, position=action.workflow.ncols + 1) column.save() # Increase the number of columns in the workflow action.workflow.ncols += 1 action.workflow.save() # Initial value in the data frame and store the table data_frame[track_col_name] = 0 ops.store_dataframe_in_db(data_frame, action.workflow.id) # Partition the list of emails into chunks as per the value of EMAIL_BURST chunk_size = len(msgs) wait_time = 0 if ontask_settings.EMAIL_BURST: chunk_size = ontask_settings.EMAIL_BURST wait_time = ontask_settings.EMAIL_BURST_PAUSE msg_chunks = [ msgs[i:i + chunk_size] for i in range(0, len(msgs), chunk_size) ] for idx, msg_chunk in enumerate(msg_chunks): # Mass mail! try: connection = mail.get_connection() connection.send_messages(msg_chunk) except Exception as e: # Something went wrong, notify above return str(e) if idx != len(msg_chunks) - 1: logger.info( 'Email Burst ({0}) reached. Waiting for {1} secs'.format( len(msg_chunk), wait_time)) sleep(wait_time) # Log the events (one per email) now = datetime.datetime.now(pytz.timezone(ontask_settings.TIME_ZONE)) context = { 'user': user.id, 'action': action.id, 'email_sent_datetime': str(now), } for msg, track_id in zip(msgs, track_ids): context['subject'] = msg.subject context['body'] = msg.body context['from_email'] = msg.from_email context['to_email'] = msg.to[0] if track_id: context['track_id'] = track_id Log.objects.register(user, Log.ACTION_EMAIL_SENT, action.workflow, context) # Update data in the log item log_item.payload['objects_sent'] = len(result) log_item.payload['filter_present'] = cfilter is not None log_item.payload['datetime'] = str( datetime.datetime.now(pytz.timezone(ontask_settings.TIME_ZONE))) log_item.save() # If no confirmation email is required, done if not send_confirmation: return None # Creating the context for the confirmation email context = { 'user': user, 'action': action, 'num_messages': len(msgs), 'email_sent_datetime': now, 'filter_present': cfilter is not None, 'num_rows': action.workflow.nrows, 'num_selected': cfilter.n_rows_selected if cfilter else -1 } # Create template and render with context try: html_content = Template(str(getattr(settings, 'NOTIFICATION_TEMPLATE'))).render( Context(context)) text_content = strip_tags(html_content) except TemplateSyntaxError as e: return _('Syntax error detected in OnTask notification template ' '({0})').format(e) # Log the event Log.objects.register( user, Log.ACTION_EMAIL_NOTIFY, action.workflow, { 'user': user.id, 'action': action.id, 'num_messages': len(msgs), 'email_sent_datetime': str(now), 'filter_present': cfilter is not None, 'num_rows': action.workflow.nrows, 'subject': str(getattr(settings, 'NOTIFICATION_SUBJECT')), 'body': text_content, 'from_email': str(getattr(settings, 'NOTIFICATION_SENDER')), 'to_email': [user.email] }) # Send email out try: send_mail(str(getattr(settings, 'NOTIFICATION_SUBJECT')), text_content, str(getattr(settings, 'NOTIFICATION_SENDER')), [user.email], html_message=html_content) except Exception as e: return _('An error occurred when sending your notification: ' '{0}').format(e) return None
def column_add(request, pk=None): # TODO: Encapsulate operations in a function so that is available for the # API # Data to send as JSON response data = {} # Detect if this operation is to add a new column or a new question (in # the edit in page) is_question = pk is not None # Get the workflow element workflow = get_workflow(request) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) if workflow.nrows == 0: data['form_is_valid'] = True data['html_redirect'] = '' if is_question: messages.error(request, _('Cannot add question to a workflow without data')) else: messages.error(request, _('Cannot add column to a workflow without data')) return JsonResponse(data) action_id = None if pk: # Get the action and the columns try: action = Action.objects.filter( Q(workflow__user=request.user) | Q(workflow__shared=request.user)).distinct().get(pk=pk) except ObjectDoesNotExist: messages.error(request, _('Cannot find action to add question.')) return JsonResponse({'html_redirect': reverse('action:index')}) action_id = action.id # Form to read/process data if is_question: form = QuestionAddForm(request.POST or None, workflow=workflow) else: form = ColumnAddForm(request.POST or None, workflow=workflow) # If a GET or incorrect request, render the form again if request.method == 'GET' or not form.is_valid(): if is_question: template = 'workflow/includes/partial_question_addedit.html' else: template = 'workflow/includes/partial_column_addedit.html' data['html_form'] = render_to_string(template, { 'form': form, 'is_question': is_question, 'action_id': action_id, 'add': True }, request=request) return JsonResponse(data) # Processing now a valid POST request # Access the updated information column_initial_value = form.initial_valid_value # Save the column object attached to the form column = form.save(commit=False) # Fill in the remaining fields in the column column.workflow = workflow column.is_key = False # Update the data frame, which must be stored in the form because # it was loaded when validating it. df = pandas_db.load_from_db(workflow.id) # Add the column with the initial value to the dataframe df = ops.data_frame_add_column(df, column, column_initial_value) # Update the column type with the value extracted from the data frame column.data_type = \ pandas_db.pandas_datatype_names[df[column.name].dtype.name] # Update the positions of the appropriate columns workflow.reposition_columns(workflow.ncols + 1, column.position) column.save() # Store the df to DB ops.store_dataframe_in_db(df, workflow.id) # If the column is a question, add it to the action if is_question: action.columns.add(column) # Log the event if is_question: event_type = Log.QUESTION_ADD else: event_type = Log.COLUMN_ADD Log.objects.register( request.user, event_type, workflow, { 'id': workflow.id, 'name': workflow.name, 'column_name': column.name, 'column_type': column.data_type }) data['form_is_valid'] = True data['html_redirect'] = '' return JsonResponse(data)
def column_edit(request, pk): # TODO: Encapsulate operations in a function so that is available for the # API # Data to send as JSON response data = {} # Detect if this operation is to edit a new column or a new question (in # the edit in page) is_question = 'question_edit' in request.path_info # Get the workflow element workflow = get_workflow(request) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) # Get the column object and make sure it belongs to the workflow try: column = Column.objects.get(pk=pk, workflow=workflow) except ObjectDoesNotExist: # Something went wrong, redirect to the workflow detail page data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:detail', kwargs={'pk': workflow.id}) return JsonResponse(data) # Form to read/process data if is_question: form = QuestionRenameForm(request.POST or None, workflow=workflow, instance=column) else: form = ColumnRenameForm(request.POST or None, workflow=workflow, instance=column) old_name = column.name # Keep a copy of the previous position old_position = column.position context = {'form': form, 'cname': old_name, 'pk': pk} if request.method == 'GET' or not form.is_valid(): if is_question: template = 'workflow/includes/partial_question_addedit.html' else: template = 'workflow/includes/partial_column_addedit.html' data['html_form'] = render_to_string(template, context, request=request) return JsonResponse(data) # Processing a POST request with valid data in the form # Process further only if any data changed. if form.changed_data: # Some field changed value, so save the result, but # no commit as we need to propagate the info to the df column = form.save(commit=False) # Get the data frame from the form (should be # loaded) df = form.data_frame # If there is a new name, rename the data frame columns if 'name' in form.changed_data: # Rename the column in the data frame df = ops.rename_df_column(df, workflow, old_name, column.name) if 'position' in form.changed_data: # Update the positions of the appropriate columns workflow.reposition_columns(old_position, column.position) # Save the column information form.save() # Changes in column require rebuilding the query_builder_ops workflow.set_query_builder_ops() # Save the workflow workflow.save() # And save the DF in the DB ops.store_dataframe_in_db(df, workflow.id) data['form_is_valid'] = True data['html_redirect'] = '' # Log the event if is_question: event_type = Log.QUESTION_ADD else: event_type = Log.COLUMN_ADD Log.objects.register( request.user, event_type, workflow, { 'id': workflow.id, 'name': workflow.name, 'column_name': old_name, 'new_name': column.name }) # Done processing the correct POST request return JsonResponse(data)
def random_column_add(request): """ Function that creates a column with random values (Modal) :param request: :return: """ # TODO: Encapsulate operations in a function so that is available for the # API # Data to send as JSON response, in principle, assume form is not valid data = {'form_is_valid': False} # Get the workflow element workflow = get_workflow(request) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) if workflow.nrows == 0: data['form_is_valid'] = True data['html_redirect'] = '' messages.error(request, _('Cannot add column to a workflow without data')) return JsonResponse(data) # Form to read/process data form = RandomColumnAddForm(data=request.POST or None) # If a GET or incorrect request, render the form again if request.method == 'GET' or not form.is_valid(): data['html_form'] = render_to_string( 'workflow/includes/partial_random_column_add.html', {'form': form}, request=request) return JsonResponse(data) # Processing now a valid POST request # Save the column object attached to the form and add additional fields column = form.save(commit=False) column.workflow = workflow column.is_key = False # Save the instance try: column = form.save() form.save_m2m() except IntegrityError: form.add_error('name', _('A column with that name already exists')) data['html_form'] = render_to_string( 'workflow/includes/partial_random_column_add.html', {'form': form}, request=request) return JsonResponse(data) # Update the data frame df = pandas_db.load_from_db(workflow.id) # Get the values and interpret its meaning values = form.cleaned_data['values'] int_value = None # First, try to see if the field is a valid integer try: int_value = int(values) except ValueError: pass if int_value: # At this point the field is an integer if int_value <= 1: form.add_error('values', _('The integer value has to be larger than 1')) data['html_form'] = render_to_string( 'workflow/includes/partial_random_column_add.html', {'form': form}, request=request) return JsonResponse(data) df[column.name] = [ random.randint(1, int_value) for __ in range(workflow.nrows) ] else: # At this point the field is a string and the values are the comma # separated strings. vals = [x.strip() for x in values.strip().split(',') if x] if not vals: form.add_error('values', _('The value has to be a comma-separated list')) data['html_form'] = render_to_string( 'workflow/includes/partial_random_column_add.html', {'form': form}, request=request) return JsonResponse(data) df[column.name] = [random.choice(vals) for __ in range(workflow.nrows)] # Populate the column type column.data_type = \ pandas_db.pandas_datatype_names[df[column.name].dtype.name] # Update the positions of the appropriate columns workflow.reposition_columns(workflow.ncols + 1, column.position) column.save() # Store the df to DB ops.store_dataframe_in_db(df, workflow.id) # Log the event Log.objects.register( request.user, Log.COLUMN_ADD_RANDOM, workflow, { 'id': workflow.id, 'name': workflow.name, 'column_name': column.name, 'column_type': column.data_type, 'value': values }) # The form has been successfully processed data['form_is_valid'] = True data['html_redirect'] = '' # Refresh the page return JsonResponse(data)
def formula_column_add(request): # TODO: Encapsulate operations in a function so that is available for the # API # Data to send as JSON response, in principle, assume form is not valid data = {'form_is_valid': False} # Get the workflow element workflow = get_workflow(request) if not workflow: data['form_is_valid'] = True data['html_redirect'] = reverse('workflow:index') return JsonResponse(data) if workflow.nrows == 0: data['form_is_valid'] = True data['html_redirect'] = '' messages.error(request, _('Cannot add column to a workflow without data')) return JsonResponse(data) # Form to read/process data form = FormulaColumnAddForm(data=request.POST or None, operands=formula_column_operands, columns=workflow.columns.all()) # If a GET or incorrect request, render the form again if request.method == 'GET' or not form.is_valid(): data['html_form'] = render_to_string( 'workflow/includes/partial_formula_column_add.html', {'form': form}, request=request) return JsonResponse(data) # Processing now a valid POST request # Save the column object attached to the form and add additional fields column = form.save(commit=False) column.workflow = workflow column.is_key = False # Save the instance try: column = form.save() form.save_m2m() except IntegrityError: form.add_error('name', _('A column with that name already exists')) data['html_form'] = render_to_string( 'workflow/includes/partial_formula_column_add.html', {'form': form}, request=request) return JsonResponse(data) # Update the data frame df = pandas_db.load_from_db(workflow.id) try: # Add the column with the appropriate computation operation = form.cleaned_data['op_type'] cnames = [c.name for c in form.selected_columns] if operation == 'sum': df[column.name] = df[cnames].sum(axis=1) elif operation == 'prod': df[column.name] = df[cnames].prod(axis=1) elif operation == 'max': df[column.name] = df[cnames].max(axis=1) elif operation == 'min': df[column.name] = df[cnames].min(axis=1) elif operation == 'mean': df[column.name] = df[cnames].mean(axis=1) elif operation == 'median': df[column.name] = df[cnames].median(axis=1) elif operation == 'std': df[column.name] = df[cnames].std(axis=1) elif operation == 'all': df[column.name] = df[cnames].all(axis=1) elif operation == 'any': df[column.name] = df[cnames].any(axis=1) else: raise Exception(_('Operand {0} not implemented').format(operation)) except Exception as e: # Something went wrong in pandas, we need to remove the column column.delete() # Notify in the form form.add_error( None, _('Unable to perform the requested operation ({0})').format( e.message)) data['html_form'] = render_to_string( 'workflow/includes/partial_formula_column_add.html', {'form': form}, request=request) return JsonResponse(data) # Populate the column type column.data_type = \ pandas_db.pandas_datatype_names[df[column.name].dtype.name] # Update the positions of the appropriate columns workflow.reposition_columns(workflow.ncols + 1, column.position) column.save() # Store the df to DB ops.store_dataframe_in_db(df, workflow.id) # Log the event Log.objects.register( request.user, Log.COLUMN_ADD_FORMULA, workflow, { 'id': workflow.id, 'name': workflow.name, 'column_name': column.name, 'column_type': column.data_type }) # The form has been successfully processed data['form_is_valid'] = True data['html_redirect'] = '' # Refresh the page return JsonResponse(data)