def test_02_second_plugin(self): # Login self.login('*****@*****.**') self.open(reverse('workflow:index')) # GO TO THE WORKFLOW PAGE WebDriverWait(self.selenium, 10).until(EC.title_is('OnTask :: Workflows')) self.assertIn('New workflow', self.selenium.page_source) self.assertIn('Import workflow', self.selenium.page_source) # Open the workflow wf_link = self.selenium.find_element_by_link_text('Plugin test') wf_link.click() WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Open the transform page self.selenium.find_element_by_link_text("Dataops").click() self.selenium.find_element_by_link_text("Transform").click() WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located( (By.ID, 'transform-table_previous'))) # Click in the second plugin self.selenium.find_element_by_xpath( "//table[@id='transform-table']/tbody/tr[5]/td[7]/div/a").click() WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.NAME, 'csrfmiddlewaretoken'))) # Provide the execution data self.selenium.find_element_by_id("id_merge_key").click() Select(self.selenium.find_element_by_id( "id_merge_key")).select_by_visible_text("email") # Submit the execution self.selenium.find_element_by_name("Submit").click() WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'plugin-execution-report'))) # Done. Click continue. self.selenium.find_element_by_xpath( "(//button[@type='button'])[2]").click() WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Assert the content of the dataframe wflow = Workflow.objects.get(name='Plugin test') df = pandas_db.load_from_db(wflow.id) self.assertTrue('RESULT 3' in set(df.columns)) self.assertTrue('RESULT 4' in set(df.columns)) self.assertTrue(df['RESULT 3'].equals(df['A1'] + df['A2'])) self.assertTrue(df['RESULT 4'].equals(df['A1'] - df['A2'])) # End of session self.logout()
def put(self, request, pk, format=None): # Try to retrieve the wflow to check for permissions self.get_object(pk) # Get the dst_df dst_df = pandas_db.load_from_db(pk) serializer = self.serializer_class(data=request.data) if not serializer.is_valid(): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) # Check that the parameters are correct how = serializer.validated_data['how'] if how == '' or how not in ['left', 'right', 'outer', 'inner']: raise APIException('how must be one of left, right, outer ' 'or inner') left_on = serializer.validated_data['left_on'] if not ops.is_unique_column(dst_df[left_on]): raise APIException('column' + left_on + 'does not contain a unique key.') # Operation has been accepted by the serializer src_df = serializer.validated_data['src_df'] right_on = serializer.validated_data['right_on'] if right_on not in list(src_df.columns): raise APIException('column ' + right_on + ' not found in data frame') if not ops.is_unique_column(src_df[right_on]): raise APIException('column' + right_on + 'does not contain a unique key.') merge_info = { 'how_merge': how, 'dst_selected_key': left_on, 'src_selected_key': right_on, 'initial_column_names': list(src_df.columns), 'rename_column_names': list(src_df.columns), 'columns_to_upload': [True] * len(list(src_df.columns)), } # Ready to perform the MERGE try: merge_result = ops.perform_dataframe_upload_merge( pk, dst_df, src_df, merge_info) except Exception: raise APIException('Unable to perform merge operation') if merge_result: # Something went wrong, raise the exception raise APIException(merge_result) # Merge went through. return Response(serializer.data, status=status.HTTP_201_CREATED)
def get(self, request, pk, format=None): # Try to retrieve the wflow to check for permissions self.get_object(pk, user=self.request.user) serializer = self.serializer_class({ 'src_df': pandas_db.load_from_db(pk), 'how': '', 'left_on': '', 'right_on': '' }) return Response(serializer.data)
def test_table_pandas_merge_to_outer_NaN(self): # Get the only workflow in the fixture workflow = Workflow.objects.all()[0] # Drop the column with booleans because the data type is lost workflow_delete_column( workflow, Column.objects.get(workflow=workflow, name='registered')) # Transform new table into string r_df = pd.DataFrame(self.src_df2) # Load the df from the db df = pandas_db.load_from_db(workflow.id) new_df = pd.merge(df, r_df, how="outer", left_on="sid", right_on="sid") # Get the data through the API response = self.client.put(reverse( 'table:api_pmerge', kwargs={'pk': workflow.id}), { "src_df": serializers.df_to_string(r_df), "how": "outer", "left_on": "sid", "right_on": "sid" }, format='json') # Get the new workflow workflow = Workflow.objects.all()[0] # Result should have three rows as the initial DF self.assertEqual(workflow.nrows, 4) self.assertEqual(workflow.ncols, 8) # Load the df from the db df = pandas_db.load_from_db(workflow.id) # Compare both elements and check wf df consistency self.compare_tables(df, new_df) # Check for df/wf consistency self.assertTrue(pandas_db.check_wf_df(workflow))
def df_update_inner(self): df_dst, df_src = self.parse_data_frames() self.merge_info['how_merge'] = 'inner' result = perform_dataframe_upload_merge(self.pk, df_dst, df_src, self.merge_info) # Result must be correct (None) self.assertEquals(result, None) result_df = pandas_db.load_from_db(self.pk)
def reposition_column_and_update_df(workflow, column, to_idx): """ :param workflow: Workflow object for which the repositioning is done :param column: column object to relocate :param to_idx: Destination index of the given column :return: Content reflected in the DB """ df = pandas_db.load_from_db(workflow.id) reposition_columns(workflow, column.position, to_idx) column.position = to_idx column.save() ops.store_dataframe_in_db(df, workflow.id)
def test_table_pandas_get(self): # Get the only workflow in the fixture workflow = Workflow.objects.all()[0] # Get the data through the API response = self.client.get( reverse('table:api_pops', kwargs={'pk': workflow.id})) # Transform the response into a data frame r_df = serializers.string_to_df(response.data['data_frame']) # Load the df from the db df = pandas_db.load_from_db(workflow.id) # Compare both elements self.compare_tables(r_df, df)
def test_table_JSON_get(self): # Get the only workflow in the fixture workflow = Workflow.objects.all()[0] # Get the data through the API response = self.client.get( reverse('table:api_ops', kwargs={'pk': workflow.id})) # Transform the response into a data frame r_df = pd.DataFrame(response.data['data_frame']) r_df = ops.detect_datetime_columns(r_df) # Load the df from the db df = pandas_db.load_from_db(workflow.id) # Compare both elements self.compare_tables(r_df, df)
def stat_column(request, pk): """ Render the page with stats and visualizations for the given column The page includes the following visualizations: First row: quartile information (only for integer and double) V1. Box plot. Only for columns of type integer and double. V2. Histogram. For columns of type integer, double, string, boolean :param request: HTTP request :param pk: primary key of the column :return: Render the page """ workflow = get_workflow(request) if not workflow: return redirect('workflow:index') # Get the column try: column = Column.objects.get(pk=pk, workflow=workflow) except ObjectDoesNotExist: return redirect('workflow:index') # Get the dataframe df = pandas_db.load_from_db(workflow.id) # Extract the data to show at the top of the page stat_data = pandas_db.get_column_stats_from_df(df[column.name]) vis_scripts = [] visualizations = get_column_visualisations( column, df[[column.name]], vis_scripts, context={ 'style': 'max-width:800px; max-height:450px;display:inline-block;' }) return render( request, 'table/stat_column.html', { 'column': column, 'stat_data': stat_data, 'vis_scripts': vis_scripts, 'visualizations': [v.html_content for v in visualizations] })
def test_tracking(self): # Repeat the checks two times to test if they are accumulating for idx in range(1, 3): # Iterate over the tracking items for trck in self.trck_tokens: self.client.get(reverse('trck') + '?v=' + trck) # Get the workflow and the data frame workflow = Workflow.objects.get(name=self.wflow_name) df = pandas_db.load_from_db(workflow.id) # Check that the results have been updated in the DB (to 1) for uemail in [ x[1] for x in test.user_info if x[1].startswith('student') ]: self.assertEqual( int(df.loc[df['email'] == uemail, 'EmailRead_1'].values[0]), idx)
def test_merge_inner(self): # Get the workflow self.workflow = Workflow.objects.all()[0] # Parse the source data frame df_dst, df_src = self.parse_data_frames() self.merge_info['how_merge'] = 'inner' result = perform_dataframe_upload_merge(self.workflow.id, df_dst, df_src, self.merge_info) # Load again the workflow data frame df_dst = load_from_db(self.workflow.id) # Result must be correct (None) self.assertEquals(result, None)
def test_table_pandas_merge_get(self): # Get the only workflow in the fixture workflow = Workflow.objects.all()[0] # Get the data through the API response = self.client.get( reverse('table:api_pmerge', kwargs={'pk': workflow.id})) # Transform new table into string r_df = serializers.string_to_df(response.data['src_df']) # Load the df from the db df = pandas_db.load_from_db(workflow.id) # Compare both elements and check wf df consistency self.compare_tables(r_df, df) workflow = Workflow.objects.all()[0] self.assertTrue(pandas_db.check_wf_df(workflow))
def forwards(apps, schema_editor): if schema_editor.connection.alias != 'default': return # Traverse the workflows and verify that the columns are in the same # order than the columns in the workflow for w in Workflow.objects.all(): if not pandas_db.is_wf_table_in_db(w): continue df = pandas_db.load_from_db(w.id) if len(df.columns) != w.ncols: raise Exception('Inconsistent number of columns') df_columns = list(df.columns) for c in w.columns.all(): c.position = df_columns.index(c.name) + 1 c.save()
def test_02_second_plugin(self): # Login self.login('*****@*****.**') # GO TO THE WORKFLOW PAGE self.access_workflow_from_home_page('Plugin test') # Open the transform page self.go_to_transform() # Click in the second plugin # Click in the first plugin element = self.search_table_row_by_string('transform-table', 1, 'test_plugin_2') element.find_element_by_link_text('Run').click() WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.NAME, 'csrfmiddlewaretoken')) ) # Provide the execution data self.selenium.find_element_by_id("id_merge_key").click() Select(self.selenium.find_element_by_id( "id_merge_key" )).select_by_visible_text("email") # Submit the execution self.selenium.find_element_by_name("Submit").click() self.wait_for_page(element_id='plugin-execution-report') # Done. Click continue. self.selenium.find_element_by_link_text('Continue').click() self.wait_for_datatable('column-table_previous') # Assert the content of the dataframe wflow = Workflow.objects.get(name='Plugin test') df = pandas_db.load_from_db(wflow.id) self.assertTrue('RESULT 3' in set(df.columns)) self.assertTrue('RESULT 4' in set(df.columns)) self.assertTrue(df['RESULT 3'].equals(df['A1'] + df['A2'])) self.assertTrue(df['RESULT 4'].equals(df['A1'] - df['A2'])) # End of session self.logout()
def parse_data_frames(self): # Parse the two CSV strings and return as data frames if self.workflow: # Get the workflow data frame df_dst = load_from_db(self.workflow.id) else: df_dst = pandas_db.load_df_from_csvfile( StringIO.StringIO(self.csv1), 0, 0) df_src = pandas_db.load_df_from_csvfile(StringIO.StringIO(self.csv2), 0, 0) # Fix the merge_info fields. self.merge_info['initial_column_names'] = list(df_src.columns) self.merge_info['rename_column_names'] = list(df_src.columns) self.merge_info['columns_to_upload'] = list(df_src.columns) return df_dst, df_src
def test_04_merge_right(self): self.template_merge('right') # Assert the content of the dataframe wflow = Workflow.objects.get(name=self.wf_name) df = pandas_db.load_from_db(wflow.id) self.assertTrue('key' in set(df.columns)) self.assertTrue('key2' in set(df.columns)) self.assertTrue('text3' in set(df.columns)) self.assertTrue('double3' in set(df.columns)) self.assertTrue('bool3' in set(df.columns)) self.assertTrue('date3' in set(df.columns)) assert pandas_db.check_wf_df( Workflow.objects.get(name='Testing Merge')) # End of session self.logout()
def clone_column(column, new_workflow=None, new_name=None): """ Function that given a column clones it and changes workflow and name :param column: Object to clone :param new_workflow: New workflow object to point :param new_name: New name :return: Cloned object """ # Store the old object name before squashing it old_name = column.name old_position = column.position # Clone column.id = None # Update some of the fields if new_name: column.name = new_name if new_workflow: column.workflow = new_workflow # Set column at the end column.position = column.workflow.ncols + 1 column.save() # Update the number of columns in the workflow column.workflow.ncols += 1 column.workflow.save() # Reposition the columns above the one being deleted reposition_columns(column.workflow, column.position, old_position + 1) # Set the new column in the right location column.position = old_position + 1 column.save() # Add the column to the table and update it. data_frame = pandas_db.load_from_db(column.workflow.id) data_frame[new_name] = data_frame[old_name] ops.store_dataframe_in_db(data_frame, column.workflow.id) return column
def test_05_merge_outer_fail(self): self.template_merge('outer') # Assert that the error is at the top of the page self.assertIn( 'Merge operation produced a result without any key columns.', self.selenium.page_source) # Assert the content of the dataframe wflow = Workflow.objects.get(name=self.wf_name) df = pandas_db.load_from_db(wflow.id) self.assertTrue('key' in set(df.columns)) self.assertTrue('key2' not in set(df.columns)) self.assertTrue('text3' not in set(df.columns)) self.assertTrue('double3' not in set(df.columns)) self.assertTrue('bool3' not in set(df.columns)) self.assertTrue('date3' not in set(df.columns)) assert pandas_db.check_wf_df( Workflow.objects.get(name='Testing Merge')) # End of session self.logout()
def workflow_restrict_column(workflow, column): """ Given a workflow and a column, modifies the column so that only the values already present are allowed for future updates. :param workflow: Workflow object :param column: Column object to restrict :return: String with error or None if correct """ # Load the data frame data_frame = pandas_db.load_from_db(column.workflow.id) cat_values = set(data_frame[column.name].dropna()) if not cat_values: # Column has no meaningful values. Nothing to do. return _('Column has no meaningful values') # Set categories column.set_categories(list(cat_values)) column.save() # Correct execution return None
def test_table_pandas_update(self): # Get the only workflow in the fixture workflow = Workflow.objects.all()[0] # Transform new table into string r_df = pd.DataFrame(self.new_table) r_df = ops.detect_datetime_columns(r_df) # Upload a new table response = self.client.put( reverse('table:api_pops', kwargs={'pk': workflow.id}), {'data_frame': serializers.df_to_string(r_df)}, format='json') # Load the df from the db df = pandas_db.load_from_db(workflow.id) # Compare both elements self.compare_tables(r_df, df) # Refresh wflow (has been updated) and check that the rest of the # information is correct workflow = Workflow.objects.get(pk=workflow.id) self.assertTrue(pandas_db.check_wf_df(workflow))
def create(self, validated_data, **kwargs): # Process first the used_columns field to get a sense of how many # columns, their type how many of them new, etc. etc. new_columns = [] for citem in validated_data['used_columns']: cname = citem.get('name', None) if not cname: raise Exception('Incorrect column name {0}.'.format(cname)) col = Column.objects.filter(workflow=self.context['workflow'], name=cname).first() if not col: # new column if citem['is_key']: raise Exception('New action cannot have non-existing key ' 'column {0}'.format(cname)) # Accummulate the new columns just in case we have to undo # the changes new_columns.append(citem) continue # existing column if col.data_type != citem.get('data_type', None) or \ col.is_key != citem['is_key'] or \ set(col.categories) != set(citem['categories']): # The two columns are different raise Exception( 'Imported column {0} is different from existing ' 'one.'.format(cname)) new_column_names = [x['name'] for x in new_columns] action_obj = None try: # used_columns has been verified. action_obj = Action( workflow=self.context['workflow'], name=validated_data['name'], description_text=validated_data['description_text'], is_out=validated_data['is_out'], serve_enabled=validated_data['serve_enabled'], active_from=validated_data['active_from'], active_to=validated_data['active_to'], content=validated_data.get('content', '')) action_obj.save() if new_columns: # There are some new columns that need to be created column_data = ColumnSerializer(data=new_columns, many=True, context=self.context) # And save its content if column_data.is_valid(): column_data.save() workflow = self.context['workflow'] df = pandas_db.load_from_db(self.context['workflow'].id) if df is None: # If there is no data frame, there is no point on # adding columns. Column.objects.filter( workflow=self.context['workflow'], name__in=new_column_names).delete() action_obj.delete() raise Exception('Action cannot be imported with and ' 'empty data table') for col in Column.objects.filter( workflow=workflow, name__in=new_column_names): # Add the column with the initial value df = ops.data_frame_add_column(df, col, None) # Update the column position col.position = len(df.columns) col.save() # Store the df to DB ops.store_dataframe_in_db(df, workflow.id) else: raise Exception('Unable to create column data') # Load the conditions pointing to the action condition_data = ConditionSerializer( data=validated_data.get('conditions', []), many=True, context={'action': action_obj}) if condition_data.is_valid(): condition_data.save() else: raise Exception('Unable to create condition information') # Update the condition variables for each formula if not present for condition in action_obj.conditions.all(): if condition.columns.all().count() == 0: col_names = get_variables(condition.formula) # Add the corresponding columns to the condition condition.columns.set( self.context['workflow'].columns.filter( name__in=col_names)) # Load the columns field columns = ColumnNameSerializer(data=validated_data['columns'], many=True, required=False, context=self.context) if columns.is_valid(): for citem in columns.data: column = action_obj.workflow.columns.get( name=citem['name']) action_obj.columns.add(column) columns.save() else: raise Exception('Unable to create columns field') except Exception: if action_obj and action_obj.id: action_obj.delete() Column.objects.filter(workflow=self.context['workflow'], name__in=new_column_names).delete() raise return action_obj
def data_frame(self): # Function used by the serializer to access the data frame in the DB if self.data_frame_table_name: return pandas_db.load_from_db(self.id) return None
def get_row_visualisations(request, view_id=None): # If there is no workflow object, go back to the index workflow = get_workflow(request) if not workflow: return redirect('workflow:index') # If the workflow has no data, something went wrong, go back to the # workflow details page if workflow.nrows == 0: return redirect('worflow:detail', workflow.id) # Get the pair key,value to fetch the row from the table update_key = request.GET.get('key', None) update_val = request.GET.get('val', None) if not update_key or not update_val: # Malformed request return render(request, 'error.html', {'message': _('Unable to visualize table row')}) # If a view is given, filter the columns columns_to_view = workflow.columns.all() column_names = None if view_id: try: view = View.objects.get(pk=view_id) except ObjectDoesNotExist: # View not found. Redirect to workflow detail return redirect('workflow:detail', workflow.id) columns_to_view = view.columns.all() column_names = [c.name for c in columns_to_view] df = pandas_db.load_from_db(workflow.id, column_names, view.formula) else: # No view given, fetch the entire data frame df = pandas_db.load_from_db(workflow.id) # Get the rows from the table row = pandas_db.execute_select_on_table(workflow.id, [update_key], [update_val], column_names)[0] vis_scripts = [] visualizations = [] idx = -1 context = {'style': 'width:400px; height:225px;'} for column in columns_to_view: idx += 1 # Skip primary keys (no point to represent any of them) if column.is_key: continue # Add the title and surrounding container visualizations.append('<h4>' + column.name + '</h4>') # If all values are empty, no need to proceed if all([not x for x in df[column.name]]): visualizations.append("<p>" + _('No values in this column') + "</p><hr/>") continue if row[idx] is None or row[idx] == '': visualizations.append( '<p class="alert-warning">' + \ _('No value for this student in this column</p>') ) visualizations.append('<div style="display: inline-flex;">') v = get_column_visualisations(column, df[[column.name]], vis_scripts=vis_scripts, id='column_{0}'.format(idx), single_val=row[idx], context=context) visualizations.extend([x.html_content for x in v]) visualizations.append('</div><hr/>') return render( request, 'table/stat_row.html', { 'value': update_val, 'vis_scripts': vis_scripts, 'visualizations': visualizations })
def get_view_visualisations(request, view_id=None): """ Function that returns the visualisations for a view (or the entire table) :param request: HTTP request :param view_id: View id being used :return: HTTP response TODO: Review this function and get_row_visualisation because there is a significant overlap. """ # If there is no workflow object, go back to the index workflow = get_workflow(request) if not workflow: return redirect('workflow:index') # If the workflow has no data, something went wrong, go back to the # workflow details page if workflow.nrows == 0: messages.error(request, _('Unable to provide visualisation without data.')) return redirect('worflow:detail', workflow.id) # If a view is given, filter the columns columns_to_view = workflow.columns.all() formula = None view = None if view_id: try: view = View.objects.get(pk=view_id) except ObjectDoesNotExist: # View not found. Redirect to workflow detail return redirect('workflow:detail', workflow.id) columns_to_view = view.columns.all() df = pandas_db.load_from_db(workflow.id, [x.name for x in columns_to_view], view.formula) else: # No view given, fetch the entire data frame df = pandas_db.load_from_db(workflow.id) vis_scripts = [] visualizations = [] idx = -1 context = {'style': 'width:400px; height:225px;'} for column in columns_to_view: idx += 1 # Skip primary keys (no point to represent any of them) if column.is_key: continue # Add the title and surrounding container visualizations.append('<h4>' + column.name + '</h4>') # If all values are empty, no need to proceed if all([not x for x in df[column.name]]): visualizations.append("<p>No values in this column</p><hr/>") continue visualizations.append('<div style="display: inline-flex;">') v = get_column_visualisations(column, df[[column.name]], vis_scripts=vis_scripts, id='column_{0}'.format(idx), context=context) visualizations.extend([x.html_content for x in v]) visualizations.append('</div><hr/>') return render(request, 'table/stat_view.html', { 'view': view, 'vis_scripts': vis_scripts, 'visualizations': visualizations })
def row_create(request): """ Receives a POST request to create a new row in the data table :param request: Request object with all the data. :return: """ # If there is no workflow object, go back to the index workflow = get_workflow(request) if not workflow: return redirect('workflow:index') # If the workflow has no data, the operation should not be allowed if workflow.nrows == 0: return redirect('dataops:list') # Create the form form = RowForm(request.POST or None, workflow=workflow) if request.method == 'GET' or not form.is_valid(): return render( request, 'dataops/row_create.html', { 'workflow': workflow, 'form': form, 'cancel_url': reverse('table:display') }) # Create the query to update the row columns = workflow.get_columns() column_names = [c.name for c in columns] field_name = field_prefix + '%s' row_vals = [ form.cleaned_data[field_name % idx] for idx in range(len(columns)) ] # Load the existing df from the db df = pandas_db.load_from_db(workflow.id) # Perform the row addition in the DF first # df2 = pd.DataFrame([[5, 6], [7, 8]], columns=list('AB')) # df.append(df2, ignore_index=True) new_row = pd.DataFrame([row_vals], columns=column_names) df = df.append(new_row, ignore_index=True) # Verify that the unique columns remain unique for ucol in [c for c in columns if c.is_key]: if not ops.is_unique_column(df[ucol.name]): form.add_error( None, 'Repeated value in column ' + ucol.name + '.' + ' It must be different to maintain Key property') return render( request, 'dataops/row_create.html', { 'workflow': workflow, 'form': form, 'cancel_url': reverse('table:display') }) # Restore the dataframe to the DB ops.store_dataframe_in_db(df, workflow.id) # Log the event log_payload = zip(column_names, [str(x) for x in row_vals]) logs.ops.put(request.user, 'tablerow_create', workflow, { 'id': workflow.id, 'name': workflow.name, 'new_values': log_payload }) # Done. Back to the table view return redirect('table:display')
def upload_s4(request): """ Step 4: Show the user the expected effect of the merge and perform it. ASSUMES: initial_column_names: List of column names in the initial file. column_types: List of column types as detected by pandas src_is_key_column: Boolean list with src columns that are unique step_1: URL name of the first step rename_column_names: Modified column names to remove ambiguity when merging. columns_to_upload: Boolean list denoting the columns in SRC that are marked for upload. dst_column_names: List of column names in destination frame dst_is_unique_column: Boolean list with dst columns that are unique dst_unique_col_names: List with the column names that are unique dst_selected_key: Key column name selected in DST src_selected_key: Key column name selected in SRC how_merge: How to merge. One of {left, right, outter, inner} :param request: Web request :return: """ # Get the workflow id we are processing workflow = get_workflow(request) if not workflow: return redirect('workflow:index') # Get the dictionary containing the information about the upload upload_data = request.session.get('upload_data', None) if not upload_data: # If there is nsendo object, someone is trying to jump directly here. return redirect('dataops:uploadmerge') # Check the type of request that is being processed if request.method == 'POST': # We are processing a POST request # Get the dataframes to merge try: dst_df = pandas_db.load_from_db(workflow.id) src_df = ops.load_upload_from_db(workflow.id) except Exception: return render(request, 'error.html', {'message': _('Exception while loading data frame')}) # Performing the merge status = ops.perform_dataframe_upload_merge(workflow.id, dst_df, src_df, upload_data) # Nuke the temporary table pandas_db.delete_upload_table(workflow.id) col_info = workflow.get_column_info() if status: Log.objects.register( request.user, Log.WORKFLOW_DATA_FAILEDMERGE, workflow, { 'id': workflow.id, 'name': workflow.name, 'num_rows': workflow.nrows, 'num_cols': workflow.ncols, 'column_names': col_info[0], 'column_types': col_info[1], 'column_unique': col_info[2], 'error_msg': status }) messages.error(request, _('Merge operation failed.') + ' (' + status + ')'), return redirect(reverse('dataops:uploadmerge')) # Log the event Log.objects.register( request.user, Log.WORKFLOW_DATA_MERGE, workflow, { 'id': workflow.id, 'name': workflow.name, 'num_rows': workflow.nrows, 'num_cols': workflow.ncols, 'column_names': col_info[0], 'column_types': col_info[1], 'column_unique': col_info[2] }) # Remove the csvupload from the session object request.session.pop('upload_data', None) return redirect(reverse('workflow:detail', kwargs={'pk': workflow.id})) # We are processing a GET request # Create the information to include in the final report table dst_column_names = upload_data['dst_column_names'] dst_selected_key = upload_data['dst_selected_key'] src_selected_key = upload_data['src_selected_key'] # List of final column names final_columns = sorted(set().union(dst_column_names, upload_data['rename_column_names'])) # Dictionary with (new src column name: (old name, is_uploaded?) src_info = { x: (y, z) for (x, y, z) in zip(upload_data['rename_column_names'], upload_data['initial_column_names'], upload_data['columns_to_upload']) } # Create the strings to show in the table for each of the rows explaining # what is going to be the effect of the update operation over them. # # There are 8 cases depending on the column name being a key column, # in DST, SRC, if SRC is being renamed, and SRC is being loaded. # # Case 1: The column is the key column used for the merge (skip it) # # Case 2: in DST, NOT in SRC: # Dst | | # # Case 3: in DST, in SRC, NOT LOADED # Dst Name | <-- | Src new name (Ignored) # # Case 4: NOT in DST, in SRC, NOT LOADED # | | Src new name (Ignored) # # Case 5: in DST, in SRC, Loaded, no rename: # Dst Name (Update) | <-- | Src name # # Case 6: in DST, in SRC, loaded, rename: # Dst Name (Update) | <-- | Src new name (Renamed) # # Case 7: NOT in DST, in SRC, loaded, no rename # Dst Name (NEW) | <-- | src name # # Case 8: NOT in DST, in SRC, loaded, renamed # Dst Name (NEW) | <-- | src name (renamed) # info = [] for colname in final_columns: # Case 1: Skip the keys if colname == src_selected_key or colname == dst_selected_key: continue # Case 2: Column is in DST and left untouched (no counter part in SRC) if colname not in src_info.keys(): info.append((colname, False, '')) continue # Get old name and if it is going to be loaded old_name, toLoad = src_info[colname] # Column is not going to be loaded anyway if not toLoad: if colname in dst_column_names: # Case 3 info.append((colname, False, colname + _(' (Ignored)'))) else: # Case 4 info.append(('', False, colname + _(' (Ignored)'))) continue # Initial name on the dst data frame dst_name = colname # Column not present in DST, so it is a new column if colname not in dst_column_names: dst_name += _(' (New)') else: dst_name += _(' (Update)') src_name = colname if colname != old_name: src_name += _(' (Renamed)') # Cases 5 - 8 info.append((dst_name, True, src_name)) # Store the value in the request object and update request.session['upload_data'] = upload_data return render( request, 'dataops/upload_s4.html', { 'prev_step': reverse('dataops:upload_s3'), 'info': info, 'next_name': 'Finish' })
def upload_s2(request): """ The four step process will populate the following dictionary with name upload_data (divided by steps in which they are set ASSUMES: initial_column_names: List of column names in the initial file. column_types: List of column types as detected by pandas src_is_key_column: Boolean list with src columns that are unique step_1: URL name of the first step CREATES: rename_column_names: Modified column names to remove ambiguity when merging. columns_to_upload: Boolean list denoting the columns in SRC that are marked for upload. keep_key_column: Boolean list with those key columns that need to be kept. :param request: Web request :return: the dictionary upload_data in the session object """ workflow = get_workflow(request) if not workflow: return redirect('workflow:index') # Get the dictionary to store information about the upload # is stored in the session. upload_data = request.session.get('upload_data', None) if not upload_data: # If there is no object, or it is an empty dict, it denotes a direct # jump to this step, get back to the dataops page return redirect('dataops:uploadmerge') # Get the column names, types, and those that are unique from the data frame try: initial_columns = upload_data.get('initial_column_names') column_types = upload_data.get('column_types') src_is_key_column = upload_data.get('src_is_key_column') except KeyError: # The page has been invoked out of order return redirect( upload_data.get('step_1', reverse('dataops:uploadmerge'))) # Get or create the list with the renamed column names rename_column_names = upload_data.get('rename_column_names', None) if rename_column_names is None: rename_column_names = initial_columns[:] upload_data['rename_column_names'] = rename_column_names # Get or create list of booleans identifying columns to be uploaded columns_to_upload = upload_data.get('columns_to_upload', None) if columns_to_upload is None: columns_to_upload = [True] * len(initial_columns) upload_data['columns_to_upload'] = columns_to_upload # Get or create list of booleans identifying key columns to be kept keep_key_column = upload_data.get('keep_key_column', None) if keep_key_column is None: keep_key_column = upload_data['src_is_key_column'][:] upload_data['keep_key_column'] = keep_key_column # Bind the form with the received data (remember unique columns) form = SelectColumnUploadForm(request.POST or None, column_names=rename_column_names, columns_to_upload=columns_to_upload, is_key=src_is_key_column, keep_key=keep_key_column) # Get a hold of the fields to create a list to be processed in the page load_fields = [f for f in form if f.name.startswith('upload_')] newname_fields = [f for f in form if f.name.startswith('new_name_')] src_key_fields = [ form['make_key_%s' % idx] if src_is_key_column[idx] else None for idx in range(len(src_is_key_column)) ] # Create one of the context elements for the form. Pack the lists so that # they can be iterated in the template df_info = [ list(i) for i in zip(load_fields, initial_columns, newname_fields, column_types, src_key_fields) ] # Process the initial loading of the form and return if request.method != 'POST' or not form.is_valid(): # Update the dictionary with the session information request.session['upload_data'] = upload_data context = { 'form': form, 'wid': workflow.id, 'prev_step': upload_data['step_1'], 'df_info': df_info } if not ops.workflow_id_has_table(workflow.id): # It is an upload, not a merge, set the next step to finish context['next_name'] = _('Finish') return render(request, 'dataops/upload_s2.html', context) # At this point we are processing a valid POST request # We need to modify upload_data with the information received in the post for i in range(len(initial_columns)): new_name = form.cleaned_data['new_name_%s' % i] upload_data['rename_column_names'][i] = new_name upload = form.cleaned_data['upload_%s' % i] upload_data['columns_to_upload'][i] = upload if src_is_key_column[i]: # If the column is key, check if the user wants to keep it keep_key_column[i] = form.cleaned_data['make_key_%s' % i] # Update the dictionary with the session information request.session['upload_data'] = upload_data # Load the existing DF or None if it doesn't exist existing_df = pandas_db.load_from_db(workflow.id) if existing_df is not None: # This is a merge operation, so move to Step 3 return redirect('dataops:upload_s3') # This is an upload operation (not a merge) save the uploaded dataframe in # the DB and finish. # Get the uploaded data_frame try: data_frame = ops.load_upload_from_db(workflow.id) except Exception: return render( request, 'error.html', {'message': _('Exception while retrieving the data frame')}) # Update the data frame status = ops.perform_dataframe_upload_merge(workflow.id, existing_df, data_frame, upload_data) if status: # Something went wrong. Flag it and reload context = { 'form': form, 'wid': workflow.id, 'prev_step': upload_data['step_1'], 'df_info': df_info } return render(request, 'dataops/upload_s2.html', context) # Nuke the temporary table pandas_db.delete_upload_table(workflow.id) # Log the event col_info = workflow.get_column_info() Log.objects.register( request.user, Log.WORKFLOW_DATA_UPLOAD, workflow, { 'id': workflow.id, 'name': workflow.name, 'num_rows': workflow.nrows, 'num_cols': workflow.ncols, 'column_names': col_info[0], 'column_types': col_info[1], 'column_unique': col_info[2] }) # Go back to show the workflow detail return redirect(reverse('workflow:detail', kwargs={'pk': workflow.id}))
def run(request, pk): """ View provided as the first step to execute a plugin. :param request: HTTP request received :param pk: primary key of the plugin :return: Page offering to select the columns to invoke """ # Get the workflow and the plugin information workflow = get_workflow(request) if not workflow: return redirect('workflow:index') try: plugin_info = PluginRegistry.objects.get(pk=pk) except PluginRegistry.DoesNotExist: return redirect('workflow:index') plugin_instance, msgs = load_plugin(plugin_info.filename) if plugin_instance is None: messages.error( request, 'Unable to instantiate plugin "{0}"'.format(plugin_info.name)) return redirect('dataops:transform') if len(plugin_instance.input_column_names) > 0: # The plug in works with a fixed set of columns cnames = workflow.columns.all().values_list('name', flat=True) if not set(plugin_instance.input_column_names) < set(cnames): # The set of columns are not part of the workflow messages.error( request, 'Workflow does not have the correct columns to run this plugin' ) return redirect('dataops:transform') # create the form to select the columns and the corresponding dictionary form = SelectColumnForm(request.POST or None, workflow=workflow, plugin_instance=plugin_instance) # Set the basic elements in the context context = { 'form': form, 'output_column_fields': [x for x in list(form) if x.name.startswith(field_prefix + 'output')], 'parameters': [ x for x in list(form) if x.name.startswith(field_prefix + 'parameter') ], 'pinstance': plugin_instance, 'id': workflow.id } # If it is a GET request or non valid, render the form. if request.method == 'GET' or not form.is_valid(): return render(request, 'dataops/plugin_info_for_run.html', context) # POST is correct proceed with execution # Get the data frame and select the appropriate columns try: dst_df = pandas_db.load_from_db(workflow.id) except Exception: messages.error(request, 'Exception while retrieving the data frame') return render(request, 'error.html', {}) # Take the list of inputs from the form if empty list is given. if not plugin_instance.input_column_names: plugin_instance.input_column_names = \ [c.name for c in form.cleaned_data['columns']] # Get the proper subset of the data frame sub_df = dst_df[[form.cleaned_data['merge_key']] + plugin_instance.input_column_names] # Process the output columns for idx, output_cname in enumerate(plugin_instance.output_column_names): new_cname = form.cleaned_data[field_prefix + 'output_%s' % idx] if form.cleaned_data['out_column_suffix']: new_cname += form.cleaned_data['out_column_suffix'] plugin_instance.output_column_names[idx] = new_cname # Pack the parameters params = dict() for idx, tpl in enumerate(plugin_instance.parameters): params[tpl[0]] = form.cleaned_data[field_prefix + 'parameter_%s' % idx] # Execute the plugin result_df, status = run_plugin(plugin_instance, sub_df, form.cleaned_data['merge_key'], params) if status is not None: context['exec_status'] = status # Log the event logs.ops.put(request.user, 'plugin_execute', workflow, { 'id': plugin_info.id, 'name': plugin_info.name, 'status': status }) return render(request, 'dataops/plugin_execution_report.html', context) # Additional checks # Result has the same number of rows if result_df.shape[0] != dst_df.shape[0]: status = 'Incorrect number of rows in result data frame.' context['exec_status'] = status # Log the event logs.ops.put(request.user, 'plugin_execute', workflow, { 'id': plugin_info.id, 'name': plugin_info.name, 'status': status }) return render(request, 'dataops/plugin_execution_report.html', context) # Result column names are consistent if set(result_df.columns) != \ set([form.cleaned_data['merge_key']] + plugin_instance.output_column_names): status = 'Incorrect columns in result data frame.' context['exec_status'] = status # Log the event logs.ops.put(request.user, 'plugin_execute', workflow, { 'id': plugin_info.id, 'name': plugin_info.name, 'status': status }) return render(request, 'dataops/plugin_execution_report.html', context) # Proceed with the merge try: result = ops.perform_dataframe_upload_merge( workflow.id, dst_df, result_df, { 'how_merge': 'left', 'dst_selected_key': form.cleaned_data['merge_key'], 'src_selected_key': form.cleaned_data['merge_key'], 'initial_column_names': list(result_df.columns), 'rename_column_names': list(result_df.columns), 'columns_to_upload': [True] * len(list(result_df.columns)) }) except Exception as e: context['exec_status'] = e.message return render(request, 'dataops/plugin_execution_report.html', context) if isinstance(result, str): # Something went wrong context['exec_status'] = result return render(request, 'dataops/plugin_execution_report.html', context) # Get the resulting dataframe final_df = pandas_db.load_from_db(workflow.id) # Update execution time plugin_info.executed = datetime.now( pytz.timezone(ontask_settings.TIME_ZONE)) plugin_info.save() # List of pairs (column name, column type) in the result to create the # log event result_columns = zip(list(result_df.columns), pandas_db.df_column_types_rename(result_df)) # Log the event logs.ops.put( request.user, 'plugin_execute', workflow, { 'id': plugin_info.id, 'name': plugin_info.name, 'status': status, 'result_columns': result_columns }) # Create the table information to show in the report. column_info = [] dst_names = list(dst_df.columns) result_names = list(result_df.columns) for c in list(final_df.columns): if c not in result_names: column_info.append((c, '')) elif c not in dst_names: column_info.append(('', c + ' (New)')) else: if c == form.cleaned_data['merge_key']: column_info.append((c, c)) else: column_info.append((c + ' (Update)', c)) context['info'] = column_info context['key'] = form.cleaned_data['merge_key'] context['id'] = workflow.id # Redirect to the notification page with the proper info return render(request, 'dataops/plugin_execution_report.html', context)
def trck(request): """ Receive a request with a token from email read tracking :param request: Request object :return: Reflects in the DB the reception and (optionally) in the data table of the workflow """ if request.method != 'GET': raise Http404 # Detected attempt to track event track_id = request.GET.get('v', None) if not track_id: raise Http404 # If the track_id is not correctly signed, out. try: track_id = signing.loads(track_id) except signing.BadSignature: raise Http404 # The request is legit and the value has been verified. Track_id has now # the dictionary with the information included in the tracking # Get the objects related to the ping try: user = get_user_model().objects.get(email=track_id['sender']) action = Action.objects.get(pk=track_id['action']) except Exception: raise Http404 # If the track comes with column_dst, the event needs to be reflected # back in the data frame column_dst = track_id.get('column_dst', '') if column_dst: # Load the dataframe data_frame = pandas_db.load_from_db(action.workflow.id) # Extract the relevant fields from the track_id column_to = track_id['column_to'] msg_to = track_id['to'] track_col_name = track_id['column_dst'] # New val in DF: df.loc[df['a']==1,'b'] = VAL data_frame.loc[data_frame[column_to] == msg_to, track_col_name] += 1 # Save DF ops.store_dataframe_in_db(data_frame, action.workflow.id) # Get the tracking column and update all the conditions in the # actions that have this column as part of their formulas track_col = action.workflow.columns.get(name=track_col_name) for action in action.workflow.actions.all(): action.update_n_rows_selected(track_col) # Record the event logs.ops.put( user, 'action_email_read', action.workflow, { 'to': track_id['to'], # The destination of the email 'email_column': track_id['column_to'], # The column used to get the 'column_dst': column_dst }) return HttpResponse(settings.PIXEL, content_type='image/png')
def test_table_create_derived_columns(self): # Login self.login('*****@*****.**') self.open(reverse('workflow:index')) # GO TO THE WORKFLOW PAGE WebDriverWait(self.selenium, 10).until(EC.title_is('OnTask :: Workflows')) self.assertIn('New Workflow', self.selenium.page_source) self.assertIn('Import Workflow', self.selenium.page_source) # Open the workflow wf_link = self.selenium.find_element_by_link_text(self.wflow_name) wf_link.click() # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D1 = c1 + c2 self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d1") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_id("id_op_type").click() Select(self.selenium.find_element_by_id( "id_op_type")).select_by_visible_text("sum: Sum selected columns") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[2]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[3]").click() self.selenium.find_element_by_css_selector( "div.sol-current-selection").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D2 = c3 * c4 self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d2") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_id("id_op_type").click() Select( self.selenium.find_element_by_id("id_op_type") ).select_by_visible_text("prod: Product of the selected columns") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[4]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[5]").click() self.selenium.find_element_by_css_selector( "div.sol-current-selection").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D3 = max(c5, c6) self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d3") self.selenium.find_element_by_id("id_op_type").click() Select(self.selenium.find_element_by_id("id_op_type") ).select_by_visible_text("max: Maximum of the selected columns") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[6]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[7]").click() self.selenium.find_element_by_css_selector("div.modal-body").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D4 = min(c7, c8) self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d4") self.selenium.find_element_by_id("id_op_type").click() Select(self.selenium.find_element_by_id("id_op_type") ).select_by_visible_text("min: Minimum of the selected columns") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[8]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[9]").click() self.selenium.find_element_by_css_selector("div.modal-body").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D5 = mean(c1, c2) self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d5") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_id("id_op_type").click() Select(self.selenium.find_element_by_id("id_op_type") ).select_by_visible_text("mean: Mean of the selected columns") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[2]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[3]").click() self.selenium.find_element_by_css_selector( "div.sol-current-selection").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D6 = median(c3, c4) self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d6") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_id("id_op_type").click() Select( self.selenium.find_element_by_id("id_op_type") ).select_by_visible_text("median: Median of the selected columns") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[4]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[5]").click() self.selenium.find_element_by_css_selector( "div.sol-current-selection").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D7 = std(c5, c6) self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d7") self.selenium.find_element_by_id("id_op_type").click() Select(self.selenium.find_element_by_id( "id_op_type")).select_by_visible_text( "std: Standard deviation over the selected columns") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[6]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[7]").click() self.selenium.find_element_by_css_selector("div.modal-body").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D8 = all(c91, c92) self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d8") self.selenium.find_element_by_id("id_op_type").click() Select(self.selenium.find_element_by_id( "id_op_type")).select_by_visible_text( "all: True when all elements in selected columns are true") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() # Wait for JS to do its thing WebDriverWait(self.selenium, 10).until( EC.element_to_be_clickable( (By.XPATH, "(//input[@name='columns'])[12]"))) self.selenium.find_element_by_xpath( "(//input[@name='columns'])[11]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[12]").click() self.selenium.find_element_by_css_selector("div.modal-body").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Click in the add derived column button self.selenium.find_element_by_css_selector( "button.btn.btn-primary.btn-sm.js-workflow-formula-column-add" ).click() # Wait for the form to create the derived column WebDriverWait(self.selenium, 10).until( EC.text_to_be_present_in_element( (By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"), 'Add derived column')) # Fill out the details for D9 = any(c91, c92) self.selenium.find_element_by_id("id_name").click() self.selenium.find_element_by_id("id_name").clear() self.selenium.find_element_by_id("id_name").send_keys("d9") self.selenium.find_element_by_id("id_op_type").click() Select(self.selenium.find_element_by_id( "id_op_type")).select_by_visible_text( "any: True when any element in selected columns is true") self.selenium.find_element_by_css_selector( "div.sol-input-container > input[type=\"text\"]").click() # Wait for JS to do its thing WebDriverWait(self.selenium, 10).until( EC.element_to_be_clickable( (By.XPATH, "(//input[@name='columns'])[12]"))) self.selenium.find_element_by_xpath( "(//input[@name='columns'])[11]").click() self.selenium.find_element_by_xpath( "(//input[@name='columns'])[12]").click() self.selenium.find_element_by_css_selector("div.modal-body").click() self.selenium.find_element_by_css_selector( "div.modal-footer > button.btn.btn-primary").click() # MODAL WAITING WebDriverWait(self.selenium, 10).until_not( EC.presence_of_element_located((By.CLASS_NAME, 'modal-open'))) # Wait for the table to be refreshed WebDriverWait(self.selenium, 10).until( EC.presence_of_element_located((By.ID, 'column-table_previous'))) # Check that the data is correct df = pandas_db.load_from_db(Workflow.objects.all()[0].id) # d1 = c1 + c2 self.assertTrue((df['d1'] == df[['c1', 'c2']].sum(axis=1)).all()) # d2 = c3 * c4 self.assertTrue((df['d2'] == df[['c3', 'c4']].prod(axis=1)).all()) # d3 = max(c5, c6) self.assertTrue((df['d3'] == df[['c5', 'c6']].max(axis=1)).all()) # d4 = min(c7, c8) self.assertTrue((df['d4'] == df[['c7', 'c8']].min(axis=1)).all()) # d5 = mean(c1, c2) self.assertTrue((df['d5'] == df[['c1', 'c2']].mean(axis=1)).all()) # d6 = median(c3, c4) self.assertTrue((df['d6'] == df[['c3', 'c4']].median(axis=1)).all()) # d7 = std(c5, c6) (error below 10^{-11}) self.assertTrue( ((df['d7'] - df[['c5', 'c6']].std(axis=1)).abs() < 0.1e-12).all()) # d8 = all(c91, c92) self.assertTrue((df['d8'] == df[['c91', 'c92']].all(axis=1)).all()) # d9 = any(c91, c92) self.assertTrue((df['d9'] == df[['c91', 'c92']].any(axis=1)).all()) # End of session self.logout()