def test_02_second_plugin(self):
# Login
self.login('*****@*****.**')
self.open(reverse('workflow:index'))
# GO TO THE WORKFLOW PAGE
WebDriverWait(self.selenium,
10).until(EC.title_is('OnTask :: Workflows'))
self.assertIn('New workflow', self.selenium.page_source)
self.assertIn('Import workflow', self.selenium.page_source)
# Open the workflow
wf_link = self.selenium.find_element_by_link_text('Plugin test')
wf_link.click()
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Open the transform page
self.selenium.find_element_by_link_text("Dataops").click()
self.selenium.find_element_by_link_text("Transform").click()
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located(
(By.ID, 'transform-table_previous')))
# Click in the second plugin
self.selenium.find_element_by_xpath(
"//table[@id='transform-table']/tbody/tr[5]/td[7]/div/a").click()
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.NAME, 'csrfmiddlewaretoken')))
# Provide the execution data
self.selenium.find_element_by_id("id_merge_key").click()
Select(self.selenium.find_element_by_id(
"id_merge_key")).select_by_visible_text("email")
# Submit the execution
self.selenium.find_element_by_name("Submit").click()
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'plugin-execution-report')))
# Done. Click continue.
self.selenium.find_element_by_xpath(
"(//button[@type='button'])[2]").click()
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Assert the content of the dataframe
wflow = Workflow.objects.get(name='Plugin test')
df = pandas_db.load_from_db(wflow.id)
self.assertTrue('RESULT 3' in set(df.columns))
self.assertTrue('RESULT 4' in set(df.columns))
self.assertTrue(df['RESULT 3'].equals(df['A1'] + df['A2']))
self.assertTrue(df['RESULT 4'].equals(df['A1'] - df['A2']))
# End of session
self.logout()
def put(self, request, pk, format=None):
# Try to retrieve the wflow to check for permissions
self.get_object(pk)
# Get the dst_df
dst_df = pandas_db.load_from_db(pk)
serializer = self.serializer_class(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
# Check that the parameters are correct
how = serializer.validated_data['how']
if how == '' or how not in ['left', 'right', 'outer', 'inner']:
raise APIException('how must be one of left, right, outer '
'or inner')
left_on = serializer.validated_data['left_on']
if not ops.is_unique_column(dst_df[left_on]):
raise APIException('column' + left_on +
'does not contain a unique key.')
# Operation has been accepted by the serializer
src_df = serializer.validated_data['src_df']
right_on = serializer.validated_data['right_on']
if right_on not in list(src_df.columns):
raise APIException('column ' + right_on +
' not found in data frame')
if not ops.is_unique_column(src_df[right_on]):
raise APIException('column' + right_on +
'does not contain a unique key.')
merge_info = {
'how_merge': how,
'dst_selected_key': left_on,
'src_selected_key': right_on,
'initial_column_names': list(src_df.columns),
'rename_column_names': list(src_df.columns),
'columns_to_upload': [True] * len(list(src_df.columns)),
}
# Ready to perform the MERGE
try:
merge_result = ops.perform_dataframe_upload_merge(
pk, dst_df, src_df, merge_info)
except Exception:
raise APIException('Unable to perform merge operation')
if merge_result:
# Something went wrong, raise the exception
raise APIException(merge_result)
# Merge went through.
return Response(serializer.data, status=status.HTTP_201_CREATED)
def get(self, request, pk, format=None):
# Try to retrieve the wflow to check for permissions
self.get_object(pk, user=self.request.user)
serializer = self.serializer_class({
'src_df': pandas_db.load_from_db(pk),
'how': '',
'left_on': '',
'right_on': ''
})
return Response(serializer.data)
def test_table_pandas_merge_to_outer_NaN(self):
# Get the only workflow in the fixture
workflow = Workflow.objects.all()[0]
# Drop the column with booleans because the data type is lost
workflow_delete_column(
workflow, Column.objects.get(workflow=workflow, name='registered'))
# Transform new table into string
r_df = pd.DataFrame(self.src_df2)
# Load the df from the db
df = pandas_db.load_from_db(workflow.id)
new_df = pd.merge(df, r_df, how="outer", left_on="sid", right_on="sid")
# Get the data through the API
response = self.client.put(reverse(
'table:api_pmerge', kwargs={'pk': workflow.id}), {
"src_df": serializers.df_to_string(r_df),
"how": "outer",
"left_on": "sid",
"right_on": "sid"
},
format='json')
# Get the new workflow
workflow = Workflow.objects.all()[0]
# Result should have three rows as the initial DF
self.assertEqual(workflow.nrows, 4)
self.assertEqual(workflow.ncols, 8)
# Load the df from the db
df = pandas_db.load_from_db(workflow.id)
# Compare both elements and check wf df consistency
self.compare_tables(df, new_df)
# Check for df/wf consistency
self.assertTrue(pandas_db.check_wf_df(workflow))
def df_update_inner(self):
df_dst, df_src = self.parse_data_frames()
self.merge_info['how_merge'] = 'inner'
result = perform_dataframe_upload_merge(self.pk, df_dst, df_src,
self.merge_info)
# Result must be correct (None)
self.assertEquals(result, None)
result_df = pandas_db.load_from_db(self.pk)
def reposition_column_and_update_df(workflow, column, to_idx):
"""
:param workflow: Workflow object for which the repositioning is done
:param column: column object to relocate
:param to_idx: Destination index of the given column
:return: Content reflected in the DB
"""
df = pandas_db.load_from_db(workflow.id)
reposition_columns(workflow, column.position, to_idx)
column.position = to_idx
column.save()
ops.store_dataframe_in_db(df, workflow.id)
def test_table_pandas_get(self):
# Get the only workflow in the fixture
workflow = Workflow.objects.all()[0]
# Get the data through the API
response = self.client.get(
reverse('table:api_pops', kwargs={'pk': workflow.id}))
# Transform the response into a data frame
r_df = serializers.string_to_df(response.data['data_frame'])
# Load the df from the db
df = pandas_db.load_from_db(workflow.id)
# Compare both elements
self.compare_tables(r_df, df)
def test_table_JSON_get(self):
# Get the only workflow in the fixture
workflow = Workflow.objects.all()[0]
# Get the data through the API
response = self.client.get(
reverse('table:api_ops', kwargs={'pk': workflow.id}))
# Transform the response into a data frame
r_df = pd.DataFrame(response.data['data_frame'])
r_df = ops.detect_datetime_columns(r_df)
# Load the df from the db
df = pandas_db.load_from_db(workflow.id)
# Compare both elements
self.compare_tables(r_df, df)
def stat_column(request, pk):
"""
Render the page with stats and visualizations for the given column
The page includes the following visualizations:
First row: quartile information (only for integer and double)
V1. Box plot. Only for columns of type integer and double.
V2. Histogram. For columns of type integer, double, string, boolean
:param request: HTTP request
:param pk: primary key of the column
:return: Render the page
"""
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
# Get the column
try:
column = Column.objects.get(pk=pk, workflow=workflow)
except ObjectDoesNotExist:
return redirect('workflow:index')
# Get the dataframe
df = pandas_db.load_from_db(workflow.id)
# Extract the data to show at the top of the page
stat_data = pandas_db.get_column_stats_from_df(df[column.name])
vis_scripts = []
visualizations = get_column_visualisations(
column,
df[[column.name]],
vis_scripts,
context={
'style': 'max-width:800px; max-height:450px;display:inline-block;'
})
return render(
request, 'table/stat_column.html', {
'column': column,
'stat_data': stat_data,
'vis_scripts': vis_scripts,
'visualizations': [v.html_content for v in visualizations]
})
def test_tracking(self):
# Repeat the checks two times to test if they are accumulating
for idx in range(1, 3):
# Iterate over the tracking items
for trck in self.trck_tokens:
self.client.get(reverse('trck') + '?v=' + trck)
# Get the workflow and the data frame
workflow = Workflow.objects.get(name=self.wflow_name)
df = pandas_db.load_from_db(workflow.id)
# Check that the results have been updated in the DB (to 1)
for uemail in [
x[1] for x in test.user_info if x[1].startswith('student')
]:
self.assertEqual(
int(df.loc[df['email'] == uemail,
'EmailRead_1'].values[0]), idx)
def test_merge_inner(self):
# Get the workflow
self.workflow = Workflow.objects.all()[0]
# Parse the source data frame
df_dst, df_src = self.parse_data_frames()
self.merge_info['how_merge'] = 'inner'
result = perform_dataframe_upload_merge(self.workflow.id, df_dst,
df_src, self.merge_info)
# Load again the workflow data frame
df_dst = load_from_db(self.workflow.id)
# Result must be correct (None)
self.assertEquals(result, None)
def test_table_pandas_merge_get(self):
# Get the only workflow in the fixture
workflow = Workflow.objects.all()[0]
# Get the data through the API
response = self.client.get(
reverse('table:api_pmerge', kwargs={'pk': workflow.id}))
# Transform new table into string
r_df = serializers.string_to_df(response.data['src_df'])
# Load the df from the db
df = pandas_db.load_from_db(workflow.id)
# Compare both elements and check wf df consistency
self.compare_tables(r_df, df)
workflow = Workflow.objects.all()[0]
self.assertTrue(pandas_db.check_wf_df(workflow))
def forwards(apps, schema_editor):
if schema_editor.connection.alias != 'default':
return
# Traverse the workflows and verify that the columns are in the same
# order than the columns in the workflow
for w in Workflow.objects.all():
if not pandas_db.is_wf_table_in_db(w):
continue
df = pandas_db.load_from_db(w.id)
if len(df.columns) != w.ncols:
raise Exception('Inconsistent number of columns')
df_columns = list(df.columns)
for c in w.columns.all():
c.position = df_columns.index(c.name) + 1
c.save()
def test_02_second_plugin(self):
# Login
self.login('*****@*****.**')
# GO TO THE WORKFLOW PAGE
self.access_workflow_from_home_page('Plugin test')
# Open the transform page
self.go_to_transform()
# Click in the second plugin
# Click in the first plugin
element = self.search_table_row_by_string('transform-table',
1,
'test_plugin_2')
element.find_element_by_link_text('Run').click()
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.NAME, 'csrfmiddlewaretoken'))
)
# Provide the execution data
self.selenium.find_element_by_id("id_merge_key").click()
Select(self.selenium.find_element_by_id(
"id_merge_key"
)).select_by_visible_text("email")
# Submit the execution
self.selenium.find_element_by_name("Submit").click()
self.wait_for_page(element_id='plugin-execution-report')
# Done. Click continue.
self.selenium.find_element_by_link_text('Continue').click()
self.wait_for_datatable('column-table_previous')
# Assert the content of the dataframe
wflow = Workflow.objects.get(name='Plugin test')
df = pandas_db.load_from_db(wflow.id)
self.assertTrue('RESULT 3' in set(df.columns))
self.assertTrue('RESULT 4' in set(df.columns))
self.assertTrue(df['RESULT 3'].equals(df['A1'] + df['A2']))
self.assertTrue(df['RESULT 4'].equals(df['A1'] - df['A2']))
# End of session
self.logout()
def parse_data_frames(self):
# Parse the two CSV strings and return as data frames
if self.workflow:
# Get the workflow data frame
df_dst = load_from_db(self.workflow.id)
else:
df_dst = pandas_db.load_df_from_csvfile(
StringIO.StringIO(self.csv1), 0, 0)
df_src = pandas_db.load_df_from_csvfile(StringIO.StringIO(self.csv2),
0, 0)
# Fix the merge_info fields.
self.merge_info['initial_column_names'] = list(df_src.columns)
self.merge_info['rename_column_names'] = list(df_src.columns)
self.merge_info['columns_to_upload'] = list(df_src.columns)
return df_dst, df_src
def test_04_merge_right(self):
self.template_merge('right')
# Assert the content of the dataframe
wflow = Workflow.objects.get(name=self.wf_name)
df = pandas_db.load_from_db(wflow.id)
self.assertTrue('key' in set(df.columns))
self.assertTrue('key2' in set(df.columns))
self.assertTrue('text3' in set(df.columns))
self.assertTrue('double3' in set(df.columns))
self.assertTrue('bool3' in set(df.columns))
self.assertTrue('date3' in set(df.columns))
assert pandas_db.check_wf_df(
Workflow.objects.get(name='Testing Merge'))
# End of session
self.logout()
def clone_column(column, new_workflow=None, new_name=None):
"""
Function that given a column clones it and changes workflow and name
:param column: Object to clone
:param new_workflow: New workflow object to point
:param new_name: New name
:return: Cloned object
"""
# Store the old object name before squashing it
old_name = column.name
old_position = column.position
# Clone
column.id = None
# Update some of the fields
if new_name:
column.name = new_name
if new_workflow:
column.workflow = new_workflow
# Set column at the end
column.position = column.workflow.ncols + 1
column.save()
# Update the number of columns in the workflow
column.workflow.ncols += 1
column.workflow.save()
# Reposition the columns above the one being deleted
reposition_columns(column.workflow, column.position, old_position + 1)
# Set the new column in the right location
column.position = old_position + 1
column.save()
# Add the column to the table and update it.
data_frame = pandas_db.load_from_db(column.workflow.id)
data_frame[new_name] = data_frame[old_name]
ops.store_dataframe_in_db(data_frame, column.workflow.id)
return column
def test_05_merge_outer_fail(self):
self.template_merge('outer')
# Assert that the error is at the top of the page
self.assertIn(
'Merge operation produced a result without any key columns.',
self.selenium.page_source)
# Assert the content of the dataframe
wflow = Workflow.objects.get(name=self.wf_name)
df = pandas_db.load_from_db(wflow.id)
self.assertTrue('key' in set(df.columns))
self.assertTrue('key2' not in set(df.columns))
self.assertTrue('text3' not in set(df.columns))
self.assertTrue('double3' not in set(df.columns))
self.assertTrue('bool3' not in set(df.columns))
self.assertTrue('date3' not in set(df.columns))
assert pandas_db.check_wf_df(
Workflow.objects.get(name='Testing Merge'))
# End of session
self.logout()
def workflow_restrict_column(workflow, column):
"""
Given a workflow and a column, modifies the column so that only the
values already present are allowed for future updates.
:param workflow: Workflow object
:param column: Column object to restrict
:return: String with error or None if correct
"""
# Load the data frame
data_frame = pandas_db.load_from_db(column.workflow.id)
cat_values = set(data_frame[column.name].dropna())
if not cat_values:
# Column has no meaningful values. Nothing to do.
return _('Column has no meaningful values')
# Set categories
column.set_categories(list(cat_values))
column.save()
# Correct execution
return None
def test_table_pandas_update(self):
# Get the only workflow in the fixture
workflow = Workflow.objects.all()[0]
# Transform new table into string
r_df = pd.DataFrame(self.new_table)
r_df = ops.detect_datetime_columns(r_df)
# Upload a new table
response = self.client.put(
reverse('table:api_pops', kwargs={'pk': workflow.id}),
{'data_frame': serializers.df_to_string(r_df)},
format='json')
# Load the df from the db
df = pandas_db.load_from_db(workflow.id)
# Compare both elements
self.compare_tables(r_df, df)
# Refresh wflow (has been updated) and check that the rest of the
# information is correct
workflow = Workflow.objects.get(pk=workflow.id)
self.assertTrue(pandas_db.check_wf_df(workflow))
def create(self, validated_data, **kwargs):
# Process first the used_columns field to get a sense of how many
# columns, their type how many of them new, etc. etc.
new_columns = []
for citem in validated_data['used_columns']:
cname = citem.get('name', None)
if not cname:
raise Exception('Incorrect column name {0}.'.format(cname))
col = Column.objects.filter(workflow=self.context['workflow'],
name=cname).first()
if not col:
# new column
if citem['is_key']:
raise Exception('New action cannot have non-existing key '
'column {0}'.format(cname))
# Accummulate the new columns just in case we have to undo
# the changes
new_columns.append(citem)
continue
# existing column
if col.data_type != citem.get('data_type', None) or \
col.is_key != citem['is_key'] or \
set(col.categories) != set(citem['categories']):
# The two columns are different
raise Exception(
'Imported column {0} is different from existing '
'one.'.format(cname))
new_column_names = [x['name'] for x in new_columns]
action_obj = None
try:
# used_columns has been verified.
action_obj = Action(
workflow=self.context['workflow'],
name=validated_data['name'],
description_text=validated_data['description_text'],
is_out=validated_data['is_out'],
serve_enabled=validated_data['serve_enabled'],
active_from=validated_data['active_from'],
active_to=validated_data['active_to'],
content=validated_data.get('content', ''))
action_obj.save()
if new_columns:
# There are some new columns that need to be created
column_data = ColumnSerializer(data=new_columns,
many=True,
context=self.context)
# And save its content
if column_data.is_valid():
column_data.save()
workflow = self.context['workflow']
df = pandas_db.load_from_db(self.context['workflow'].id)
if df is None:
# If there is no data frame, there is no point on
# adding columns.
Column.objects.filter(
workflow=self.context['workflow'],
name__in=new_column_names).delete()
action_obj.delete()
raise Exception('Action cannot be imported with and '
'empty data table')
for col in Column.objects.filter(
workflow=workflow, name__in=new_column_names):
# Add the column with the initial value
df = ops.data_frame_add_column(df, col, None)
# Update the column position
col.position = len(df.columns)
col.save()
# Store the df to DB
ops.store_dataframe_in_db(df, workflow.id)
else:
raise Exception('Unable to create column data')
# Load the conditions pointing to the action
condition_data = ConditionSerializer(
data=validated_data.get('conditions', []),
many=True,
context={'action': action_obj})
if condition_data.is_valid():
condition_data.save()
else:
raise Exception('Unable to create condition information')
# Update the condition variables for each formula if not present
for condition in action_obj.conditions.all():
if condition.columns.all().count() == 0:
col_names = get_variables(condition.formula)
# Add the corresponding columns to the condition
condition.columns.set(
self.context['workflow'].columns.filter(
name__in=col_names))
# Load the columns field
columns = ColumnNameSerializer(data=validated_data['columns'],
many=True,
required=False,
context=self.context)
if columns.is_valid():
for citem in columns.data:
column = action_obj.workflow.columns.get(
name=citem['name'])
action_obj.columns.add(column)
columns.save()
else:
raise Exception('Unable to create columns field')
except Exception:
if action_obj and action_obj.id:
action_obj.delete()
Column.objects.filter(workflow=self.context['workflow'],
name__in=new_column_names).delete()
raise
return action_obj
def data_frame(self):
# Function used by the serializer to access the data frame in the DB
if self.data_frame_table_name:
return pandas_db.load_from_db(self.id)
return None
def get_row_visualisations(request, view_id=None):
# If there is no workflow object, go back to the index
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
# If the workflow has no data, something went wrong, go back to the
# workflow details page
if workflow.nrows == 0:
return redirect('worflow:detail', workflow.id)
# Get the pair key,value to fetch the row from the table
update_key = request.GET.get('key', None)
update_val = request.GET.get('val', None)
if not update_key or not update_val:
# Malformed request
return render(request, 'error.html',
{'message': _('Unable to visualize table row')})
# If a view is given, filter the columns
columns_to_view = workflow.columns.all()
column_names = None
if view_id:
try:
view = View.objects.get(pk=view_id)
except ObjectDoesNotExist:
# View not found. Redirect to workflow detail
return redirect('workflow:detail', workflow.id)
columns_to_view = view.columns.all()
column_names = [c.name for c in columns_to_view]
df = pandas_db.load_from_db(workflow.id, column_names, view.formula)
else:
# No view given, fetch the entire data frame
df = pandas_db.load_from_db(workflow.id)
# Get the rows from the table
row = pandas_db.execute_select_on_table(workflow.id, [update_key],
[update_val], column_names)[0]
vis_scripts = []
visualizations = []
idx = -1
context = {'style': 'width:400px; height:225px;'}
for column in columns_to_view:
idx += 1
# Skip primary keys (no point to represent any of them)
if column.is_key:
continue
# Add the title and surrounding container
visualizations.append('<h4>' + column.name + '</h4>')
# If all values are empty, no need to proceed
if all([not x for x in df[column.name]]):
visualizations.append("<p>" + _('No values in this column') +
"</p><hr/>")
continue
if row[idx] is None or row[idx] == '':
visualizations.append(
'<p class="alert-warning">' + \
_('No value for this student in this column</p>')
)
visualizations.append('<div style="display: inline-flex;">')
v = get_column_visualisations(column,
df[[column.name]],
vis_scripts=vis_scripts,
id='column_{0}'.format(idx),
single_val=row[idx],
context=context)
visualizations.extend([x.html_content for x in v])
visualizations.append('</div><hr/>')
return render(
request, 'table/stat_row.html', {
'value': update_val,
'vis_scripts': vis_scripts,
'visualizations': visualizations
})
def get_view_visualisations(request, view_id=None):
"""
Function that returns the visualisations for a view (or the entire table)
:param request: HTTP request
:param view_id: View id being used
:return: HTTP response
TODO: Review this function and get_row_visualisation because there is a
significant overlap.
"""
# If there is no workflow object, go back to the index
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
# If the workflow has no data, something went wrong, go back to the
# workflow details page
if workflow.nrows == 0:
messages.error(request,
_('Unable to provide visualisation without data.'))
return redirect('worflow:detail', workflow.id)
# If a view is given, filter the columns
columns_to_view = workflow.columns.all()
formula = None
view = None
if view_id:
try:
view = View.objects.get(pk=view_id)
except ObjectDoesNotExist:
# View not found. Redirect to workflow detail
return redirect('workflow:detail', workflow.id)
columns_to_view = view.columns.all()
df = pandas_db.load_from_db(workflow.id,
[x.name for x in columns_to_view],
view.formula)
else:
# No view given, fetch the entire data frame
df = pandas_db.load_from_db(workflow.id)
vis_scripts = []
visualizations = []
idx = -1
context = {'style': 'width:400px; height:225px;'}
for column in columns_to_view:
idx += 1
# Skip primary keys (no point to represent any of them)
if column.is_key:
continue
# Add the title and surrounding container
visualizations.append('<h4>' + column.name + '</h4>')
# If all values are empty, no need to proceed
if all([not x for x in df[column.name]]):
visualizations.append("<p>No values in this column</p><hr/>")
continue
visualizations.append('<div style="display: inline-flex;">')
v = get_column_visualisations(column,
df[[column.name]],
vis_scripts=vis_scripts,
id='column_{0}'.format(idx),
context=context)
visualizations.extend([x.html_content for x in v])
visualizations.append('</div><hr/>')
return render(request, 'table/stat_view.html', {
'view': view,
'vis_scripts': vis_scripts,
'visualizations': visualizations
})
def row_create(request):
"""
Receives a POST request to create a new row in the data table
:param request: Request object with all the data.
:return:
"""
# If there is no workflow object, go back to the index
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
# If the workflow has no data, the operation should not be allowed
if workflow.nrows == 0:
return redirect('dataops:list')
# Create the form
form = RowForm(request.POST or None, workflow=workflow)
if request.method == 'GET' or not form.is_valid():
return render(
request, 'dataops/row_create.html', {
'workflow': workflow,
'form': form,
'cancel_url': reverse('table:display')
})
# Create the query to update the row
columns = workflow.get_columns()
column_names = [c.name for c in columns]
field_name = field_prefix + '%s'
row_vals = [
form.cleaned_data[field_name % idx] for idx in range(len(columns))
]
# Load the existing df from the db
df = pandas_db.load_from_db(workflow.id)
# Perform the row addition in the DF first
# df2 = pd.DataFrame([[5, 6], [7, 8]], columns=list('AB'))
# df.append(df2, ignore_index=True)
new_row = pd.DataFrame([row_vals], columns=column_names)
df = df.append(new_row, ignore_index=True)
# Verify that the unique columns remain unique
for ucol in [c for c in columns if c.is_key]:
if not ops.is_unique_column(df[ucol.name]):
form.add_error(
None, 'Repeated value in column ' + ucol.name + '.' +
' It must be different to maintain Key property')
return render(
request, 'dataops/row_create.html', {
'workflow': workflow,
'form': form,
'cancel_url': reverse('table:display')
})
# Restore the dataframe to the DB
ops.store_dataframe_in_db(df, workflow.id)
# Log the event
log_payload = zip(column_names, [str(x) for x in row_vals])
logs.ops.put(request.user, 'tablerow_create', workflow, {
'id': workflow.id,
'name': workflow.name,
'new_values': log_payload
})
# Done. Back to the table view
return redirect('table:display')
def upload_s4(request):
"""
Step 4: Show the user the expected effect of the merge and perform it.
ASSUMES:
initial_column_names: List of column names in the initial file.
column_types: List of column types as detected by pandas
src_is_key_column: Boolean list with src columns that are unique
step_1: URL name of the first step
rename_column_names: Modified column names to remove ambiguity when
merging.
columns_to_upload: Boolean list denoting the columns in SRC that are
marked for upload.
dst_column_names: List of column names in destination frame
dst_is_unique_column: Boolean list with dst columns that are unique
dst_unique_col_names: List with the column names that are unique
dst_selected_key: Key column name selected in DST
src_selected_key: Key column name selected in SRC
how_merge: How to merge. One of {left, right, outter, inner}
:param request: Web request
:return:
"""
# Get the workflow id we are processing
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
# Get the dictionary containing the information about the upload
upload_data = request.session.get('upload_data', None)
if not upload_data:
# If there is nsendo object, someone is trying to jump directly here.
return redirect('dataops:uploadmerge')
# Check the type of request that is being processed
if request.method == 'POST':
# We are processing a POST request
# Get the dataframes to merge
try:
dst_df = pandas_db.load_from_db(workflow.id)
src_df = ops.load_upload_from_db(workflow.id)
except Exception:
return render(request, 'error.html',
{'message': _('Exception while loading data frame')})
# Performing the merge
status = ops.perform_dataframe_upload_merge(workflow.id, dst_df,
src_df, upload_data)
# Nuke the temporary table
pandas_db.delete_upload_table(workflow.id)
col_info = workflow.get_column_info()
if status:
Log.objects.register(
request.user, Log.WORKFLOW_DATA_FAILEDMERGE, workflow, {
'id': workflow.id,
'name': workflow.name,
'num_rows': workflow.nrows,
'num_cols': workflow.ncols,
'column_names': col_info[0],
'column_types': col_info[1],
'column_unique': col_info[2],
'error_msg': status
})
messages.error(request,
_('Merge operation failed.') + ' (' + status + ')'),
return redirect(reverse('dataops:uploadmerge'))
# Log the event
Log.objects.register(
request.user, Log.WORKFLOW_DATA_MERGE, workflow, {
'id': workflow.id,
'name': workflow.name,
'num_rows': workflow.nrows,
'num_cols': workflow.ncols,
'column_names': col_info[0],
'column_types': col_info[1],
'column_unique': col_info[2]
})
# Remove the csvupload from the session object
request.session.pop('upload_data', None)
return redirect(reverse('workflow:detail', kwargs={'pk': workflow.id}))
# We are processing a GET request
# Create the information to include in the final report table
dst_column_names = upload_data['dst_column_names']
dst_selected_key = upload_data['dst_selected_key']
src_selected_key = upload_data['src_selected_key']
# List of final column names
final_columns = sorted(set().union(dst_column_names,
upload_data['rename_column_names']))
# Dictionary with (new src column name: (old name, is_uploaded?)
src_info = {
x: (y, z)
for (x, y, z) in zip(upload_data['rename_column_names'],
upload_data['initial_column_names'],
upload_data['columns_to_upload'])
}
# Create the strings to show in the table for each of the rows explaining
# what is going to be the effect of the update operation over them.
#
# There are 8 cases depending on the column name being a key column,
# in DST, SRC, if SRC is being renamed, and SRC is being loaded.
#
# Case 1: The column is the key column used for the merge (skip it)
#
# Case 2: in DST, NOT in SRC:
# Dst | |
#
# Case 3: in DST, in SRC, NOT LOADED
# Dst Name | <-- | Src new name (Ignored)
#
# Case 4: NOT in DST, in SRC, NOT LOADED
# | | Src new name (Ignored)
#
# Case 5: in DST, in SRC, Loaded, no rename:
# Dst Name (Update) | <-- | Src name
#
# Case 6: in DST, in SRC, loaded, rename:
# Dst Name (Update) | <-- | Src new name (Renamed)
#
# Case 7: NOT in DST, in SRC, loaded, no rename
# Dst Name (NEW) | <-- | src name
#
# Case 8: NOT in DST, in SRC, loaded, renamed
# Dst Name (NEW) | <-- | src name (renamed)
#
info = []
for colname in final_columns:
# Case 1: Skip the keys
if colname == src_selected_key or colname == dst_selected_key:
continue
# Case 2: Column is in DST and left untouched (no counter part in SRC)
if colname not in src_info.keys():
info.append((colname, False, ''))
continue
# Get old name and if it is going to be loaded
old_name, toLoad = src_info[colname]
# Column is not going to be loaded anyway
if not toLoad:
if colname in dst_column_names:
# Case 3
info.append((colname, False, colname + _(' (Ignored)')))
else:
# Case 4
info.append(('', False, colname + _(' (Ignored)')))
continue
# Initial name on the dst data frame
dst_name = colname
# Column not present in DST, so it is a new column
if colname not in dst_column_names:
dst_name += _(' (New)')
else:
dst_name += _(' (Update)')
src_name = colname
if colname != old_name:
src_name += _(' (Renamed)')
# Cases 5 - 8
info.append((dst_name, True, src_name))
# Store the value in the request object and update
request.session['upload_data'] = upload_data
return render(
request, 'dataops/upload_s4.html', {
'prev_step': reverse('dataops:upload_s3'),
'info': info,
'next_name': 'Finish'
})
def upload_s2(request):
"""
The four step process will populate the following dictionary with name
upload_data (divided by steps in which they are set
ASSUMES:
initial_column_names: List of column names in the initial file.
column_types: List of column types as detected by pandas
src_is_key_column: Boolean list with src columns that are unique
step_1: URL name of the first step
CREATES:
rename_column_names: Modified column names to remove ambiguity when
merging.
columns_to_upload: Boolean list denoting the columns in SRC that are
marked for upload.
keep_key_column: Boolean list with those key columns that need to be kept.
:param request: Web request
:return: the dictionary upload_data in the session object
"""
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
# Get the dictionary to store information about the upload
# is stored in the session.
upload_data = request.session.get('upload_data', None)
if not upload_data:
# If there is no object, or it is an empty dict, it denotes a direct
# jump to this step, get back to the dataops page
return redirect('dataops:uploadmerge')
# Get the column names, types, and those that are unique from the data frame
try:
initial_columns = upload_data.get('initial_column_names')
column_types = upload_data.get('column_types')
src_is_key_column = upload_data.get('src_is_key_column')
except KeyError:
# The page has been invoked out of order
return redirect(
upload_data.get('step_1', reverse('dataops:uploadmerge')))
# Get or create the list with the renamed column names
rename_column_names = upload_data.get('rename_column_names', None)
if rename_column_names is None:
rename_column_names = initial_columns[:]
upload_data['rename_column_names'] = rename_column_names
# Get or create list of booleans identifying columns to be uploaded
columns_to_upload = upload_data.get('columns_to_upload', None)
if columns_to_upload is None:
columns_to_upload = [True] * len(initial_columns)
upload_data['columns_to_upload'] = columns_to_upload
# Get or create list of booleans identifying key columns to be kept
keep_key_column = upload_data.get('keep_key_column', None)
if keep_key_column is None:
keep_key_column = upload_data['src_is_key_column'][:]
upload_data['keep_key_column'] = keep_key_column
# Bind the form with the received data (remember unique columns)
form = SelectColumnUploadForm(request.POST or None,
column_names=rename_column_names,
columns_to_upload=columns_to_upload,
is_key=src_is_key_column,
keep_key=keep_key_column)
# Get a hold of the fields to create a list to be processed in the page
load_fields = [f for f in form if f.name.startswith('upload_')]
newname_fields = [f for f in form if f.name.startswith('new_name_')]
src_key_fields = [
form['make_key_%s' % idx] if src_is_key_column[idx] else None
for idx in range(len(src_is_key_column))
]
# Create one of the context elements for the form. Pack the lists so that
# they can be iterated in the template
df_info = [
list(i) for i in zip(load_fields, initial_columns, newname_fields,
column_types, src_key_fields)
]
# Process the initial loading of the form and return
if request.method != 'POST' or not form.is_valid():
# Update the dictionary with the session information
request.session['upload_data'] = upload_data
context = {
'form': form,
'wid': workflow.id,
'prev_step': upload_data['step_1'],
'df_info': df_info
}
if not ops.workflow_id_has_table(workflow.id):
# It is an upload, not a merge, set the next step to finish
context['next_name'] = _('Finish')
return render(request, 'dataops/upload_s2.html', context)
# At this point we are processing a valid POST request
# We need to modify upload_data with the information received in the post
for i in range(len(initial_columns)):
new_name = form.cleaned_data['new_name_%s' % i]
upload_data['rename_column_names'][i] = new_name
upload = form.cleaned_data['upload_%s' % i]
upload_data['columns_to_upload'][i] = upload
if src_is_key_column[i]:
# If the column is key, check if the user wants to keep it
keep_key_column[i] = form.cleaned_data['make_key_%s' % i]
# Update the dictionary with the session information
request.session['upload_data'] = upload_data
# Load the existing DF or None if it doesn't exist
existing_df = pandas_db.load_from_db(workflow.id)
if existing_df is not None:
# This is a merge operation, so move to Step 3
return redirect('dataops:upload_s3')
# This is an upload operation (not a merge) save the uploaded dataframe in
# the DB and finish.
# Get the uploaded data_frame
try:
data_frame = ops.load_upload_from_db(workflow.id)
except Exception:
return render(
request, 'error.html',
{'message': _('Exception while retrieving the data frame')})
# Update the data frame
status = ops.perform_dataframe_upload_merge(workflow.id, existing_df,
data_frame, upload_data)
if status:
# Something went wrong. Flag it and reload
context = {
'form': form,
'wid': workflow.id,
'prev_step': upload_data['step_1'],
'df_info': df_info
}
return render(request, 'dataops/upload_s2.html', context)
# Nuke the temporary table
pandas_db.delete_upload_table(workflow.id)
# Log the event
col_info = workflow.get_column_info()
Log.objects.register(
request.user, Log.WORKFLOW_DATA_UPLOAD, workflow, {
'id': workflow.id,
'name': workflow.name,
'num_rows': workflow.nrows,
'num_cols': workflow.ncols,
'column_names': col_info[0],
'column_types': col_info[1],
'column_unique': col_info[2]
})
# Go back to show the workflow detail
return redirect(reverse('workflow:detail', kwargs={'pk': workflow.id}))
def run(request, pk):
"""
View provided as the first step to execute a plugin.
:param request: HTTP request received
:param pk: primary key of the plugin
:return: Page offering to select the columns to invoke
"""
# Get the workflow and the plugin information
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
try:
plugin_info = PluginRegistry.objects.get(pk=pk)
except PluginRegistry.DoesNotExist:
return redirect('workflow:index')
plugin_instance, msgs = load_plugin(plugin_info.filename)
if plugin_instance is None:
messages.error(
request,
'Unable to instantiate plugin "{0}"'.format(plugin_info.name))
return redirect('dataops:transform')
if len(plugin_instance.input_column_names) > 0:
# The plug in works with a fixed set of columns
cnames = workflow.columns.all().values_list('name', flat=True)
if not set(plugin_instance.input_column_names) < set(cnames):
# The set of columns are not part of the workflow
messages.error(
request,
'Workflow does not have the correct columns to run this plugin'
)
return redirect('dataops:transform')
# create the form to select the columns and the corresponding dictionary
form = SelectColumnForm(request.POST or None,
workflow=workflow,
plugin_instance=plugin_instance)
# Set the basic elements in the context
context = {
'form':
form,
'output_column_fields':
[x for x in list(form) if x.name.startswith(field_prefix + 'output')],
'parameters': [
x for x in list(form)
if x.name.startswith(field_prefix + 'parameter')
],
'pinstance':
plugin_instance,
'id':
workflow.id
}
# If it is a GET request or non valid, render the form.
if request.method == 'GET' or not form.is_valid():
return render(request, 'dataops/plugin_info_for_run.html', context)
# POST is correct proceed with execution
# Get the data frame and select the appropriate columns
try:
dst_df = pandas_db.load_from_db(workflow.id)
except Exception:
messages.error(request, 'Exception while retrieving the data frame')
return render(request, 'error.html', {})
# Take the list of inputs from the form if empty list is given.
if not plugin_instance.input_column_names:
plugin_instance.input_column_names = \
[c.name for c in form.cleaned_data['columns']]
# Get the proper subset of the data frame
sub_df = dst_df[[form.cleaned_data['merge_key']] +
plugin_instance.input_column_names]
# Process the output columns
for idx, output_cname in enumerate(plugin_instance.output_column_names):
new_cname = form.cleaned_data[field_prefix + 'output_%s' % idx]
if form.cleaned_data['out_column_suffix']:
new_cname += form.cleaned_data['out_column_suffix']
plugin_instance.output_column_names[idx] = new_cname
# Pack the parameters
params = dict()
for idx, tpl in enumerate(plugin_instance.parameters):
params[tpl[0]] = form.cleaned_data[field_prefix + 'parameter_%s' % idx]
# Execute the plugin
result_df, status = run_plugin(plugin_instance, sub_df,
form.cleaned_data['merge_key'], params)
if status is not None:
context['exec_status'] = status
# Log the event
logs.ops.put(request.user, 'plugin_execute', workflow, {
'id': plugin_info.id,
'name': plugin_info.name,
'status': status
})
return render(request, 'dataops/plugin_execution_report.html', context)
# Additional checks
# Result has the same number of rows
if result_df.shape[0] != dst_df.shape[0]:
status = 'Incorrect number of rows in result data frame.'
context['exec_status'] = status
# Log the event
logs.ops.put(request.user, 'plugin_execute', workflow, {
'id': plugin_info.id,
'name': plugin_info.name,
'status': status
})
return render(request, 'dataops/plugin_execution_report.html', context)
# Result column names are consistent
if set(result_df.columns) != \
set([form.cleaned_data['merge_key']] +
plugin_instance.output_column_names):
status = 'Incorrect columns in result data frame.'
context['exec_status'] = status
# Log the event
logs.ops.put(request.user, 'plugin_execute', workflow, {
'id': plugin_info.id,
'name': plugin_info.name,
'status': status
})
return render(request, 'dataops/plugin_execution_report.html', context)
# Proceed with the merge
try:
result = ops.perform_dataframe_upload_merge(
workflow.id, dst_df, result_df, {
'how_merge': 'left',
'dst_selected_key': form.cleaned_data['merge_key'],
'src_selected_key': form.cleaned_data['merge_key'],
'initial_column_names': list(result_df.columns),
'rename_column_names': list(result_df.columns),
'columns_to_upload': [True] * len(list(result_df.columns))
})
except Exception as e:
context['exec_status'] = e.message
return render(request, 'dataops/plugin_execution_report.html', context)
if isinstance(result, str):
# Something went wrong
context['exec_status'] = result
return render(request, 'dataops/plugin_execution_report.html', context)
# Get the resulting dataframe
final_df = pandas_db.load_from_db(workflow.id)
# Update execution time
plugin_info.executed = datetime.now(
pytz.timezone(ontask_settings.TIME_ZONE))
plugin_info.save()
# List of pairs (column name, column type) in the result to create the
# log event
result_columns = zip(list(result_df.columns),
pandas_db.df_column_types_rename(result_df))
# Log the event
logs.ops.put(
request.user, 'plugin_execute', workflow, {
'id': plugin_info.id,
'name': plugin_info.name,
'status': status,
'result_columns': result_columns
})
# Create the table information to show in the report.
column_info = []
dst_names = list(dst_df.columns)
result_names = list(result_df.columns)
for c in list(final_df.columns):
if c not in result_names:
column_info.append((c, ''))
elif c not in dst_names:
column_info.append(('', c + ' (New)'))
else:
if c == form.cleaned_data['merge_key']:
column_info.append((c, c))
else:
column_info.append((c + ' (Update)', c))
context['info'] = column_info
context['key'] = form.cleaned_data['merge_key']
context['id'] = workflow.id
# Redirect to the notification page with the proper info
return render(request, 'dataops/plugin_execution_report.html', context)
def trck(request):
"""
Receive a request with a token from email read tracking
:param request: Request object
:return: Reflects in the DB the reception and (optionally) in the data
table of the workflow
"""
if request.method != 'GET':
raise Http404
# Detected attempt to track event
track_id = request.GET.get('v', None)
if not track_id:
raise Http404
# If the track_id is not correctly signed, out.
try:
track_id = signing.loads(track_id)
except signing.BadSignature:
raise Http404
# The request is legit and the value has been verified. Track_id has now
# the dictionary with the information included in the tracking
# Get the objects related to the ping
try:
user = get_user_model().objects.get(email=track_id['sender'])
action = Action.objects.get(pk=track_id['action'])
except Exception:
raise Http404
# If the track comes with column_dst, the event needs to be reflected
# back in the data frame
column_dst = track_id.get('column_dst', '')
if column_dst:
# Load the dataframe
data_frame = pandas_db.load_from_db(action.workflow.id)
# Extract the relevant fields from the track_id
column_to = track_id['column_to']
msg_to = track_id['to']
track_col_name = track_id['column_dst']
# New val in DF: df.loc[df['a']==1,'b'] = VAL
data_frame.loc[data_frame[column_to] == msg_to, track_col_name] += 1
# Save DF
ops.store_dataframe_in_db(data_frame, action.workflow.id)
# Get the tracking column and update all the conditions in the
# actions that have this column as part of their formulas
track_col = action.workflow.columns.get(name=track_col_name)
for action in action.workflow.actions.all():
action.update_n_rows_selected(track_col)
# Record the event
logs.ops.put(
user,
'action_email_read',
action.workflow,
{
'to': track_id['to'], # The destination of the email
'email_column':
track_id['column_to'], # The column used to get the
'column_dst': column_dst
})
return HttpResponse(settings.PIXEL, content_type='image/png')
def test_table_create_derived_columns(self):
# Login
self.login('*****@*****.**')
self.open(reverse('workflow:index'))
# GO TO THE WORKFLOW PAGE
WebDriverWait(self.selenium,
10).until(EC.title_is('OnTask :: Workflows'))
self.assertIn('New Workflow', self.selenium.page_source)
self.assertIn('Import Workflow', self.selenium.page_source)
# Open the workflow
wf_link = self.selenium.find_element_by_link_text(self.wflow_name)
wf_link.click()
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D1 = c1 + c2
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d1")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_id("id_op_type").click()
Select(self.selenium.find_element_by_id(
"id_op_type")).select_by_visible_text("sum: Sum selected columns")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[2]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[3]").click()
self.selenium.find_element_by_css_selector(
"div.sol-current-selection").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D2 = c3 * c4
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d2")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_id("id_op_type").click()
Select(
self.selenium.find_element_by_id("id_op_type")
).select_by_visible_text("prod: Product of the selected columns")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[4]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[5]").click()
self.selenium.find_element_by_css_selector(
"div.sol-current-selection").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D3 = max(c5, c6)
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d3")
self.selenium.find_element_by_id("id_op_type").click()
Select(self.selenium.find_element_by_id("id_op_type")
).select_by_visible_text("max: Maximum of the selected columns")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[6]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[7]").click()
self.selenium.find_element_by_css_selector("div.modal-body").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D4 = min(c7, c8)
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d4")
self.selenium.find_element_by_id("id_op_type").click()
Select(self.selenium.find_element_by_id("id_op_type")
).select_by_visible_text("min: Minimum of the selected columns")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[8]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[9]").click()
self.selenium.find_element_by_css_selector("div.modal-body").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D5 = mean(c1, c2)
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d5")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_id("id_op_type").click()
Select(self.selenium.find_element_by_id("id_op_type")
).select_by_visible_text("mean: Mean of the selected columns")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[2]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[3]").click()
self.selenium.find_element_by_css_selector(
"div.sol-current-selection").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D6 = median(c3, c4)
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d6")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_id("id_op_type").click()
Select(
self.selenium.find_element_by_id("id_op_type")
).select_by_visible_text("median: Median of the selected columns")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[4]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[5]").click()
self.selenium.find_element_by_css_selector(
"div.sol-current-selection").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D7 = std(c5, c6)
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d7")
self.selenium.find_element_by_id("id_op_type").click()
Select(self.selenium.find_element_by_id(
"id_op_type")).select_by_visible_text(
"std: Standard deviation over the selected columns")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[6]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[7]").click()
self.selenium.find_element_by_css_selector("div.modal-body").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D8 = all(c91, c92)
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d8")
self.selenium.find_element_by_id("id_op_type").click()
Select(self.selenium.find_element_by_id(
"id_op_type")).select_by_visible_text(
"all: True when all elements in selected columns are true")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
# Wait for JS to do its thing
WebDriverWait(self.selenium, 10).until(
EC.element_to_be_clickable(
(By.XPATH, "(//input[@name='columns'])[12]")))
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[11]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[12]").click()
self.selenium.find_element_by_css_selector("div.modal-body").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Click in the add derived column button
self.selenium.find_element_by_css_selector(
"button.btn.btn-primary.btn-sm.js-workflow-formula-column-add"
).click()
# Wait for the form to create the derived column
WebDriverWait(self.selenium, 10).until(
EC.text_to_be_present_in_element(
(By.XPATH, "//div[@id='modal-item']/div/div/form/div/h4"),
'Add derived column'))
# Fill out the details for D9 = any(c91, c92)
self.selenium.find_element_by_id("id_name").click()
self.selenium.find_element_by_id("id_name").clear()
self.selenium.find_element_by_id("id_name").send_keys("d9")
self.selenium.find_element_by_id("id_op_type").click()
Select(self.selenium.find_element_by_id(
"id_op_type")).select_by_visible_text(
"any: True when any element in selected columns is true")
self.selenium.find_element_by_css_selector(
"div.sol-input-container > input[type=\"text\"]").click()
# Wait for JS to do its thing
WebDriverWait(self.selenium, 10).until(
EC.element_to_be_clickable(
(By.XPATH, "(//input[@name='columns'])[12]")))
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[11]").click()
self.selenium.find_element_by_xpath(
"(//input[@name='columns'])[12]").click()
self.selenium.find_element_by_css_selector("div.modal-body").click()
self.selenium.find_element_by_css_selector(
"div.modal-footer > button.btn.btn-primary").click()
# MODAL WAITING
WebDriverWait(self.selenium, 10).until_not(
EC.presence_of_element_located((By.CLASS_NAME, 'modal-open')))
# Wait for the table to be refreshed
WebDriverWait(self.selenium, 10).until(
EC.presence_of_element_located((By.ID, 'column-table_previous')))
# Check that the data is correct
df = pandas_db.load_from_db(Workflow.objects.all()[0].id)
# d1 = c1 + c2
self.assertTrue((df['d1'] == df[['c1', 'c2']].sum(axis=1)).all())
# d2 = c3 * c4
self.assertTrue((df['d2'] == df[['c3', 'c4']].prod(axis=1)).all())
# d3 = max(c5, c6)
self.assertTrue((df['d3'] == df[['c5', 'c6']].max(axis=1)).all())
# d4 = min(c7, c8)
self.assertTrue((df['d4'] == df[['c7', 'c8']].min(axis=1)).all())
# d5 = mean(c1, c2)
self.assertTrue((df['d5'] == df[['c1', 'c2']].mean(axis=1)).all())
# d6 = median(c3, c4)
self.assertTrue((df['d6'] == df[['c3', 'c4']].median(axis=1)).all())
# d7 = std(c5, c6) (error below 10^{-11})
self.assertTrue(
((df['d7'] - df[['c5', 'c6']].std(axis=1)).abs() < 0.1e-12).all())
# d8 = all(c91, c92)
self.assertTrue((df['d8'] == df[['c91', 'c92']].all(axis=1)).all())
# d9 = any(c91, c92)
self.assertTrue((df['d9'] == df[['c91', 'c92']].any(axis=1)).all())
# End of session
self.logout()