def override(self, request, pk, format=None):
"""
Method to override the content in the workflow
:param request: Received request object
:param pk: Workflow ID
:param format: format for the response
:return:
"""
# Try to retrieve the wflow to check for permissions
wflow = self.get_object(pk)
serializer = self.serializer_class(data=request.data)
if not serializer.is_valid():
# Flag the error
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
# Data received is a correct data frame.
df = serializer.validated_data['data_frame']
# Store the content in the db and...
ops.store_dataframe_in_db(df, pk)
# Update all the counters in the conditions
for action in wflow.actions.all():
action.update_n_rows_selected()
return Response(None, status=status.HTTP_201_CREATED)
def clone_column(column, new_workflow=None, new_name=None):
"""
Function that given a column clones it and changes workflow and name
:param column: Object to clone
:param new_workflow: New workflow object to point
:param new_name: New name
:return: Cloned object
"""
# Store the old object name before squashing it
old_id = column.id
old_name = column.name
# Clone
column.id = None
# Update some of the fields
if new_name:
column.name = new_name
if new_workflow:
column.workflow = new_workflow
# Update
column.save()
# Add the column to the table and update it.
data_frame = pandas_db.load_from_db(column.workflow.id)
data_frame[new_name] = data_frame[old_name]
ops.store_dataframe_in_db(data_frame, column.workflow.id)
return column
def override(self, request, pk, format=None):
# Try to retrieve the wflow to check for permissions
self.get_object(pk, user=self.request.user)
serializer = self.serializer_class(data=request.data)
if serializer.is_valid():
df = serializer.validated_data['data_frame']
ops.store_dataframe_in_db(df, pk)
return Response(None, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def reposition_column_and_update_df(workflow, column, to_idx):
"""
:param workflow: Workflow object for which the repositioning is done
:param column: column object to relocate
:param to_idx: Destination index of the given column
:return: Content reflected in the DB
"""
df = pandas_db.load_from_db(workflow.id)
workflow.reposition_columns(column.position, to_idx)
column.position = to_idx
column.save()
ops.store_dataframe_in_db(df, workflow.id)
def clone_column(column, new_workflow=None, new_name=None):
"""
Function that given a column clones it and changes workflow and name
:param column: Object to clone
:param new_workflow: New workflow object to point
:param new_name: New name
:return: Cloned object
"""
# Store the old object name before squashing it
old_name = column.name
old_position = column.position
# Clone
column.id = None
# Update some of the fields
if new_name:
column.name = new_name
if new_workflow:
column.workflow = new_workflow
# Set column at the end
column.position = column.workflow.ncols + 1
column.save()
# Update the number of columns in the workflow
column.workflow.ncols += 1
column.workflow.save()
# Reposition the columns above the one being deleted
column.workflow.reposition_columns(column.position, old_position + 1)
# Add the column to the table and update it.
data_frame = pandas_db.load_from_db(column.workflow.id)
data_frame[new_name] = data_frame[old_name]
ops.store_dataframe_in_db(data_frame, column.workflow.id)
return column
def create(self, validated_data, **kwargs):
# Process first the used_columns field to get a sense of how many
# columns, their type how many of them new, etc. etc.
new_columns = []
for citem in validated_data['used_columns']:
cname = citem.get('name', None)
if not cname:
raise Exception('Incorrect column name {0}.'.format(cname))
col = Column.objects.filter(workflow=self.context['workflow'],
name=cname).first()
if not col:
# new column
if citem['is_key']:
raise Exception('New action cannot have non-existing key '
'column {0}'.format(cname))
# Accummulate the new columns just in case we have to undo
# the changes
new_columns.append(citem)
continue
# existing column
if col.data_type != citem.get('data_type', None) or \
col.is_key != citem['is_key'] or \
set(col.categories) != set(citem['categories']):
# The two columns are different
raise Exception(
'Imported column {0} is different from existing '
'one.'.format(cname))
new_column_names = [x['name'] for x in new_columns]
action_obj = None
try:
# used_columns has been verified.
action_obj = Action(
workflow=self.context['workflow'],
name=validated_data['name'],
description_text=validated_data['description_text'],
is_out=validated_data['is_out'],
serve_enabled=validated_data['serve_enabled'],
active_from=validated_data['active_from'],
active_to=validated_data['active_to'],
content=validated_data.get('content', ''))
action_obj.save()
if new_columns:
# There are some new columns that need to be created
column_data = ColumnSerializer(data=new_columns,
many=True,
context=self.context)
# And save its content
if column_data.is_valid():
column_data.save()
workflow = self.context['workflow']
df = pandas_db.load_from_db(self.context['workflow'].id)
if df is None:
# If there is no data frame, there is no point on
# adding columns.
Column.objects.filter(
workflow=self.context['workflow'],
name__in=new_column_names).delete()
action_obj.delete()
raise Exception('Action cannot be imported with and '
'empty data table')
for col in Column.objects.filter(
workflow=workflow, name__in=new_column_names):
# Add the column with the initial value
df = ops.data_frame_add_column(df, col, None)
# Update the column position
col.position = len(df.columns)
col.save()
# Store the df to DB
ops.store_dataframe_in_db(df, workflow.id)
else:
raise Exception('Unable to create column data')
# Load the conditions pointing to the action
condition_data = ConditionSerializer(
data=validated_data.get('conditions', []),
many=True,
context={'action': action_obj})
if condition_data.is_valid():
condition_data.save()
else:
raise Exception('Unable to create condition information')
# Update the condition variables for each formula if not present
for condition in action_obj.conditions.all():
if condition.columns.all().count() == 0:
col_names = get_variables(condition.formula)
# Add the corresponding columns to the condition
condition.columns.set(
self.context['workflow'].columns.filter(
name__in=col_names))
# Load the columns field
columns = ColumnNameSerializer(data=validated_data['columns'],
many=True,
required=False,
context=self.context)
if columns.is_valid():
for citem in columns.data:
column = action_obj.workflow.columns.get(
name=citem['name'])
action_obj.columns.add(column)
columns.save()
else:
raise Exception('Unable to create columns field')
except Exception:
if action_obj and action_obj.id:
action_obj.delete()
Column.objects.filter(workflow=self.context['workflow'],
name__in=new_column_names).delete()
raise
return action_obj
def create(self, validated_data, **kwargs):
# Initial values
workflow_obj = None
try:
workflow_obj = Workflow(
user=self.context['user'],
name=self.context['name'],
description_text=validated_data['description_text'],
nrows=0,
ncols=0,
attributes=validated_data['attributes'],
query_builder_ops=validated_data.get('query_builder_ops', {}))
workflow_obj.save()
# Create the columns
column_data = ColumnSerializer(data=validated_data.get(
'columns', []),
many=True,
context={'workflow': workflow_obj})
# And save its content
if column_data.is_valid():
column_data.save()
else:
raise Exception('Unable to save column information')
# If there is any column with position = 0, recompute (this is to
# guarantee backward compatibility.
if workflow_obj.columns.filter(position=0).exists():
for idx, c in enumerate(workflow_obj.columns.all()):
c.position = idx + 1
c.save()
# Load the data frame
data_frame = validated_data.get('data_frame', None)
if data_frame is not None:
ops.store_dataframe_in_db(data_frame, workflow_obj.id)
# Reconcile now the information in workflow and columns with the
# one loaded
workflow_obj.data_frame_table_name = \
pandas_db.create_table_name(workflow_obj.pk)
workflow_obj.ncols = validated_data['ncols']
workflow_obj.nrows = validated_data['nrows']
workflow_obj.save()
# Create the actions pointing to the workflow
action_data = ActionSerializer(data=validated_data.get(
'actions', []),
many=True,
context={'workflow': workflow_obj})
if action_data.is_valid():
action_data.save()
else:
raise Exception('Unable to save column information')
# Create the views pointing to the workflow
view_data = ViewSerializer(data=validated_data.get('views', []),
many=True,
context={'workflow': workflow_obj})
if view_data.is_valid():
view_data.save()
else:
raise Exception('Unable to save column information')
except Exception:
# Get rid of the objects created
if workflow_obj:
if workflow_obj.has_data_frame():
pandas_db.delete_table(workflow_obj.id)
if workflow_obj.id:
workflow_obj.delete()
raise
return workflow_obj
def create(self, validated_data, **kwargs):
workflow_obj = Workflow(
user=self.context['user'],
name=self.context['name'],
description_text=validated_data['description_text'],
nrows=0,
ncols=0,
attributes=validated_data['attributes'],
query_builder_ops=validated_data.get('query_builder_ops', {})
)
workflow_obj.save()
# Create the columns
column_data = ColumnSerializer(
data=validated_data.get('columns', []),
many=True,
context={'workflow': workflow_obj})
# And save its content
if column_data.is_valid():
column_data.save()
else:
workflow_obj.delete()
return None
# Create the actions pointing to the workflow
action_data = ActionSerializer(
data=validated_data.get('actions', []),
many=True,
context={'workflow': workflow_obj}
)
if action_data.is_valid():
action_data.save()
else:
workflow_obj.delete()
return None
# Create the views pointing to the workflow
view_data = ViewSerializer(
data=validated_data.get('views', []),
many=True,
context={'workflow': workflow_obj}
)
if view_data.is_valid():
view_data.save()
else:
workflow_obj.delete()
return None
# Load the data frame
data_frame = validated_data.get('data_frame', None)
if data_frame is not None:
ops.store_dataframe_in_db(data_frame, workflow_obj.id)
# Reconcile now the information in workflow and columns with the
# one loaded
workflow_obj.data_frame_table_name = \
pandas_db.create_table_name(workflow_obj.pk)
workflow_obj.ncols = validated_data['ncols']
workflow_obj.nrows = validated_data['nrows']
workflow_obj.save()
return workflow_obj
def row_create(request):
"""
Receives a POST request to create a new row in the data table
:param request: Request object with all the data.
:return:
"""
# If there is no workflow object, go back to the index
workflow = get_workflow(request)
if not workflow:
return redirect('workflow:index')
# If the workflow has no data, the operation should not be allowed
if workflow.nrows == 0:
return redirect('dataops:uploadmerge')
# Create the form
form = RowForm(request.POST or None, workflow=workflow)
if request.method == 'GET' or not form.is_valid():
return render(
request, 'dataops/row_create.html', {
'workflow': workflow,
'form': form,
'cancel_url': reverse('table:display')
})
# Create the query to update the row
columns = workflow.get_columns()
column_names = [c.name for c in columns]
field_name = field_prefix + '%s'
row_vals = [
form.cleaned_data[field_name % idx] for idx in range(len(columns))
]
# Load the existing df from the db
df = pandas_db.load_from_db(workflow.id)
# Perform the row addition in the DF first
# df2 = pd.DataFrame([[5, 6], [7, 8]], columns=list('AB'))
# df.append(df2, ignore_index=True)
new_row = pd.DataFrame([row_vals], columns=column_names)
df = df.append(new_row, ignore_index=True)
# Verify that the unique columns remain unique
for ucol in [c for c in columns if c.is_key]:
if not ops.is_unique_column(df[ucol.name]):
form.add_error(
None, 'Repeated value in column ' + ucol.name + '.' +
' It must be different to maintain Key property')
return render(
request, 'dataops/row_create.html', {
'workflow': workflow,
'form': form,
'cancel_url': reverse('table:display')
})
# Restore the dataframe to the DB
ops.store_dataframe_in_db(df, workflow.id)
# Recompute all the values of the conditions in each of the actions
for act in workflow.actions.all():
act.update_n_rows_selected()
# Log the event
log_payload = zip(column_names, [str(x) for x in row_vals])
logs.ops.put(request.user, 'tablerow_create', workflow, {
'id': workflow.id,
'name': workflow.name,
'new_values': log_payload
})
# Done. Back to the table view
return redirect('table:display')
def clone(request, pk):
"""
AJAX view to clone a workflow
:param request: HTTP request
:param pk: Workflow id
:return: JSON data
"""
# JSON response
data = dict()
# Get the current workflow
workflow = get_workflow(request, pk)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
# Initial data in the context
data['form_is_valid'] = False
context = {'pk': pk, 'name': workflow.name}
if request.method == 'GET':
data['html_form'] = render_to_string(
'workflow/includes/partial_workflow_clone.html',
context,
request=request)
return JsonResponse(data)
# POST REQUEST
# Get the new name appending as many times as needed the 'Copy of '
new_name = 'Copy of ' + workflow.name
while Workflow.objects.filter(name=new_name).exists():
new_name = 'Copy of ' + new_name
workflow.id = None
workflow.name = new_name
try:
workflow.save()
except IntegrityError:
data['form_is_valid'] = True
data['html_redirect'] = ''
messages.error(request, _('Unable to clone workflow'))
return JsonResponse(data)
# Get the initial object back
workflow_new = workflow
workflow = get_workflow(request, pk)
# Clone the data frame
data_frame = pandas_db.load_from_db(workflow.pk)
ops.store_dataframe_in_db(data_frame, workflow_new.id)
# Clone actions
action.ops.clone_actions([a for a in workflow.actions.all()], workflow_new)
# Done!
workflow_new.save()
# Log event
Log.objects.register(
request.user, Log.WORKFLOW_CLONE, workflow_new, {
'id_old': workflow_new.id,
'id_new': workflow.id,
'name_old': workflow_new.name,
'name_new': workflow.name
})
messages.success(request, _('Workflow successfully cloned.'))
data['form_is_valid'] = True
data['html_redirect'] = "" # Reload page
return JsonResponse(data)
def send_messages(user, action, subject, email_column, from_email,
cc_email_list, bcc_email_list, send_confirmation, track_read,
exclude_values, log_item):
"""
Sends the emails for the given action and with the
given subject. The subject will be evaluated also with respect to the
rows, attributes, and conditions.
The messages are sent in bursts with a pause in seconds as specified by the
configuration variables EMAIL_BURST and EMAIL_BURST_PAUSE
:param user: User object that executed the action
:param action: Action from where to take the messages
:param subject: Email subject
:param email_column: Name of the column from which to extract emails
:param from_email: Email of the sender
:param cc_email_list: List of emails to include in the CC
:param bcc_email_list: List of emails to include in the BCC
:param send_confirmation: Boolean to send confirmation to sender
:param track_read: Should read tracking be included?
:param exclude_values: List of values to exclude from the mailing
:param log_item: Log object to store results
:return: Send the emails
"""
# Evaluate the action string, evaluate the subject, and get the value of
# the email column.
result = evaluate_action(action,
extra_string=subject,
column_name=email_column,
exclude_values=exclude_values)
# Check the type of the result to see if it was successful
if not isinstance(result, list):
# Something went wrong. The result contains a message
return result
track_col_name = ''
data_frame = None
if track_read:
data_frame = pandas_db.load_from_db(action.workflow.id)
# Make sure the column name does not collide with an existing one
i = 0 # Suffix to rename
while True:
i += 1
track_col_name = 'EmailRead_{0}'.format(i)
if track_col_name not in data_frame.columns:
break
# Get the log item payload to store the tracking column
log_item.payload['track_column'] = track_col_name
log_item.save()
# Update the number of filtered rows if the action has a filter (table
# might have changed)
cfilter = action.get_filter()
if cfilter and cfilter.n_rows_selected != len(result):
cfilter.n_rows_selected = len(result)
cfilter.save()
# Set the cc_email_list and bcc_email_list to the right values
if not cc_email_list:
cc_email_list = []
if not bcc_email_list:
bcc_email_list = []
# Check that cc and bcc contain list of valid email addresses
if not all([validate_email(x) for x in cc_email_list]):
return _('Invalid email address in cc email')
if not all([validate_email(x) for x in bcc_email_list]):
return _('Invalid email address in bcc email')
# Everything seemed to work to create the messages.
msgs = []
track_ids = []
for msg_body, msg_subject, msg_to in result:
# If read tracking is on, add suffix for message (or empty)
if track_read:
# The track id must identify: action & user
track_id = {
'action': action.id,
'sender': user.email,
'to': msg_to,
'column_to': email_column,
'column_dst': track_col_name
}
track_str = \
"""<img src="https://{0}{1}{2}?v={3}" alt=""
style="position:absolute; visibility:hidden"/>""".format(
Site.objects.get_current().domain,
ontask_settings.BASE_URL,
reverse('trck'),
signing.dumps(track_id)
)
else:
track_str = ''
# Get the plain text content and bundle it together with the HTML in
# a message to be added to the list.
text_content = html2text.html2text(msg_body)
msg = EmailMultiAlternatives(msg_subject,
text_content,
from_email, [msg_to],
bcc=bcc_email_list,
cc=cc_email_list)
msg.attach_alternative(msg_body + track_str, "text/html")
msgs.append(msg)
track_ids.append(track_str)
# Add the column if needed (before the mass email to avoid overload
if track_read:
# Create the new column and store
column = Column(
name=track_col_name,
description_text='Emails sent with action {0} on {1}'.format(
action.name, str(timezone.now())),
workflow=action.workflow,
data_type='integer',
is_key=False,
position=action.workflow.ncols + 1)
column.save()
# Increase the number of columns in the workflow
action.workflow.ncols += 1
action.workflow.save()
# Initial value in the data frame and store the table
data_frame[track_col_name] = 0
ops.store_dataframe_in_db(data_frame, action.workflow.id)
# Partition the list of emails into chunks as per the value of EMAIL_BURST
chunk_size = len(msgs)
wait_time = 0
if ontask_settings.EMAIL_BURST:
chunk_size = ontask_settings.EMAIL_BURST
wait_time = ontask_settings.EMAIL_BURST_PAUSE
msg_chunks = [
msgs[i:i + chunk_size] for i in range(0, len(msgs), chunk_size)
]
for idx, msg_chunk in enumerate(msg_chunks):
# Mass mail!
try:
connection = mail.get_connection()
connection.send_messages(msg_chunk)
except Exception as e:
# Something went wrong, notify above
return str(e)
if idx != len(msg_chunks) - 1:
logger.info(
'Email Burst ({0}) reached. Waiting for {1} secs'.format(
len(msg_chunk), wait_time))
sleep(wait_time)
# Log the events (one per email)
now = datetime.datetime.now(pytz.timezone(ontask_settings.TIME_ZONE))
context = {
'user': user.id,
'action': action.id,
'email_sent_datetime': str(now),
}
for msg, track_id in zip(msgs, track_ids):
context['subject'] = msg.subject
context['body'] = msg.body
context['from_email'] = msg.from_email
context['to_email'] = msg.to[0]
if track_id:
context['track_id'] = track_id
Log.objects.register(user, Log.ACTION_EMAIL_SENT, action.workflow,
context)
# Update data in the log item
log_item.payload['objects_sent'] = len(result)
log_item.payload['filter_present'] = cfilter is not None
log_item.payload['datetime'] = str(
datetime.datetime.now(pytz.timezone(ontask_settings.TIME_ZONE)))
log_item.save()
# If no confirmation email is required, done
if not send_confirmation:
return None
# Creating the context for the confirmation email
context = {
'user': user,
'action': action,
'num_messages': len(msgs),
'email_sent_datetime': now,
'filter_present': cfilter is not None,
'num_rows': action.workflow.nrows,
'num_selected': cfilter.n_rows_selected if cfilter else -1
}
# Create template and render with context
try:
html_content = Template(str(getattr(settings,
'NOTIFICATION_TEMPLATE'))).render(
Context(context))
text_content = strip_tags(html_content)
except TemplateSyntaxError as e:
return _('Syntax error detected in OnTask notification template '
'({0})').format(e)
# Log the event
Log.objects.register(
user, Log.ACTION_EMAIL_NOTIFY, action.workflow, {
'user': user.id,
'action': action.id,
'num_messages': len(msgs),
'email_sent_datetime': str(now),
'filter_present': cfilter is not None,
'num_rows': action.workflow.nrows,
'subject': str(getattr(settings, 'NOTIFICATION_SUBJECT')),
'body': text_content,
'from_email': str(getattr(settings, 'NOTIFICATION_SENDER')),
'to_email': [user.email]
})
# Send email out
try:
send_mail(str(getattr(settings, 'NOTIFICATION_SUBJECT')),
text_content,
str(getattr(settings, 'NOTIFICATION_SENDER')), [user.email],
html_message=html_content)
except Exception as e:
return _('An error occurred when sending your notification: '
'{0}').format(e)
return None
def column_add(request, pk=None):
# TODO: Encapsulate operations in a function so that is available for the
# API
# Data to send as JSON response
data = {}
# Detect if this operation is to add a new column or a new question (in
# the edit in page)
is_question = pk is not None
# Get the workflow element
workflow = get_workflow(request)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
if workflow.nrows == 0:
data['form_is_valid'] = True
data['html_redirect'] = ''
if is_question:
messages.error(request,
_('Cannot add question to a workflow without data'))
else:
messages.error(request,
_('Cannot add column to a workflow without data'))
return JsonResponse(data)
action_id = None
if pk:
# Get the action and the columns
try:
action = Action.objects.filter(
Q(workflow__user=request.user)
| Q(workflow__shared=request.user)).distinct().get(pk=pk)
except ObjectDoesNotExist:
messages.error(request, _('Cannot find action to add question.'))
return JsonResponse({'html_redirect': reverse('action:index')})
action_id = action.id
# Form to read/process data
if is_question:
form = QuestionAddForm(request.POST or None, workflow=workflow)
else:
form = ColumnAddForm(request.POST or None, workflow=workflow)
# If a GET or incorrect request, render the form again
if request.method == 'GET' or not form.is_valid():
if is_question:
template = 'workflow/includes/partial_question_addedit.html'
else:
template = 'workflow/includes/partial_column_addedit.html'
data['html_form'] = render_to_string(template, {
'form': form,
'is_question': is_question,
'action_id': action_id,
'add': True
},
request=request)
return JsonResponse(data)
# Processing now a valid POST request
# Access the updated information
column_initial_value = form.initial_valid_value
# Save the column object attached to the form
column = form.save(commit=False)
# Fill in the remaining fields in the column
column.workflow = workflow
column.is_key = False
# Update the data frame, which must be stored in the form because
# it was loaded when validating it.
df = pandas_db.load_from_db(workflow.id)
# Add the column with the initial value to the dataframe
df = ops.data_frame_add_column(df, column, column_initial_value)
# Update the column type with the value extracted from the data frame
column.data_type = \
pandas_db.pandas_datatype_names[df[column.name].dtype.name]
# Update the positions of the appropriate columns
workflow.reposition_columns(workflow.ncols + 1, column.position)
column.save()
# Store the df to DB
ops.store_dataframe_in_db(df, workflow.id)
# If the column is a question, add it to the action
if is_question:
action.columns.add(column)
# Log the event
if is_question:
event_type = Log.QUESTION_ADD
else:
event_type = Log.COLUMN_ADD
Log.objects.register(
request.user, event_type, workflow, {
'id': workflow.id,
'name': workflow.name,
'column_name': column.name,
'column_type': column.data_type
})
data['form_is_valid'] = True
data['html_redirect'] = ''
return JsonResponse(data)
def column_edit(request, pk):
# TODO: Encapsulate operations in a function so that is available for the
# API
# Data to send as JSON response
data = {}
# Detect if this operation is to edit a new column or a new question (in
# the edit in page)
is_question = 'question_edit' in request.path_info
# Get the workflow element
workflow = get_workflow(request)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
# Get the column object and make sure it belongs to the workflow
try:
column = Column.objects.get(pk=pk, workflow=workflow)
except ObjectDoesNotExist:
# Something went wrong, redirect to the workflow detail page
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:detail',
kwargs={'pk': workflow.id})
return JsonResponse(data)
# Form to read/process data
if is_question:
form = QuestionRenameForm(request.POST or None,
workflow=workflow,
instance=column)
else:
form = ColumnRenameForm(request.POST or None,
workflow=workflow,
instance=column)
old_name = column.name
# Keep a copy of the previous position
old_position = column.position
context = {'form': form, 'cname': old_name, 'pk': pk}
if request.method == 'GET' or not form.is_valid():
if is_question:
template = 'workflow/includes/partial_question_addedit.html'
else:
template = 'workflow/includes/partial_column_addedit.html'
data['html_form'] = render_to_string(template,
context,
request=request)
return JsonResponse(data)
# Processing a POST request with valid data in the form
# Process further only if any data changed.
if form.changed_data:
# Some field changed value, so save the result, but
# no commit as we need to propagate the info to the df
column = form.save(commit=False)
# Get the data frame from the form (should be
# loaded)
df = form.data_frame
# If there is a new name, rename the data frame columns
if 'name' in form.changed_data:
# Rename the column in the data frame
df = ops.rename_df_column(df, workflow, old_name, column.name)
if 'position' in form.changed_data:
# Update the positions of the appropriate columns
workflow.reposition_columns(old_position, column.position)
# Save the column information
form.save()
# Changes in column require rebuilding the query_builder_ops
workflow.set_query_builder_ops()
# Save the workflow
workflow.save()
# And save the DF in the DB
ops.store_dataframe_in_db(df, workflow.id)
data['form_is_valid'] = True
data['html_redirect'] = ''
# Log the event
if is_question:
event_type = Log.QUESTION_ADD
else:
event_type = Log.COLUMN_ADD
Log.objects.register(
request.user, event_type, workflow, {
'id': workflow.id,
'name': workflow.name,
'column_name': old_name,
'new_name': column.name
})
# Done processing the correct POST request
return JsonResponse(data)
def random_column_add(request):
"""
Function that creates a column with random values (Modal)
:param request:
:return:
"""
# TODO: Encapsulate operations in a function so that is available for the
# API
# Data to send as JSON response, in principle, assume form is not valid
data = {'form_is_valid': False}
# Get the workflow element
workflow = get_workflow(request)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
if workflow.nrows == 0:
data['form_is_valid'] = True
data['html_redirect'] = ''
messages.error(request,
_('Cannot add column to a workflow without data'))
return JsonResponse(data)
# Form to read/process data
form = RandomColumnAddForm(data=request.POST or None)
# If a GET or incorrect request, render the form again
if request.method == 'GET' or not form.is_valid():
data['html_form'] = render_to_string(
'workflow/includes/partial_random_column_add.html', {'form': form},
request=request)
return JsonResponse(data)
# Processing now a valid POST request
# Save the column object attached to the form and add additional fields
column = form.save(commit=False)
column.workflow = workflow
column.is_key = False
# Save the instance
try:
column = form.save()
form.save_m2m()
except IntegrityError:
form.add_error('name', _('A column with that name already exists'))
data['html_form'] = render_to_string(
'workflow/includes/partial_random_column_add.html', {'form': form},
request=request)
return JsonResponse(data)
# Update the data frame
df = pandas_db.load_from_db(workflow.id)
# Get the values and interpret its meaning
values = form.cleaned_data['values']
int_value = None
# First, try to see if the field is a valid integer
try:
int_value = int(values)
except ValueError:
pass
if int_value:
# At this point the field is an integer
if int_value <= 1:
form.add_error('values',
_('The integer value has to be larger than 1'))
data['html_form'] = render_to_string(
'workflow/includes/partial_random_column_add.html',
{'form': form},
request=request)
return JsonResponse(data)
df[column.name] = [
random.randint(1, int_value) for __ in range(workflow.nrows)
]
else:
# At this point the field is a string and the values are the comma
# separated strings.
vals = [x.strip() for x in values.strip().split(',') if x]
if not vals:
form.add_error('values',
_('The value has to be a comma-separated list'))
data['html_form'] = render_to_string(
'workflow/includes/partial_random_column_add.html',
{'form': form},
request=request)
return JsonResponse(data)
df[column.name] = [random.choice(vals) for __ in range(workflow.nrows)]
# Populate the column type
column.data_type = \
pandas_db.pandas_datatype_names[df[column.name].dtype.name]
# Update the positions of the appropriate columns
workflow.reposition_columns(workflow.ncols + 1, column.position)
column.save()
# Store the df to DB
ops.store_dataframe_in_db(df, workflow.id)
# Log the event
Log.objects.register(
request.user, Log.COLUMN_ADD_RANDOM, workflow, {
'id': workflow.id,
'name': workflow.name,
'column_name': column.name,
'column_type': column.data_type,
'value': values
})
# The form has been successfully processed
data['form_is_valid'] = True
data['html_redirect'] = '' # Refresh the page
return JsonResponse(data)
def formula_column_add(request):
# TODO: Encapsulate operations in a function so that is available for the
# API
# Data to send as JSON response, in principle, assume form is not valid
data = {'form_is_valid': False}
# Get the workflow element
workflow = get_workflow(request)
if not workflow:
data['form_is_valid'] = True
data['html_redirect'] = reverse('workflow:index')
return JsonResponse(data)
if workflow.nrows == 0:
data['form_is_valid'] = True
data['html_redirect'] = ''
messages.error(request,
_('Cannot add column to a workflow without data'))
return JsonResponse(data)
# Form to read/process data
form = FormulaColumnAddForm(data=request.POST or None,
operands=formula_column_operands,
columns=workflow.columns.all())
# If a GET or incorrect request, render the form again
if request.method == 'GET' or not form.is_valid():
data['html_form'] = render_to_string(
'workflow/includes/partial_formula_column_add.html',
{'form': form},
request=request)
return JsonResponse(data)
# Processing now a valid POST request
# Save the column object attached to the form and add additional fields
column = form.save(commit=False)
column.workflow = workflow
column.is_key = False
# Save the instance
try:
column = form.save()
form.save_m2m()
except IntegrityError:
form.add_error('name', _('A column with that name already exists'))
data['html_form'] = render_to_string(
'workflow/includes/partial_formula_column_add.html',
{'form': form},
request=request)
return JsonResponse(data)
# Update the data frame
df = pandas_db.load_from_db(workflow.id)
try:
# Add the column with the appropriate computation
operation = form.cleaned_data['op_type']
cnames = [c.name for c in form.selected_columns]
if operation == 'sum':
df[column.name] = df[cnames].sum(axis=1)
elif operation == 'prod':
df[column.name] = df[cnames].prod(axis=1)
elif operation == 'max':
df[column.name] = df[cnames].max(axis=1)
elif operation == 'min':
df[column.name] = df[cnames].min(axis=1)
elif operation == 'mean':
df[column.name] = df[cnames].mean(axis=1)
elif operation == 'median':
df[column.name] = df[cnames].median(axis=1)
elif operation == 'std':
df[column.name] = df[cnames].std(axis=1)
elif operation == 'all':
df[column.name] = df[cnames].all(axis=1)
elif operation == 'any':
df[column.name] = df[cnames].any(axis=1)
else:
raise Exception(_('Operand {0} not implemented').format(operation))
except Exception as e:
# Something went wrong in pandas, we need to remove the column
column.delete()
# Notify in the form
form.add_error(
None,
_('Unable to perform the requested operation ({0})').format(
e.message))
data['html_form'] = render_to_string(
'workflow/includes/partial_formula_column_add.html',
{'form': form},
request=request)
return JsonResponse(data)
# Populate the column type
column.data_type = \
pandas_db.pandas_datatype_names[df[column.name].dtype.name]
# Update the positions of the appropriate columns
workflow.reposition_columns(workflow.ncols + 1, column.position)
column.save()
# Store the df to DB
ops.store_dataframe_in_db(df, workflow.id)
# Log the event
Log.objects.register(
request.user, Log.COLUMN_ADD_FORMULA, workflow, {
'id': workflow.id,
'name': workflow.name,
'column_name': column.name,
'column_type': column.data_type
})
# The form has been successfully processed
data['form_is_valid'] = True
data['html_redirect'] = '' # Refresh the page
return JsonResponse(data)
