def restrict_column(user, column: models.Column):
"""Set category of the column to the existing set of values.
Given a workflow and a column, modifies the column so that only the
values already present are allowed for future updates.
:param user: User executing this action
:param column: Column object to restrict
:return: String with error or None if correct
"""
# Load the data frame
data_frame = pandas.load_table(column.workflow.get_data_frame_table_name())
cat_values = set(data_frame[column.name].dropna())
if not cat_values:
raise services.OnTaskWorkflowNoCategoryValues(
message=_('The column has no meaningful values'))
# Set categories
column.set_categories(list(cat_values))
column.save()
# Re-evaluate the operands in the workflow
column.log(user, models.Log.COLUMN_RESTRICT)
column.workflow.set_query_builder_ops()
column.workflow.save()
def workflow_restrict_column(column: Column) -> Optional[str]:
"""Set category of the column to the existing set of values.
Given a workflow and a column, modifies the column so that only the
values already present are allowed for future updates.
:param column: Column object to restrict
:return: String with error or None if correct
"""
# Load the data frame
data_frame = load_table(column.workflow.get_data_frame_table_name())
cat_values = set(data_frame[column.name].dropna())
if not cat_values:
# Column has no meaningful values. Nothing to do.
return _('Column has no meaningful values')
# Set categories
column.set_categories(list(cat_values))
column.save()
# Re-evaluate the operands in the workflow
column.workflow.set_query_builder_ops()
column.workflow.save()
# Correct execution
return None
def do_clone_column_only(
column: Column,
new_workflow: Optional[Workflow] = None,
new_name: Optional[str] = None,
) -> Column:
"""Clone a column.
:param column: Object to clone.
:param new_workflow: Optional new worklow object to link to.
:param new_name: Optional new name to use.
:result: New object.
"""
if new_name is None:
new_name = column.name
if new_workflow is None:
new_workflow = column.workflow
new_column = Column(
name=new_name,
description_text=column.description_text,
workflow=new_workflow,
data_type=column.data_type,
is_key=column.is_key,
position=column.position,
in_viz=column.in_viz,
categories=copy.deepcopy(column.categories),
active_from=column.active_from,
active_to=column.active_to,
)
new_column.save()
return new_column
def add_column_to_workflow(
user,
workflow: models.Workflow,
column: models.Column,
column_initial_value: Any,
action_column_event: Optional[str] = None,
action: Optional[models.Action] = None,
):
"""Add a new column to the workflow.
:param user: User executing the operation
:param workflow: Used workflow
:param column: Column to insert
:param column_initial_value: Initial value for the column
:param action_column_event: Event to record if the action/column needs to
be stored
:param action: If the column is a question, this is the action in which it
is being used.
:return: Nothing. Column added to Wflow and DB
"""
# Catch the special case of integer type and no initial value. Pandas
# encodes it as NaN but a cycle through the database transforms it into
# a string. To avoid this case, integer + empty value => double
if column.data_type == 'integer' and column_initial_value is None:
column.data_type = 'double'
# Fill in the remaining fields in the column
column.workflow = workflow
column.is_key = False
# Update the positions of the appropriate columns
workflow.reposition_columns(workflow.ncols + 1, column.position)
# Save column, refresh workflow, and increase number of columns
column.save()
workflow.refresh_from_db()
workflow.ncols += 1
workflow.set_query_builder_ops()
workflow.save()
# Add the new column to the DB
try:
sql.add_column_to_db(
workflow.get_data_frame_table_name(),
column.name,
column.data_type,
initial=column_initial_value)
except Exception as exc:
raise services.OnTaskWorkflowAddColumn(
message=_('Unable to add element: {0}').format(str(exc)),
to_delete=[column])
if action_column_event:
acc, __ = models.ActionColumnConditionTuple.objects.get_or_create(
action=action,
column=column,
condition=None)
acc.log(user, action_column_event)
column.log(user, models.Log.COLUMN_ADD)
def update_column(
user,
workflow: models.Workflow,
column: models.Column,
old_name: str,
old_position: int,
action_column_item: Optional[models.Action] = None,
action_column_event: Optional[str] = None,
):
"""Update information in a column.
:param user: User performing the operation
:param workflow: Workflow being processed
:param column: Column being modified
:param old_name: Old name to detect renaming
:param old_position: old position to detect repositioning
:param action_column_item: Action/column item in case the action is stored
in that table.
:param action_column_event: Event to record the operation on the
action/column
:return: Nothing. Side effect in the workflow.
"""
# If there is a new name, rename the data frame columns
if old_name != column.name:
sql.db_rename_column(
workflow.get_data_frame_table_name(),
old_name,
column.name)
pandas.rename_df_column(workflow, old_name, column.name)
if old_position != column.position:
# Update the positions of the appropriate columns
workflow.reposition_columns(old_position, column.position)
column.save()
# Go back to the DB because the prefetch columns are not valid
# any more
workflow = models.Workflow.objects.prefetch_related('columns').get(
id=workflow.id,
)
# Changes in column require rebuilding the query_builder_ops
workflow.set_query_builder_ops()
# Save the workflow
workflow.save()
if action_column_item:
action_column_item.log(user, action_column_event)
# Log the event
column.log(user, models.Log.COLUMN_EDIT)
def add_formula_column(
user,
workflow: models.Workflow,
column: models.Column,
operation: str,
selected_columns: List[models.Column],
):
"""Add the formula column to the workflow.
:param user: User making the request
:param workflow: Workflow to add the column
:param column: Column being added
:param operation: string denoting the operation
:param selected_columns: List of columns selected for the operation.
:return: Column is added to the workflow
"""
# Save the column object attached to the form and add additional fields
column.workflow = workflow
column.is_key = False
# Save the instance
column.save()
# Update the data frame
df = pandas.load_table(workflow.get_data_frame_table_name())
# Add the column with the appropriate computation
cnames = [col.name for col in selected_columns]
df[column.name] = _op_distrib[operation](df[cnames])
# Populate the column type
column.data_type = pandas.datatype_names.get(
df[column.name].dtype.name)
# Update the positions of the appropriate columns
workflow.reposition_columns(workflow.ncols + 1, column.position)
column.save()
workflow.refresh_from_db()
# Store the df to DB
try:
pandas.store_dataframe(df, workflow)
except Exception as exc:
raise services.OnTaskWorkflowAddColumn(
message=_('Unable to add column: {0}').format(str(exc)),
to_delete=[column])
workflow.ncols = workflow.columns.count()
workflow.save()
column.log(user, models.Log.COLUMN_ADD_FORMULA)
def create(self, validated_data, **kwargs):
"""Create a new column."""
# Preliminary checks
data_type = validated_data.get('data_type')
if (data_type is None
or data_type not in list(pandas_datatype_names.values())):
# The data type is not legal
raise Exception(_('Incorrect data type {0}.').format(data_type))
column_obj = None
try:
# Create the object, but point to the given workflow
column_obj = Column(
name=validated_data['name'],
description_text=validated_data.get('description_text', ''),
workflow=self.context['workflow'],
data_type=data_type,
is_key=validated_data.get('is_key', False),
position=validated_data.get('position', 0),
in_viz=validated_data.get('in_viz', True),
active_from=validated_data.get('active_from'),
active_to=validated_data.get('active_to'),
)
# Set the categories if they exists
column_obj.set_categories(validated_data.get('categories', []),
True)
if (column_obj.active_from and column_obj.active_to
and column_obj.active_from > column_obj.active_to):
raise Exception(
_('Incorrect date/times in the active window for ' +
'column {0}').format(validated_data['name']))
# All tests passed, proceed to save the object.
column_obj.save()
except Exception as exc:
if column_obj:
column_obj.delete()
raise exc
return column_obj
def _create_track_column(action: Action) -> str:
"""Create an additional column for email tracking.
:param action: Action to consider
:return: column name
"""
# Make sure the column name does not collide with an existing one
idx = 0 # Suffix to rename
cnames = [col.name for col in action.workflow.columns.all()]
while True:
idx += 1
track_col_name = 'EmailRead_{0}'.format(idx)
if track_col_name not in cnames:
break
# Add the column if needed (before the mass email to avoid overload
# Create the new column and store
column = Column(
name=track_col_name,
description_text='Emails sent with action {0} on {1}'.format(
action.name,
simplify_datetime_str(
datetime.datetime.now(pytz.timezone(settings.TIME_ZONE))),
),
workflow=action.workflow,
data_type='integer',
is_key=False,
position=action.workflow.ncols + 1,
)
column.save()
# Increase the number of columns in the workflow
action.workflow.ncols += 1
action.workflow.save()
# Add the column to the DB table
add_column_to_db(action.workflow.get_data_frame_table_name(),
track_col_name, 'integer', 0)
return track_col_name
def add_random_column(
user,
workflow: models.Workflow,
column: models.Column,
data_frame: pd.DataFrame,
):
"""Add the formula column to the workflow.
:param user: User making the request
:param workflow: Workflow to add the column
:param column: Column being added
:param data_frame: Data frame of the current workflow
:return: Column is added to the workflow
"""
# Save the column object attached to the form and add additional fields
column.workflow = workflow
column.is_key = False
# Detect the case of a single integer as initial value so that it is
# expanded
try:
int_value = int(column.categories[0])
if int_value <= 1:
raise services.OnTaskWorkflowIntegerLowerThanOne(
field_name='values',
message=_('The integer value has to be larger than 1'))
column.set_categories([idx + 1 for idx in range(int_value)])
except (ValueError, TypeError, IndexError):
pass
column.save()
# Empty new column
new_column = [None] * workflow.nrows
# Create the random partitions
partitions = _partition([idx for idx in range(workflow.nrows)],
len(column.categories))
# Assign values to partitions
for idx, indexes in enumerate(partitions):
for col_idx in indexes:
new_column[col_idx] = column.categories[idx]
# Assign the new column to the data frame
data_frame[column.name] = new_column
# Update the positions of the appropriate columns
workflow.reposition_columns(workflow.ncols + 1, column.position)
workflow.refresh_from_db()
# Store the df to DB
try:
pandas.store_dataframe(data_frame, workflow)
except Exception as exc:
raise services.OnTaskWorkflowStoreError(
message=_('Unable to add the column: {0}').format(str(exc)),
to_delete=[column])
workflow.ncols = workflow.columns.count()
workflow.save()
# Log the event
column.log(user, models.Log.COLUMN_ADD_RANDOM)