def clean(self) -> Dict:
"""Store the target_url if not part of the form"""
form_data = super().clean()
# Move data to the payload so that is ready to be used
target_url = self.get_payload_field('target_url', None)
if not target_url:
self.store_field_in_dict(
'target_url', next(iter(settings.CANVAS_INFO_DICT.keys())))
if not self.get_payload_field('target_url'):
self.add_error(None,
_('No Canvas Service available for this action.'))
# The given column for email destination has to have integers or
# floats that can be transformed into integers
user_ids = sql.get_rows(
self.action.workflow.get_data_frame_table_name(),
column_names=[form_data['item_column'].name],
filter_formula=self.action.get_filter_formula())
if any(not (
isinstance(row_item[0], int) or float.is_integer(row_item[0]))
for row_item in user_ids):
self.add_error('item_column',
_('The column does not contain valid Canvas IDs.'))
return form_data
def test_eval_conditions(self):
# Get the action first
self.action = Action.objects.get(name=self.action_name)
# Get wflow table, filter and column names
wflow_table = self.action.workflow.get_data_frame_table_name()
filter_formula = self.action.get_filter_formula()
column_names = self.action.workflow.get_column_names()
conditions = self.action.conditions.filter(is_filter=False)
# Get dataframe
df = get_subframe(wflow_table, filter_formula, column_names)
# Get the query set
qs = get_rows(wflow_table,
column_names=column_names,
filter_formula=filter_formula)
# Iterate over the rows in the dataframe and compare
for idx, row in enumerate(qs):
row_value_df = dict(list(zip(column_names, df.loc[idx, :])))
row_value_qs = dict(list(zip(column_names, row)))
cond_eval1 = [
evaluate_formula(x.formula, EVAL_EXP, row_value_df)
for x in conditions
]
cond_eval2 = [
evaluate_formula(x.formula, EVAL_EXP, row_value_qs)
for x in conditions
]
assert cond_eval1 == cond_eval2
def clean(self) -> Dict:
"""Verify email values."""
form_data = super().clean()
# Move data to the payload so that is ready to be used
self.store_fields_in_dict([('send_confirmation', None),
('track_read', None)])
# Check if the values in the item_column are correct emails
pcolumn = form_data['item_column']
try:
column_data = sql.get_rows(
self.action.workflow.get_data_frame_table_name(),
column_names=[pcolumn.name])
incorrect_email = get_incorrect_email(
[iname[0] for iname in column_data])
if incorrect_email:
# column has incorrect email addresses
self.add_error(
'item_column',
_('Incorrect email address "{0}".').format(
incorrect_email))
except TypeError:
self.add_error(
'item_column',
_('The column with email addresses has incorrect values.'))
return form_data
def update_luser_email_column(
user,
pk: int,
workflow: models.Workflow,
column: models.Column,
):
"""Update the field luser_email in the workflow.
:param user: User making the request
:param pk: Column ID to obtain the user id
:param workflow: Workflow being manipulated.
:param column: Column being used to update the luser field.
:return:
"""
if not pk:
# Empty pk, means reset the field.
workflow.luser_email_column = None
workflow.luser_email_column_md5 = ''
workflow.lusers.set([])
workflow.save(
update_fields=['luser_email_column', 'luser_email_column_md5'])
return
table_name = workflow.get_data_frame_table_name()
# Get the column content
emails = sql.get_rows(table_name, column_names=[column.name])
# Verify that the column as a valid set of emails
incorrect_email = get_incorrect_email([row[column.name] for row in emails])
if incorrect_email:
raise services.OnTaskWorkflowEmailError(message=_(
'Incorrect email addresses "{0}".').format(incorrect_email))
# Update the column
workflow.luser_email_column = column
workflow.save(update_fields=['luser_email_column'])
# Calculate the MD5 value
md5_hash = sql.get_text_column_hash(table_name, column.name)
if workflow.luser_email_column_md5 == md5_hash:
return
# Change detected, run the update in batch mode
workflow.luser_email_column_md5 = md5_hash
workflow.save(update_fields=['luser_email_column_md5'])
# Log the event with the status "preparing updating"
log_item = workflow.log(user, models.Log.WORKFLOW_UPDATE_LUSERS)
# Push the update of lusers to batch processing
tasks.execute_operation.delay(
operation_type=models.Log.WORKFLOW_UPDATE_LUSERS,
user_id=user.id,
workflow_id=workflow.id,
log_id=log_item.id)
def evaluate_action(
action: models.Action,
extra_string: str = None,
column_name: str = None,
exclude_values: List[str] = None,
) -> List[List]:
"""Evaluate the content in an action based on the values in the columns.
Given an action object and an optional string:
1) Access the attached workflow
2) Obtain the data from the appropriate data frame
3) Loop over each data row and
3.1) Evaluate the conditions with respect to the values in the row
3.2) Create a context with the result of evaluating the conditions,
attributes and column names to values
3.3) Run the template with the context
3.4) Run the optional string argument with the template and the context
3.5) Select the optional column_name
4) Return the resulting objects:
List of (HTMLs body, extra string, column name value)
or an error message
:param action: Action object with pointers to conditions, filter,
workflow, etc.
:param extra_string: An extra string to process (something like the email
subject line) with the same dictionary as the text in the action.
:param column_name: Column from where to extract the special value (
typically the email address) and include it in the result.
:param exclude_values: List of values in the column to exclude
:return: list of lists resulting from the evaluation of the action. Each
element in the list contains the HTML body, the extra string (if
provided) and the column value.
"""
# Get the table data
rows = sql.get_rows(action.workflow.get_data_frame_table_name(),
filter_formula=action.get_filter_formula())
list_of_renders = []
for row in rows:
if exclude_values and str(row[column_name]) in exclude_values:
# Skip the row with the col_name in exclude values
continue
# Step 4: Create the context with the attributes, the evaluation of the
# conditions and the values of the columns.
context = get_action_evaluation_context(action, row)
# Append result
list_of_renders.append(
_render_tuple_result(action, context, extra_string, column_name), )
if settings.DEBUG:
# Check that n_rows_selected is equal to rows.rowcount
action_filter = action.get_filter()
if action_filter and action_filter.n_rows_selected != rows.rowcount:
raise ontask.OnTaskException('Inconsistent n_rows_selected')
return list_of_renders
def __init__(self, form_data, *args, **kwargs):
"""Store action, column name and exclude init, adjust fields."""
self.column_name: str = kwargs.pop('column_name', None)
self.exclude_init: List[str] = kwargs.pop('exclude_values', list)
super().__init__(form_data, *args, **kwargs)
self.fields['exclude_values'].choices = sql.get_rows(
self.action.workflow.get_data_frame_table_name(),
column_names=[self.column_name, self.column_name],
filter_formula=self.action.get_filter_formula()).fetchall()
self.set_field_from_dict('exclude_values')
def ot_insert_column_list(context, column_name) -> str:
"""Insert in the text a column list."""
action = context['ONTASK_ACTION_CONTEXT_VARIABLE___']
column_values = [
str(citem[0])
for citem in sql.get_rows(action.workflow.get_data_frame_table_name(),
column_names=[column_name],
filter_formula=action.get_filter_formula())
]
if action.action_type == models.Action.JSON_LIST:
return mark_safe(json.dumps(column_values))
return ', '.join(column_values)
def _create_eval_data_tuple(
action: models.Action,
item_column: models.Column,
exclude_values: List,
user_fname_column: Optional[models.Column],
) -> List[Tuple[str, str, str]]:
"""Evaluate text and create list of tuples [filename, part id, text].
Evaluate the conditions in the actions based on the given
item_column excluding the values in exclude_values. This returns a
list with tuples [action text, None, participant column value]. Process
that list to insert as second element of the tuple the corresponding
value in user_fname_column (if given).
The final result is a list of triplets with:
- Filename
- part id as extracted from the participation column
- HTML body text
:param action: Action being processed
:param item_column: The column used to iterate
:param exclude_values: List of values to exclude from evaluation
:param user_fname_column: Column to use for filename creation
:return: List[Tuple[text, text, text]]
"""
# Obtain the personalised text
action_evals = evaluate_action(action,
column_name=item_column.name,
exclude_values=exclude_values)
if user_fname_column:
# Get the user_fname_column values
user_fname_data = [
row[user_fname_column.name] for row in sql.get_rows(
action.workflow.get_data_frame_table_name(),
column_names=[user_fname_column.name],
filter_formula=None).fetchall()
]
else:
# Array of empty strings to concatenate
user_fname_data = [''] * len(action_evals)
return [(user_fname, part_id, _HTML_BODY.format(msg_body))
for (msg_body,
part_id), user_fname in zip(action_evals, user_fname_data)]
def do_workflow_update_lusers(workflow: Workflow, log_item: Log):
"""Recalculate the field lusers.
Recalculate the elements in the field lusers of the workflow based on the
fields luser_email_column and luser_email_column_MD5
:param workflow: Workflow to update
:param log_item: Log where to leave the status of the operation
:return: Changes in the lusers ManyToMany relationships
"""
# Get the column content
emails = get_rows(workflow.get_data_frame_table_name(),
column_names=[workflow.luser_email_column.name])
luser_list = []
created = 0
for row in emails:
uemail = row[workflow.luser_email_column.name]
luser = get_user_model().objects.filter(email=uemail).first()
if not luser:
# Create user
if settings.DEBUG:
# Define users with the same password in development
password = '******' # NOQA
else:
password = get_random_string(length=RANDOM_PWD_LENGTH)
luser = get_user_model().objects.create_user(
email=uemail,
password=password,
)
created += 1
luser_list.append(luser)
# Assign result
workflow.lusers.set(luser_list)
# Report status
log_item.payload['total_users'] = emails.rowcount
log_item.payload['new_users'] = created
log_item.payload['status'] = ugettext(
'Learner emails successfully updated.', )
log_item.save()
def clean(self) -> Dict:
"""Detect uniques values in one column, and different column names."""
form_data = super().clean()
# Move data to the payload so that is ready to be used
pcolumn = form_data['user_fname_column']
self.store_field_in_dict('user_fname_column',
pcolumn.pk if pcolumn else None)
self.store_fields_in_dict([('file_suffix', None),
('zip_for_moodle', None)])
# Participant column must be unique
pcolumn = form_data['item_column']
ufname_column = form_data['user_fname_column']
# If both values are given and they are identical, return with error
if pcolumn and ufname_column and pcolumn == ufname_column:
self.add_error(None, _('The two columns must be different'))
return form_data
# If a moodle zip has been requested
if form_data.get('zip_for_moodle'):
if not pcolumn or not ufname_column:
self.add_error(None,
_('A Moodle ZIP requires two column names'))
return form_data
# Participant columns must match the pattern 'Participant [0-9]+'
pcolumn_data = sql.get_rows(
self.action.workflow.get_data_frame_table_name(),
column_names=[pcolumn.name])
participant_error = any(
not PARTICIPANT_RE.search(str(row[pcolumn.name]))
for row in pcolumn_data)
if participant_error:
self.add_error(
'item_column',
_('Values in column must have format ' +
'"Participant [number]"'))
return form_data
def ot_insert_report(context, *args) -> str:
"""Insert in the text a column list."""
action = context[ACTION_CONTEXT_VAR]
real_args = [evaluate.RTR_ITEM(argitem) for argitem in args]
all_column_values = []
for column_name in real_args:
all_column_values.append([
str(citem[0]) for citem in sql.get_rows(
action.workflow.get_data_frame_table_name(),
column_names=[evaluate.RTR_ITEM(column_name)],
filter_formula=action.get_filter_formula())])
if action.action_type == models.Action.JSON_REPORT:
return mark_safe(json.dumps({
cname: cval for cname, cval in zip(real_args, all_column_values)}))
# return the content rendered as a table
return render_to_string(
'table.html',
{
'column_names': real_args,
'rows': zip(*all_column_values)})
def get_subframe(
table_name: str,
filter_formula,
column_names: List[str],
) -> pd.DataFrame:
"""Load the subframe using the filter and column names.
Execute a select query to extract a subset of the dataframe and turn the
resulting query set into a data frame.
:param table_name: Table
:param filter_formula: Formula to filter the data (or None)
:param column_names: [list of column names], QuerySet with the data rows
:return: DataFrame
"""
# Create the DataFrame and set the column names
return pd.DataFrame.from_records(sql.get_rows(
table_name,
column_names=column_names,
filter_formula=filter_formula,
).fetchall(),
columns=column_names,
coerce_float=True)
def get_table_row_by_index(
workflow,
filter_formula,
idx: int,
):
"""Select the set of elements in the row with the given index.
:param workflow: Workflow object storing the data
:param filter_formula: Condition object to filter the data (or None)
:param idx: Row number to get (first row is idx = 1)
:return: A dictionary with the (column_name, value) data or None if the
index is out of bounds
"""
# Get the data
df_data = sql.get_rows(workflow.get_data_frame_table_name(),
column_names=workflow.get_column_names(),
filter_formula=filter_formula)
# If the data is not there, return None
if idx > df_data.rowcount:
return None
return df_data.fetchall()[idx - 1]
def extra_validation(self, validated_data: Dict):
"""Validate the presence of certain fields."""
super().extra_validation(validated_data)
action = validated_data['action']
payload = validated_data['payload']
item_column_name = payload.get('item_column')
if not item_column_name:
raise APIException(
_('Personalized text need a column name in payload '
'field item_column.'))
item_column = action.workflow.columns.filter(
name=item_column_name).first()
if not item_column:
raise APIException(
_('Incorrect column name in field item_column.'))
if action.action_type != models.Action.PERSONALIZED_TEXT:
raise APIException(_('Incorrect type of action to schedule.'))
subject = payload.get('subject')
if not subject:
raise APIException(_('Personalized text needs a subject.'))
# Check if the values in the email column are correct emails
try:
column_data = sql.get_rows(
action.workflow.get_data_frame_table_name(),
column_names=[item_column.name])
if not all(
is_correct_email(row[item_column.name]) for row in column_data
):
# column has incorrect email addresses
raise APIException(
_('The column with email addresses has incorrect values.'))
except TypeError:
raise APIException(
_('The column with email addresses has incorrect values.'))
payload['item_column'] = item_column.id
try:
if not all(
is_correct_email(email)
for email in payload.get('cc_email', '').split() if email
):
raise APIException(
_('cc_email must be a space-separated list of emails.'))
except Exception:
raise APIException(
_('cc_email must be a space-separated list of emails.'))
try:
if not all(
is_correct_email(email)
for email in payload.get('bcc_email', '').split() if email
):
raise APIException(
_('bcc_email must be a space-separated list of emails.'))
except Exception:
raise APIException(
_('bcc_email must be a space-separated list of emails.'))
def execute_operation(
self,
user,
workflow: Optional[models.Workflow] = None,
action: Optional[models.Action] = None,
payload: Optional[Dict] = None,
log_item: Optional[models.Log] = None,
):
"""Recalculate lusers field.
Recalculate the elements in field lusers of the workflow based on the
fields luser_email_column and luser_email_column_MD5
:param user: User object that is executing the action
:param workflow: Workflow being processed (if applicable)
:param action: Action being executed (if applicable)
:param payload: Dictionary with the execution parameters
:param log_item: Id of the log object where the status is reflected
:return: Nothing, the result is stored in the log with log_id
"""
del action
if not log_item and self.log_event:
log_item = workflow.log(user,
operation_type=self.log_event,
**payload)
# First get the log item to make sure we can record diagnostics
try:
# Get the column content
emails = sql.get_rows(
workflow.get_data_frame_table_name(),
column_names=[workflow.luser_email_column.name])
luser_list = []
created = 0
for row in emails:
uemail = row[workflow.luser_email_column.name]
luser = get_user_model().objects.filter(email=uemail).first()
if not luser:
# Create user
if settings.DEBUG:
# Define users with the same password in development
password = '******' # NOQA
else:
password = get_random_string(length=RANDOM_PWD_LENGTH)
luser = get_user_model().objects.create_user(
email=uemail,
password=password,
)
created += 1
luser_list.append(luser)
# Assign result
workflow.lusers.set(luser_list)
workflow.lusers_is_outdated = False
workflow.save()
# Report status
log_item.payload['total_users'] = emails.rowcount
log_item.payload['new_users'] = created
log_item.payload['status'] = ugettext(
'Learner emails successfully updated.', )
log_item.save(update_fields=['payload'])
# Reflect status in the log event
log_item.payload['status'] = 'Execution finished successfully'
log_item.save(update_fields=['payload'])
except Exception as exc:
log_item.payload['status'] = ugettext('Error: {0}').format(exc)
log_item.save(update_fields=['payload'])
def assign_luser_column(
request: HttpRequest,
pk: Optional[int] = None,
workflow: Optional[Workflow] = None,
column: Optional[Column] = None,
) -> JsonResponse:
"""Render the view to assign the luser column.
AJAX view to assign the column with id PK to the field luser_email_column
and calculate the hash
:param request: HTTP request
:param pk: Column id
:return: JSON data
"""
if workflow.nrows == 0:
messages.error(
request,
_('Workflow has no data. ' +
'Go to "Manage table data" to upload data.'),
)
return JsonResponse({'html_redirect': reverse('action:index')})
if not pk:
# Empty pk, means reset the field.
workflow.luser_email_column = None
workflow.luser_email_column_md5 = ''
workflow.lusers.set([])
workflow.save()
return JsonResponse({'html_redirect': ''})
table_name = workflow.get_data_frame_table_name()
# Get the column content
emails = get_rows(table_name, column_names=[column.name])
# Verify that the column as a valid set of emails
if not all(is_correct_email(row[column.name]) for row in emails):
messages.error(
request,
_('The selected column does not contain email addresses.'),
)
return JsonResponse({'html_redirect': ''})
# Update the column
workflow.luser_email_column = column
workflow.save()
# Calculate the MD5 value
md5_hash = get_text_column_hash(table_name, column.name)
if workflow.luser_email_column_md5 != md5_hash:
# Change detected, run the update in batch mode
workflow.luser_email_column_md5 = md5_hash
# Log the event with the status "preparing updating"
log_item = workflow.log(request.user, Log.WORKFLOW_UPDATE_LUSERS)
# Push the update of lusers to batch processing
workflow_update_lusers_task.delay(request.user.id, workflow.id,
log_item.id)
workflow.lusers_is_outdated = False
workflow.save()
messages.success(
request,
_('The list of workflow users will be updated shortly.'),
)
return JsonResponse({'html_redirect': ''})
