def get_row_all_false_count(self):
"""Extract the rows for which all conditions are false.
Given a table and a list of conditions return the number of rows in
which all the conditions are false. :return: Number of rows that have
all conditions equal to false
"""
if self.rows_all_false is None:
if not self.workflow.has_data_frame():
# Workflow does not have a dataframe
raise ontask.OnTaskException(
'Workflow without DF in get_table_row_count_all_false')
# Separate filter from conditions
filter_item = self.conditions.filter(is_filter=True).first()
cond_list = self.conditions.filter(is_filter=False)
if not cond_list:
# Condition list is either None or empty. No restrictions.
return 0
# Workflow has a data frame and condition list is non empty
# Get the list of indeces
self.rows_all_false = sql.select_ids_all_false(
self.workflow.get_data_frame_table_name(),
filter_item.formula if filter_item else None,
cond_list.values_list('formula', flat=True),
)
self.save()
return self.rows_all_false
def evaluate_action(
action: Action,
extra_string: str = None,
column_name: str = None,
exclude_values: List[str] = None,
) -> List[List]:
"""Evaluate the content in an action based on the values in the columns.
Given an action object and an optional string:
1) Access the attached workflow
2) Obtain the data from the appropriate data frame
3) Loop over each data row and
3.1) Evaluate the conditions with respect to the values in the row
3.2) Create a context with the result of evaluating the conditions,
attributes and column names to values
3.3) Run the template with the context
3.4) Run the optional string argument with the template and the context
3.5) Select the optional column_name
4) Return the resulting objects:
List of (HTMLs body, extra string, column name value)
or an error message
:param action: Action object with pointers to conditions, filter,
workflow, etc.
:param extra_string: An extra string to process (something like the email
subject line) with the same dictionary as the text in the action.
:param column_name: Column from where to extract the special value (
typically the email address) and include it in the result.
:param exclude_values: List of values in the column to exclude
:return: list of lists resulting from the evaluation of the action. Each
element in the list contains the HTML body, the extra string (if
provided) and the column value.
"""
# Get the table data
rows = get_rows(
action.workflow.get_data_frame_table_name(),
filter_formula=action.get_filter_formula())
list_of_renders = []
for row in rows:
if exclude_values and str(row[column_name]) in exclude_values:
# Skip the row with the col_name in exclude values
continue
# Step 4: Create the context with the attributes, the evaluation of the
# conditions and the values of the columns.
context = get_action_evaluation_context(action, row)
# Append result
list_of_renders.append(
_render_tuple_result(action, context, extra_string, column_name),
)
if settings.DEBUG:
# Check that n_rows_selected is equal to rows.rowcount
action_filter = action.get_filter()
if action_filter and action_filter.n_rows_selected != rows.rowcount:
raise ontask.OnTaskException('Inconsisten n_rows_selected')
return list_of_renders
def execute_operation(
self,
user,
workflow: Optional[models.Workflow] = None,
action: Optional[models.Action] = None,
payload: Optional[Dict] = None,
log_item: Optional[models.Log] = None,
):
"""Perform a SQL upload asynchronously.
:param user: User object
:param workflow: Optional workflow object
:param action: Empty
:param payload: has fields:
- connection_id: PK of the connection object to use
- dst_key: Key column in the existing dataframe (if any) for merge
- src_key: Key column in the external dataframe (for merge)
- how_merge: Merge method: inner, outer, left, right
- db_password: Encoded password if not stored in the connection
- db_table: Table name if not stored in the connection
:param log_item: Optional logitem object.
:return: Elements to add to the "extend" field. Empty list in this case
Upload/merge the data in the given workflow.
"""
del action
# Get the connection
conn = models.SQLConnection.objects.filter(
pk=payload['connection_id']).first()
if not conn:
msg = _('Incorrect connection identifier.')
log_item.payload['error'] = msg
log_item.save(update_fields=['payload'])
raise ontask.OnTaskException(msg)
# Get the dataframe from the connection
src_df = _load_df_from_sqlconnection(conn, payload)
# Create session
session = SESSION_STORE()
session.create()
workflow = acquire_workflow_access(user, session, workflow.id)
if not workflow:
msg = _('Unable to acquire access to workflow.')
log_item.payload['error'] = msg
log_item.save(update_fields=['payload'])
raise ontask.OnTaskException(msg)
# IDEA: How to deal with failure to acquire access?
# Include a setting with the time to wait and number of retries?
if not workflow.has_data_frame():
# Simple upload
pandas.store_dataframe(src_df, workflow)
return []
# At this point the operation is a merge
dst_df = pandas.load_table(workflow.get_data_frame_table_name())
dst_key = payload.get('dst_key')
if not dst_key:
msg = _('Missing key column name of existing table.')
log_item.payload['error'] = msg
log_item.save(update_fields=['payload'])
raise ontask.OnTaskException(msg)
src_key = payload.get('src_key')
if not src_key:
msg = _('Missing key column name of new table.')
log_item.payload['error'] = msg
log_item.save(update_fields=['payload'])
raise ontask.OnTaskException(msg)
how_merge = payload.get('how_merge')
if not how_merge:
msg = _('Missing merge method.')
log_item.payload['error'] = msg
log_item.save(update_fields=['payload'])
raise ontask.OnTaskException(msg)
# Check additional correctness properties in the parameters
error = pandas.validate_merge_parameters(dst_df, src_df, how_merge,
dst_key, src_key)
if error:
log_item.payload['error'] = error
log_item.save(update_fields=['payload'])
raise ontask.OnTaskException(error)
merge_info = {
'how_merge': how_merge,
'dst_selected_key': dst_key,
'src_selected_key': src_key,
'initial_column_names': list(src_df.columns),
'rename_column_names': list(src_df.columns),
'columns_to_upload': [True] * len(list(src_df.columns))
}
# Perform the merge
pandas.perform_dataframe_upload_merge(workflow, dst_df, src_df,
merge_info)
log_item.payload = merge_info
log_item.save(update_fields=['payload'])