def _get_df_and_columns( workflow: Workflow, pk: int, ) -> Optional[Tuple[DataFrame, List, View]]: """Get the DF and the columns to process. :param workflow: Workflow object :param pk: Optional id for a view :return: Tuple DataFrame, List of columns (None, None if error """ # If a view is given, filter the columns view = None if pk: view = workflow.views.filter(pk=pk).first() if not view: # View not found. Redirect to workflow detail return None columns_to_view = view.columns.filter(is_key=False) df = load_table(workflow.get_data_frame_table_name(), [col.name for col in columns_to_view], view.formula) else: # No view given, fetch the entire data frame columns_to_view = workflow.columns.filter(is_key=False) df = load_table(workflow.get_data_frame_table_name(), [col.name for col in columns_to_view]) return df, columns_to_view, view
def _do_clone_workflow(workflow: Workflow) -> Workflow: """Clone the workflow. :param workflow: source workflow :return: Clone object """ new_workflow = Workflow( user=workflow.user, name=create_new_name( workflow.name, Workflow.objects.filter( Q(user=workflow.user) | Q(shared=workflow.user)), ), description_text=workflow.description_text, nrows=workflow.nrows, ncols=workflow.ncols, attributes=copy.deepcopy(workflow.attributes), query_builder_ops=copy.deepcopy(workflow.query_builder_ops), luser_email_column_md5=workflow.luser_email_column_md5, lusers_is_outdated=workflow.lusers_is_outdated) new_workflow.save() try: new_workflow.shared.set(list(workflow.shared.all())) new_workflow.lusers.set(list(workflow.lusers.all())) # Clone the columns for item_obj in workflow.columns.all(): do_clone_column_only(item_obj, new_workflow=new_workflow) # Update the luser_email_column if needed: if workflow.luser_email_column: new_workflow.luser_email_column = new_workflow.columns.get( name=workflow.luser_email_column.name, ) # Clone the DB table clone_table(workflow.get_data_frame_table_name(), new_workflow.get_data_frame_table_name()) # Clone actions for item_obj in workflow.actions.all(): do_clone_action(item_obj, new_workflow) for item_obj in workflow.views.all(): do_clone_view(item_obj, new_workflow) # Done! new_workflow.save() except Exception as exc: new_workflow.delete() raise exc return new_workflow
def perform_row_delete(workflow: models.Workflow, row_key: str, row_value: Any): """Perform the row deletion operation. :param workflow: Workflow being considered :param row_key: Name of the key column to select the row :param row_value: Value of the previous column to select the row. :return: Reflected in the workflow """ # if there is no key or value, flag the message and return to table # view if not row_key or not row_value: raise OnTaskTableNoKeyValueError( message=_('Incorrect URL invoked to delete a row')) # The response will require going to the table display anyway # Proceed to delete the row sql.delete_row(workflow.get_data_frame_table_name(), (row_key, row_value)) # Update rowcount workflow.nrows -= 1 workflow.save(update_fields=['nrows']) # Update the value of all the conditions in the actions # TODO: Explore how to do this asynchronously (or lazy) for action in workflow.actions.all(): action.update_n_rows_selected()
def do_clone_workflow(user, workflow: models.Workflow) -> models.Workflow: """Clone a workflow. :param user: User performing the operation :param workflow: source workflow :return: Cloned object """ new_workflow = models.Workflow( user=workflow.user, name=create_new_name( workflow.name, models.Workflow.objects.filter( Q(user=workflow.user) | Q(shared=workflow.user)), ), description_text=workflow.description_text, nrows=workflow.nrows, ncols=workflow.ncols, attributes=copy.deepcopy(workflow.attributes), query_builder_ops=copy.deepcopy(workflow.query_builder_ops), luser_email_column_md5=workflow.luser_email_column_md5, lusers_is_outdated=workflow.lusers_is_outdated) new_workflow.save() try: new_workflow.shared.set(list(workflow.shared.all())) new_workflow.lusers.set(list(workflow.lusers.all())) # Clone the columns for item_obj in workflow.columns.all(): do_clone_column_only(item_obj, new_workflow=new_workflow) # Update the luser_email_column if needed: if workflow.luser_email_column: new_workflow.luser_email_column = new_workflow.columns.get( name=workflow.luser_email_column.name, ) # Clone the DB table sql.clone_table(workflow.get_data_frame_table_name(), new_workflow.get_data_frame_table_name()) # Clone actions for item_obj in workflow.actions.all(): do_clone_action(user, item_obj, new_workflow) for item_obj in workflow.views.all(): do_clone_view(user, item_obj, new_workflow) # Done! new_workflow.save() except Exception as exc: raise OnTaskServiceException( message=_('Error while cloning workflow: {0}').format(exc), to_delete=[new_workflow]) workflow.log(user, models.Log.WORKFLOW_CLONE, id_old=workflow.id, name_old=workflow.name) return new_workflow
def update_row_values( workflow: models.Workflow, update_key: str, update_val: Any, row_values: List[Any], ): """Update a row with the information in the form. :param workflow: Workflow object being processed. :param update_key: Column name to use to select row. :param update_val: Column value to select row. :param row_values: List of values to store in the row :return: Nothing. """ # Create the query to update the row column_names = [col.name for col in workflow.columns.all()] with transaction.atomic(): # Update the row in the db sql.update_row(workflow.get_data_frame_table_name(), column_names, row_values, filter_dict={update_key: update_val}) # verify that the "key" property is maintained in all the # columns. checks.check_key_columns(workflow) # Recompute all the values of the conditions in each of the actions # TODO: Explore how to do this asynchronously (or lazy) for act in workflow.actions.all(): act.update_n_rows_selected()
def create_row(workflow: models.Workflow, row_values: List[Any]): """Create an additional row with the information in the form. :param workflow: Workflow object being processed. :param row_values: List of values to store. :return: Nothing. """ # Create the query to update the row columns = workflow.columns.all() column_names = [col.name for col in columns] with transaction.atomic(): # Insert the new row in the db sql.insert_row(workflow.get_data_frame_table_name(), column_names, row_values) # verify that the "key" property is maintained in all the # columns. checks.check_key_columns(workflow) # Update number of rows workflow.nrows += 1 workflow.save() # Recompute all the values of the conditions in each of the actions # TODO: Explore how to do this asynchronously (or lazy) for act in workflow.actions.all(): act.update_n_rows_selected()
def _get_column_visualization_items( workflow: Workflow, column: Column, ): """Get the visualization items (scripts and HTML) for a column. :param workflow: Workflow being processed :param column: Column requested :return: Tuple stat_data with descriptive stats, visualization scripts and visualization HTML """ # Get the dataframe df = load_table(workflow.get_data_frame_table_name(), [column.name]) # Extract the data to show at the top of the page stat_data = get_column_statistics(df[column.name]) vis_scripts = [] visualizations = _get_column_visualisations( column, df[[column.name]], vis_scripts, context={ 'style': 'width:100%; height:100%;' + 'display:inline-block;' }, ) return stat_data, vis_scripts, visualizations
def add_column_to_workflow( user, workflow: models.Workflow, column: models.Column, column_initial_value: Any, action_column_event: Optional[str] = None, action: Optional[models.Action] = None, ): """Add a new column to the workflow. :param user: User executing the operation :param workflow: Used workflow :param column: Column to insert :param column_initial_value: Initial value for the column :param action_column_event: Event to record if the action/column needs to be stored :param action: If the column is a question, this is the action in which it is being used. :return: Nothing. Column added to Wflow and DB """ # Catch the special case of integer type and no initial value. Pandas # encodes it as NaN but a cycle through the database transforms it into # a string. To avoid this case, integer + empty value => double if column.data_type == 'integer' and column_initial_value is None: column.data_type = 'double' # Fill in the remaining fields in the column column.workflow = workflow column.is_key = False # Update the positions of the appropriate columns workflow.reposition_columns(workflow.ncols + 1, column.position) # Save column, refresh workflow, and increase number of columns column.save() workflow.refresh_from_db() workflow.ncols += 1 workflow.set_query_builder_ops() workflow.save() # Add the new column to the DB try: sql.add_column_to_db( workflow.get_data_frame_table_name(), column.name, column.data_type, initial=column_initial_value) except Exception as exc: raise services.OnTaskWorkflowAddColumn( message=_('Unable to add element: {0}').format(str(exc)), to_delete=[column]) if action_column_event: acc, __ = models.ActionColumnConditionTuple.objects.get_or_create( action=action, column=column, condition=None) acc.log(user, action_column_event) column.log(user, models.Log.COLUMN_ADD)
def update_luser_email_column( user, pk: int, workflow: models.Workflow, column: models.Column, ): """Update the field luser_email in the workflow. :param user: User making the request :param pk: Column ID to obtain the user id :param workflow: Workflow being manipulated. :param column: Column being used to update the luser field. :return: """ if not pk: # Empty pk, means reset the field. workflow.luser_email_column = None workflow.luser_email_column_md5 = '' workflow.lusers.set([]) workflow.save( update_fields=['luser_email_column', 'luser_email_column_md5']) return table_name = workflow.get_data_frame_table_name() # Get the column content emails = sql.get_rows(table_name, column_names=[column.name]) # Verify that the column as a valid set of emails incorrect_email = get_incorrect_email([row[column.name] for row in emails]) if incorrect_email: raise services.OnTaskWorkflowEmailError(message=_( 'Incorrect email addresses "{0}".').format(incorrect_email)) # Update the column workflow.luser_email_column = column workflow.save(update_fields=['luser_email_column']) # Calculate the MD5 value md5_hash = sql.get_text_column_hash(table_name, column.name) if workflow.luser_email_column_md5 == md5_hash: return # Change detected, run the update in batch mode workflow.luser_email_column_md5 = md5_hash workflow.save(update_fields=['luser_email_column_md5']) # Log the event with the status "preparing updating" log_item = workflow.log(user, models.Log.WORKFLOW_UPDATE_LUSERS) # Push the update of lusers to batch processing tasks.execute_operation.delay( operation_type=models.Log.WORKFLOW_UPDATE_LUSERS, user_id=user.id, workflow_id=workflow.id, log_id=log_item.id)
def check_wf_df(workflow: models.Workflow) -> bool: """Check consistency between Workflow info and the data frame. Check the consistency between the information stored in the workflow and the structure of the underlying dataframe :param workflow: Workflow object :return: Boolean stating the result of the check. True: Correct. """ # Get the df df = pandas.load_table(workflow.get_data_frame_table_name()) # Set values in case there is no df if df is not None: dfnrows = df.shape[0] dfncols = df.shape[1] df_col_names = list(df.columns) else: dfnrows = 0 dfncols = 0 df_col_names = [] # Check 1: Number of rows and columns assert workflow.nrows == dfnrows, 'Inconsistent number of rows' assert workflow.ncols == dfncols, 'Inconsistent number of columns' # Identical sets of columns wf_cols = workflow.columns.all() assert set(df_col_names) == {col.name for col in wf_cols }, ('Inconsistent set of columns') # Identical data types # for n1, n2 in zip(wf_cols, df_col_names): for col in wf_cols: df_dt = pandas.datatype_names.get(df[col.name].dtype.name) if col.data_type == 'boolean' and df_dt == 'string': # This is the case of a column with Boolean and Nulls continue assert col.data_type == df_dt, ('Inconsistent data type {0}'.format( col.name)) # Verify that the columns marked as unique are preserved for col in workflow.columns.filter(is_key=True): assert pandas.is_unique_column( df[col.name]), ('Column {0} should be unique.'.format(col.name)) # Columns are properly numbered cpos = workflow.columns.values_list('position', flat=True) rng = range(1, len(cpos) + 1) assert sorted(cpos) == list(rng) return True
def update_column( user, workflow: models.Workflow, column: models.Column, old_name: str, old_position: int, action_column_item: Optional[models.Action] = None, action_column_event: Optional[str] = None, ): """Update information in a column. :param user: User performing the operation :param workflow: Workflow being processed :param column: Column being modified :param old_name: Old name to detect renaming :param old_position: old position to detect repositioning :param action_column_item: Action/column item in case the action is stored in that table. :param action_column_event: Event to record the operation on the action/column :return: Nothing. Side effect in the workflow. """ # If there is a new name, rename the data frame columns if old_name != column.name: sql.db_rename_column( workflow.get_data_frame_table_name(), old_name, column.name) pandas.rename_df_column(workflow, old_name, column.name) if old_position != column.position: # Update the positions of the appropriate columns workflow.reposition_columns(old_position, column.position) column.save() # Go back to the DB because the prefetch columns are not valid # any more workflow = models.Workflow.objects.prefetch_related('columns').get( id=workflow.id, ) # Changes in column require rebuilding the query_builder_ops workflow.set_query_builder_ops() # Save the workflow workflow.save() if action_column_item: action_column_item.log(user, action_column_event) # Log the event column.log(user, models.Log.COLUMN_EDIT)
def add_formula_column( user, workflow: models.Workflow, column: models.Column, operation: str, selected_columns: List[models.Column], ): """Add the formula column to the workflow. :param user: User making the request :param workflow: Workflow to add the column :param column: Column being added :param operation: string denoting the operation :param selected_columns: List of columns selected for the operation. :return: Column is added to the workflow """ # Save the column object attached to the form and add additional fields column.workflow = workflow column.is_key = False # Save the instance column.save() # Update the data frame df = pandas.load_table(workflow.get_data_frame_table_name()) # Add the column with the appropriate computation cnames = [col.name for col in selected_columns] df[column.name] = _op_distrib[operation](df[cnames]) # Populate the column type column.data_type = pandas.datatype_names.get( df[column.name].dtype.name) # Update the positions of the appropriate columns workflow.reposition_columns(workflow.ncols + 1, column.position) column.save() workflow.refresh_from_db() # Store the df to DB try: pandas.store_dataframe(df, workflow) except Exception as exc: raise services.OnTaskWorkflowAddColumn( message=_('Unable to add column: {0}').format(str(exc)), to_delete=[column]) workflow.ncols = workflow.columns.count() workflow.save() column.log(user, models.Log.COLUMN_ADD_FORMULA)
def check_luser_email_column_outdated(workflow: models.Workflow): """Detect if the luser_email column is up to date. :param workflow: Workflow being manipulated. :return: Side effect, md5_hash_is_outdated is updated. """ if workflow.luser_email_column: md5_hash = sql.get_text_column_hash( workflow.get_data_frame_table_name(), workflow.luser_email_column.name) if md5_hash != workflow.luser_email_column_md5: # Information is outdated workflow.lusers_is_outdated = True workflow.save(update_fields=['lusers_is_outdated'])
def upload_step_four( request: http.HttpRequest, workflow: models.Workflow, upload_data: Dict, ) -> http.HttpResponse: """Perform the merge operation. :param request: Received request :param workflow: Workflow being processed :param upload_data: Dictionary with all the information about the merge. :return: HttpResponse """ # Get the dataframes to merge try: dst_df = pandas.load_table(workflow.get_data_frame_table_name()) src_df = pandas.load_table(workflow.get_upload_table_name()) except Exception: return render(request, 'error.html', {'message': _('Exception while loading data frame')}) try: pandas.perform_dataframe_upload_merge(workflow, dst_df, src_df, upload_data) except Exception as exc: # Nuke the temporary table sql.delete_table(workflow.get_upload_table_name()) col_info = workflow.get_column_info() workflow.log(request.user, models.Log.WORKFLOW_DATA_FAILEDMERGE, column_names=col_info[0], column_types=col_info[1], column_unique=col_info[2], error_message=str(exc)) messages.error(request, _('Merge operation failed. ') + str(exc)) return redirect(reverse('table:display')) col_info = workflow.get_column_info() workflow.log(request.user, upload_data['log_upload'], column_names=col_info[0], column_types=col_info[1], column_unique=col_info[2]) store_workflow_in_session(request.session, workflow) request.session.pop('upload_data', None) return redirect(reverse('table:display'))
def check_key_columns(workflow: models.Workflow): """Check that key columns maintain their property. Function used to verify that after changes in the DB the key columns maintain their property. :param workflow: Object to use for the verification. :return: Nothing. Raise exception if key column lost the property. """ col_name = next((col.name for col in workflow.columns.filter(is_key=True) if not sql.is_column_unique( workflow.get_data_frame_table_name(), col.name)), None) if col_name: raise Exception( _('The new data does not preserve the key ' + 'property of column "{0}"'.format(col_name)))
def do_workflow_update_lusers(workflow: Workflow, log_item: Log): """Recalculate the field lusers. Recalculate the elements in the field lusers of the workflow based on the fields luser_email_column and luser_email_column_MD5 :param workflow: Workflow to update :param log_item: Log where to leave the status of the operation :return: Changes in the lusers ManyToMany relationships """ # Get the column content emails = get_rows(workflow.get_data_frame_table_name(), column_names=[workflow.luser_email_column.name]) luser_list = [] created = 0 for row in emails: uemail = row[workflow.luser_email_column.name] luser = get_user_model().objects.filter(email=uemail).first() if not luser: # Create user if settings.DEBUG: # Define users with the same password in development password = '******' # NOQA else: password = get_random_string(length=RANDOM_PWD_LENGTH) luser = get_user_model().objects.create_user( email=uemail, password=password, ) created += 1 luser_list.append(luser) # Assign result workflow.lusers.set(luser_list) # Report status log_item.payload['total_users'] = emails.rowcount log_item.payload['new_users'] = created log_item.payload['status'] = ugettext( 'Learner emails successfully updated.', ) log_item.save()
def create_survey_table( workflow: models.Workflow, action: models.Action, dt_page: DataTablesServerSidePaging, ) -> http.JsonResponse: """Create the table with the survey entries for instructor. :param workflow: Workflow being processed :param action: Action representing the survey :param dt_page: Data tables server side paging object :return : JSon respnse """ columns = [ccpair.column for ccpair in action.column_condition_pair.all()] query_set = _create_initial_qs( workflow.get_data_frame_table_name(), action.get_filter_formula(), columns, dt_page, ) filtered = len(query_set) # Get the subset of the qs to show in the table query_set = _create_table_qsdata( action.id, query_set, dt_page, columns, next(idx for idx, col in enumerate(columns) if col.is_key), ) return http.JsonResponse({ 'draw': dt_page.draw, 'recordsTotal': workflow.nrows, 'recordsFiltered': filtered, 'data': query_set, })
def batch_load_df_from_athenaconnection( workflow: models.Workflow, conn: models.AthenaConnection, run_params: Dict, log_item: models.Log, ): """Batch load a DF from an Athena connection. run_params has: aws_secret_access_key: Optional[str] = None, aws_session_token: Optional[str] = None, table_name: Optional[str] = None key_column_name[str] = None merge_method[str] = None from pyathena import connect from pyathena.pandas_cursor import PandasCursor cursor = connect( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, s3_staging_dir=staging_dir, region_name=region_name) df = pd.read_sql('SELECT * FROM given_table_name', cursor) print(df.describe()) print(df.head()) :param workflow: Workflow to store the new data :param conn: AthenaConnection object with the connection parameters. :param run_params: Dictionary with additional connection parameters :param log_item: Log object to reflect the status of the execution :return: Nothing. """ staging_dir = 's3://{0}'.format(conn.aws_bucket_name) if conn.aws_file_path: staging_dir = staging_dir + '/' + conn.aws_file_path cursor = connect(aws_access_key_id=conn.aws_access_key, aws_secret_access_key=run_params['aws_secret_access_key'], aws_session_token=run_params['aws_session_token'], s3_staging_dir=staging_dir, region_name=conn.aws_region_name) data_frame = pd.read_sql_table(run_params['table_name'], cursor) # Strip white space from all string columns and try to convert to # datetime just in case data_frame = pandas.detect_datetime_columns(data_frame) pandas.verify_data_frame(data_frame) col_names, col_types, is_key = pandas.store_temporary_dataframe( data_frame, workflow) upload_data = { 'initial_column_names': col_names, 'col_types': col_types, 'src_is_key_column': is_key, 'rename_column_names': col_names[:], 'columns_to_upload': [True] * len(col_names), 'keep_key_column': is_key[:] } if not workflow.has_data_frame(): # Regular load operation pandas.store_workflow_table(workflow, upload_data) log_item.payload['col_names'] = col_names log_item.payload['col_types'] = col_types log_item.payload['column_unique'] = is_key log_item.payload['num_rows'] = workflow.nrows log_item.payload['num_cols'] = workflow.ncols log_item.save(update_fields=['payload']) return # Merge operation upload_data['dst_column_names'] = workflow.get_column_names() upload_data['dst_is_unique_column'] = workflow.get_column_unique() upload_data['dst_unique_col_names'] = [ cname for idx, cname in enumerate(upload_data['dst_column_names']) if upload_data['dst_column_names'][idx] ] upload_data['src_selected_key'] = run_params['merge_key'] upload_data['dst_selected_key'] = run_params['merge_key'] upload_data['how_merge'] = run_params['merge_method'] dst_df = pandas.load_table(workflow.get_data_frame_table_name()) src_df = pandas.load_table(workflow.get_data_frame_upload_table_name()) try: pandas.perform_dataframe_upload_merge(workflow, dst_df, src_df, upload_data) except Exception as exc: # Nuke the temporary table sql.delete_table(workflow.get_data_frame_upload_table_name()) raise Exception( _('Unable to perform merge operation: {0}').format(str(exc))) col_names, col_types, is_key = workflow.get_column_info() log_item.payload['col_names'] = col_names log_item.payload['col_types'] = col_types log_item.payload['column_unique'] = is_key log_item.payload['num_rows'] = workflow.nrows log_item.payload['num_cols'] = workflow.ncols log_item.save(update_fields=['payload'])
def create(self, validated_data, **kwargs): """Create the new workflow.""" wflow_name = self.context.get('name') if not wflow_name: wflow_name = self.validated_data.get('name') if not wflow_name: raise Exception(_('Unexpected empty workflow name.')) if Workflow.objects.filter(name=wflow_name, user=self.context['user']).exists(): raise Exception( _('There is a workflow with this name. ' + 'Please provide a workflow name in the import page.')) # Initial values workflow_obj = None try: workflow_obj = Workflow( user=self.context['user'], name=wflow_name, description_text=validated_data['description_text'], nrows=0, ncols=0, attributes=validated_data['attributes'], query_builder_ops=validated_data.get('query_builder_ops', {}), ) workflow_obj.save() # Create the columns column_data = ColumnSerializer(data=validated_data.get( 'columns', []), many=True, context={'workflow': workflow_obj}) # And save its content if column_data.is_valid(): columns = column_data.save() else: raise Exception(_('Unable to save column information')) # If there is any column with position = 0, recompute (this is to # guarantee backward compatibility. if any(col.position == 0 for col in columns): for idx, col in enumerate(columns): col.position = idx + 1 col.save() # Load the data frame data_frame = validated_data.get('data_frame') if data_frame is not None: # Store the table in the DB store_table( data_frame, workflow_obj.get_data_frame_table_name(), dtype={ col.name: col.data_type for col in workflow_obj.columns.all() }, ) # Reconcile now the information in workflow and columns with # the one loaded workflow_obj.ncols = validated_data['ncols'] workflow_obj.nrows = validated_data['nrows'] workflow_obj.save() # Create the actions pointing to the workflow action_data = ActionSerializer(data=validated_data.get( 'actions', []), many=True, context={ 'workflow': workflow_obj, 'columns': columns }) if action_data.is_valid(): action_data.save() else: raise Exception(_('Unable to save column information')) # Create the views pointing to the workflow view_data = ViewSerializer(data=validated_data.get('views', []), many=True, context={ 'workflow': workflow_obj, 'columns': columns }) if view_data.is_valid(): view_data.save() else: raise Exception(_('Unable to save column information')) except Exception: # Get rid of the objects created if workflow_obj: if workflow_obj.id: workflow_obj.delete() raise return workflow_obj
def delete_column( user: get_user_model(), workflow: models.Workflow, column: models.Column, cond_to_delete: Optional[List[models.Condition]] = None, ): """Remove column from ontask.workflow. Given a workflow and a column, removes it from the workflow (and the corresponding data frame :param user: User performing the operation :param workflow: Workflow object :param column: Column object to delete :param cond_to_delete: List of conditions to delete after removing the column :return: Nothing. Effect reflected in the database """ column.log(user, models.Log.COLUMN_DELETE) # Drop the column from the DB table storing the data frame sql.df_drop_column(workflow.get_data_frame_table_name(), column.name) # Reposition the columns above the one being deleted workflow.reposition_columns(column.position, workflow.ncols + 1) # Delete the column column.delete() # Update the information in the workflow workflow.ncols = workflow.ncols - 1 workflow.save() if not cond_to_delete: # The conditions to delete are not given, so calculate them # Get the conditions/actions attached to this workflow cond_to_delete = [ cond for cond in models.Condition.objects.filter( action__workflow=workflow) if column in cond.columns.all() ] # If a column disappears, the conditions that contain that variable # are removed actions_without_filters = [] for condition in cond_to_delete: if condition.is_filter: actions_without_filters.append(condition.action) # Formula has the name of the deleted column. Delete it condition.delete() # Traverse the actions for which the filter has been deleted and reassess # all their conditions # TODO: Explore how to do this asynchronously (or lazy) for act in actions_without_filters: act.update_n_rows_selected() # If a column disappears, the views that contain only that column need to # disappear as well as they are no longer relevant. for view in workflow.views.all(): if view.columns.count() == 0: view.delete()
def get_table_visualization_items( workflow: models.Workflow, rowselect_key: Optional[str] = None, rowselect_val: Optional[Any] = None, pk: Optional[int] = None, ) -> Optional[Tuple[str, Dict]]: """Get a tuple with a template, and a dictionary to visualize a table. :param workflow: Workflow being processed :param rowselect_key: Optional key name to select a row :param rowselect_val: Optional value to select a row :param pk: Primary key of a view (could be none) :return: """ # Get the data frame and the columns df_col_view = _get_df_and_columns(workflow, pk) if not df_col_view: return None df, columns_to_view, view = df_col_view if bool(rowselect_key): template = 'table/stat_row.html' row = sql.get_row( workflow.get_data_frame_table_name(), rowselect_key, rowselect_val, column_names=[col.name for col in columns_to_view], ) else: row = None template = 'table/stat_view.html' vis_scripts = [] visualizations = [] context = { 'style': 'max-width:{0}px; max-height:{1}px; margin: auto;'.format( VISUALIZATION_WIDTH, VISUALIZATION_HEIGHT), } for idx, column in enumerate(columns_to_view): # Add the title and surrounding container visualizations.append( '<hr/><h4 class="text-center">' + column.name + '</h4>') # If all values are empty, no need to proceed if all(not col_data for col_data in df[column.name]): visualizations.append( '<p>' + _('No values in this column') + '</p>') continue if row and not row[column.name]: visualizations.append( '<p class="alert-warning">' + _('No value in this column') + '</p>', ) column_viz = _get_column_visualisations( column, df[[column.name]], vis_scripts=vis_scripts, viz_id='column_{0}'.format(idx), single_val=row[column.name] if row else None, context=context) visualizations.extend([vis.html_content for vis in column_viz]) return template, { 'value': rowselect_val, 'view': view, 'vis_scripts': vis_scripts, 'visualizations': visualizations}