def find_and_replace(df_data, data): """Searches for a value and replaces if indicated""" id = data['file_id'] # Heading of column being where find and replace is being carried out heading = data['filter_column_heading'] # Might be better to just find it throughout df_data if data['find_value'] == 'nan': filter_applied = df_data[heading].isnull() else: temp = df_data[heading] temp = temp.astype('str').str.lower() filter_applied = np.array(temp == str(data['find_value']).lower()) if data['replace_pressed']: df_data[heading][filter_applied] = data['replace_value'] column_values = df_data[heading][filter_applied] if len(column_values) > 0: error_data, dtypes_dict = get_dtype_data(id) dicts = get_geolocation_dictionary() temp_prob_list, temp_error_counter = identify_col_dtype( column_values, heading, dicts) dtypes_dict[heading][filter_applied] = temp_error_counter error_data[heading] = get_prob_list(dtypes_dict[heading]) save_validation_data(error_data, id, dtypes_dict) update_data(File.objects.get(id=id).file, df_data) df_data = df_data[df_data[heading] == data['replace_value']] else: df_data = df_data[filter_applied] return df_data
def update(id, data): '''Updates cell that user edits.''' df_data = get_file_data(id) error_data, dtypes_dict = get_dtype_data(id) if not data['changeHeader'] == '': count = 2 tmp = data['header_value'] while tmp in df_data.columns: tmp = data['header_value'] + str(count) count += 1 data['header_value'] = tmp df_data = df_data.rename(columns={data['header_tobe_changed']: data['header_value']}) dtypes_dict[data['header_value']] = dtypes_dict[data['header_tobe_changed']] dtypes_dict.pop(data['header_tobe_changed'], None) error_data[data['header_value']] = error_data[data['header_tobe_changed']] error_data.pop(data['header_tobe_changed'], None) else: heading = data['column'] line_no = data['line_no'] df_data[heading][line_no] = data['cell_value'] prob_list, error_count = update_cell_type(df_data[heading][line_no], dtypes_dict[heading], line_no, heading) dtypes_dict[heading] = error_count error_data[heading] = prob_list save_validation_data(error_data, id, dtypes_dict) update_data(File.objects.get(id=id).file, df_data) return {'success' : 1}
def get_errors(data): '''Gets data that does not match the most probable data type found for each column.''' temp_error_message = {} id = data['file_id'] start_pos = data['start_pos'] end_pos = data['end_pos'] df_data = get_file_data(id) column_headings = df_data.columns error_data, dtypes_dict = get_dtype_data(id) errors, line_nos = check_dtypes(error_data, dtypes_dict, column_headings) selection = np.array(range(0, len(error_data[column_headings[0]]))) amount = end_pos - start_pos for i in errors: if len(errors[i]) > 0: counter = 0 line_no_selection = selection[line_nos[i]]#[start_pos:end_pos] errors_selection = errors[i]#[start_pos:end_pos] for j in errors_selection:#minus one for line no message = ('Found a ' + j + ' value instead of the most populous value ' + dtypes_dict[i][0][0] + '.') line_no = str(line_no_selection[counter]) temp_error_message[''.join([line_no,'|',i])] = (message) counter += 1 context = {'error_messages': temp_error_message} return context
def delete_data(id, data): """Deletes data based on data""" df_data = get_file_data(id) row_keys = list(map(int, data['row_keys'])) df_data = df_data.drop(df_data.index[row_keys]) df_data = df_data.reset_index(drop=True) error_data, dtypes_dict = get_dtype_data(id) error_data, dtypes_dict = remove_entries(error_data, dtypes_dict, row_keys) save_validation_data(error_data, id, dtypes_dict) update_data(File.objects.get(id=id).file, df_data) return {'success': 1}
def update(id, data): """Updates cell that user edits.""" df_data = get_file_data(id) error_data, dtypes_dict = get_dtype_data(id) if not data['changeHeader'] == '': count = 2 tmp = data['header_value'] while tmp in df_data.columns: tmp = data['header_value'] + str(count) count += 1 data['header_value'] = tmp df_data = df_data.rename( columns={data['header_tobe_changed']: data['header_value']}) dtypes_dict[data['header_value']] = dtypes_dict[ data['header_tobe_changed']] dtypes_dict.pop(data['header_tobe_changed'], None) error_data[data['header_value']] = error_data[ data['header_tobe_changed']] error_data.pop(data['header_tobe_changed'], None) else: heading = data['column'] line_no = data['line_no'] df_data[heading][line_no] = data['cell_value'] # TODO: Logic BUGS check everything why using if like below # First logic error on then save_validation_data. # When used test code will be to if # When used GraphQL will be to else if type(error_data[next(iter(error_data))]) == list: prob_list, error_count = update_cell_type( df_data[heading][line_no], dtypes_dict[heading], line_no, heading) else: prob_list, error_count = update_cell_type( df_data[heading][line_no], error_data[heading], line_no, heading) dtypes_dict[heading] = error_count error_data[heading] = prob_list save_validation_data(error_data, id, dtypes_dict) update_data(File.objects.get(id=id).file, df_data) return {'success': 1}
def begin_mapping(data): # Performs manual mapping process if 'mapping_dict' in data: final_file_headings = {} id = data['metadata_id'] # Get relevant data # save_mapping(id, instance) df_data = get_file_data(id) data_model_dict, filter_headings_dict, empty_entries_dict, \ multi_entry_dict, point_base_dict = split_mapping_data(data) error_data, dtypes_dict = get_dtype_data(id) # Check if any new Geolocation information to save if 'coord' in point_base_dict and \ not point_base_dict['coord']['lat'] == '': # shoud save here and get later lat = point_base_dict['coord']['lat'] lon = point_base_dict['coord']['lon'] df_data['geolocation'] = 'pointbased' dtypes_dict['geolocation'] = df_data['geolocation'].copy() df_data['geolocation'] = df_data[lon].astype( str) + "," + df_data[lat].astype('str') data_model_dict['geolocation'] = ['geolocation'] # drop the two columns dtypes_dict.pop(lat, None) dtypes_dict.pop(lon, None) df_data.drop([lon, lat], inplace=True, axis=1) # dtypes_dict['geolocation'] = point_based = True else: point_based = False # Reformat dataframe for dates or categories if len(data_model_dict['value']) > 1: df_data, dtypes_dict, data_model_dict = convert_df( df_data, multi_entry_dict, data_model_dict, empty_entries_dict.pop('empty_value_format'), dtypes_dict) # Apply missing values df_data, data_model_dict, dtypes_dict = apply_missing_values( df_data, data_model_dict, dtypes_dict, empty_entries_dict) # Check if datatypes of data is correct result, correction_mappings, context = check_mapping_dtypes( data_model_dict, dtypes_dict) if not result: print(context) # return context # Bad mapping raise Exception(context) # TODO: check is new_dtypes_dict necessary # why can't this be moved in front of check mappings? # Could be important if user wants to see which parts of data did not # get mapped and why they did not error_lines, new_dtypes_dict = generate_error_data(df_data) save_validation_data(error_lines, id, dtypes_dict) # TODO: check is this is needed for key in new_dtypes_dict: dtypes_dict[key] = new_dtypes_dict[key] # Get the reverse mapping of data_model_dict # TODO: check if this is needed final_file_headings = {} for key in data_model_dict: if data_model_dict[key]: if key == 'filters': final_file_headings[key] = data_model_dict[key] else: final_file_headings[key] = data_model_dict[key][0] # Normalise data df_data = correct_data(df_data, correction_mappings, error_lines, final_file_headings, point_based) # Filter out bad data from datafram filter_applied = ( df_data[final_file_headings['indicator']].notnull() # The sections of data model # are not allowed to be empty & df_data[final_file_headings['date']].notnull() & df_data[final_file_headings['value']].notnull() & df_data[final_file_headings['geolocation']].notnull()) df_data = df_data[filter_applied].reset_index() # Remove empty values df_data[final_file_headings['date']] = pd.to_numeric( # Convert all dates to numbers # TODO: check is this needed, normalise should do this df_data[final_file_headings['date']]).astype(int) # Create missing entries that are used in Datapoints datamodel instance, created = DateFormat.objects.get_or_create( type='YYYY') # Todo make dynamic if created: instance.save() final_file_headings['date_format'] = 'date_format' final_file_headings['metadata'] = 'metadata' metadata = File.objects.get(id=id) df_data['metadata'] = metadata df_data['date_format'] = instance # Save and get foreign key data for datapoints model ind_dict, headings_dict, geolocation_dict, value_format_dict, \ filters_dict = get_save_unique_datapoints( df_data, final_file_headings, metadata, instance, filter_headings_dict, point_based) # Save Datapoints dicts = [ ind_dict, headings_dict, geolocation_dict, value_format_dict, filters_dict ] save_datapoints(df_data, final_file_headings, filter_headings_dict, dicts) context = {'success': 1} return context else: context = { 'error_messages': 'No data in dictionary sent', 'success': 0 } raise context
def get_errors(data): """ Gets data that does not match the most probable data type found for each column """ error_messages = [] id = data['file_id'] start_pos = data['start_pos'] end_pos = data['end_pos'] df_data = get_file_data(id) column_headings = df_data.columns error_data, dtypes_dict = get_dtype_data(id) # TODO: Logic BUGS check everything why using if like below # First logic error on then save_validation_data. # When used test code will be to if # When used GraphQL will be to else two_index = False try: errors, line_nos = check_dtypes(error_data, dtypes_dict, column_headings) selection = np.array(range(0, len(error_data[column_headings[0]]))) except Exception as e: errors, line_nos = check_dtypes(dtypes_dict, error_data, column_headings) selection = np.array(range(0, len(dtypes_dict[column_headings[0]]))) two_index = False amount = end_pos - start_pos error_rows = [] for i in errors: if len(errors[i]) > 0: counter = 0 try: line_no_selection = selection[line_nos[i]] # [start_pos:end_pos] except Exception as e: line_no_selection = None pass errors_selection = errors[i] # [start_pos:end_pos] # TODO: related above bug related the below for j in errors_selection: # minus one for line no try: dtype = dtypes_dict[i][0][0] if two_index \ else dtypes_dict[i][0] row = line_no_selection[counter] if row not in error_rows: error_rows.append(row) message = ('Found a ' + j + ' value instead of the most populous value ' + dtype + '.') line_no = str(row) error_message_row = dict() error_message_row[''.join([line_no, '|', i])] = (message) error_messages.append(error_message_row) counter += 1 except Exception as e: pass context = {'error_messages': error_messages} return context, error_rows
def filter_for_errors(df_data, data): filter_column = data['error_filter_value'] error_data, dtypes_dict = get_dtype_data(data['id']) errors, line_nos = check_dtypes(error_data, dtypes_dict, [filter_column], data['start_pos'], data['end_pos']) return df_data[line_nos[filter_column]]