def addSubregions(self): if self.DataLoaded == False: messages.send_warning('No File loaded') return else: world_borders = {} for wb in self.WORLD_BORDERS: world_borders[wb] = self.INFO[wb] self.srW = subregionWindow(self, world_borders, self.SUBREGIONS_INFO_FILE) self.srW.show()
def TE_ok_pressed(self): '''saved the user specific TE values to dictionary''' te_params = json.load(open(self.TE_PARAMS_FILE)) for le in self.TE_lineEdits: try: float(le.text()) except ValueError: messages.send_warning('Parameter fields can not be empty!') te_params[le.objectName()] = le.text() print(te_params) with open(self.TE_PARAMS_FILE, 'w') as tf: json.dump(te_params, tf)
def checkSubregions(self): subregions = json.load(open(self.SUBREGIONS_INFO_FILE)) for sub in subregions.keys(): for wb in self.WORLD_BORDERS: if (wb.find('min') > -1) and (subregions[sub][wb] < self.INFO[wb]): messages.send_warning( 'Currently selcted Subregions exceed world borders') return False elif (wb.find('max') > -1) and (subregions[sub][wb] > self.INFO[wb]): messages.send_warning( 'Currently selcted Subregions exceed world borders') return False else: return True
def changeTime(self): ''' when the 'Change'Button is clicked on the time Layout, this function opens a window with sliders to set start and stop time. The selected values are passed back to mainWindows INFO dictionary''' if self.DataLoaded == False: messages.send_warning('No File loaded') return csv_dict = json.load(open(self.CSV_INFO_FILE)) delim = csv_dict['write']['delim'] df = pd.read_csv(self.TMP_FILE, header=0, sep=delim) t = df['seconds'].values f = df['frames'].values self.tW = timeWindow(self, t, f, self.INFO) self.tW.show()
def pushed_ok(self): ''' reads agent names from line edit elements, checks for validity (no empty fields, no name starting with an integer, no double names) and sends the new names to parent window. ''' self.AGENT_NAMES = [] for le in self.custom_names: text = le.text() if len(text) > 0: # don't allow empty names if text[0].isdigit(): # don't allow names to start with int messages.send_warning("Invalid or empty agent names") pass else: self.AGENT_NAMES.append(text) else: messages.send_warning("Invalid or empty agent names") pass if len(list(set(self.AGENT_NAMES))) == self.nAgents: # no duplicates print(self.AGENT_NAMES) self.parentWindow.AGENT_NAMES = self.AGENT_NAMES self.parentWindow.update_checklabels('AGENTS') self.home.close() else: messages.send_warning("Invalid or empty agent names")
def init_Info(self, tmp_file): ''' tableWindow calles this function, when 'Ok' is pressed. The tmp file created by table window is used to initialize information in mainWindow''' csv_dict = json.load(open(self.CSV_INFO_FILE)) delim = csv_dict['write']['delim'] self.TMP_FILE = tmp_file df = pd.read_csv(tmp_file, header=0, sep=delim) time_format = self.INFO['info']['time'] if tp.handle_timestamp(df['time'].values[0], time_format, self.DATE_FORMATS_FILE) == None: messages.send_warning('Time format invalid!') return False self.INFO['start_time'] = df['seconds'].values[0] self.INFO['stop_time'] = df['seconds'].values[-1] self.INFO['start_frame'] = df['frames'].values[0] self.INFO['stop_frame'] = df['frames'].values[-1] try: self.INFO['duration_time'] = self.INFO['start_time'] - self.INFO[ 'stop_time'] self.INFO['duration_frame'] = int(df['frames'].values[-1]) - int( df['frames'].values[0]) except (TypeError, ValueError) as e: return False self.INFO['x_min'] = min( min(df[an + '_x'].values) for an in self.INFO['agent_names']) self.INFO['y_min'] = min( min(df[an + '_y'].values) for an in self.INFO['agent_names']) self.INFO['x_max'] = max( max(df[an + '_x'].values) for an in self.INFO['agent_names']) self.INFO['y_max'] = max( max(df[an + '_y'].values) for an in self.INFO['agent_names']) self.update_labels() self.DataLoaded = True
def on_pushed_inspect(self): '''When the inspect Button is clicked this function cuts the data accoring to the current temporal and spatial borders. Then the data is prepared for plotting depending on the chosen format finally plotWindow is called for display.''' if self.DataLoaded == False: messages.send_warning('No File loaded') return df = basic_stats.cut_timelines(self.TMP_FILE, self.INFO, self.CSV_INFO_FILE) representation = str(self.selectType.currentText()) datatype = str(self.selectSpec.currentText()) data2plot = genPlotData.makeData2Plot(df, representation, datatype, self.INFO, self.PARAM_INFO_FILE) if data2plot == None: messages.send_warning('selection invalid') return else: self.pW = plotWindow(data2plot, representation, datatype) self.pW.exec_()
def save(self): ''' gets called by the save Button. Checks if selected columns are valid, if so, data is saved to a temporary csv file otherwise a warning is sent''' print(self.checkLabels) valid = self.check_entries() if valid: if self.build_csv(self.fileName) == False: return self.parentWindow.INFO['agent_names'] = self.AGENT_NAMES basic_stats.speed_and_dist(self.TMP_FILE_TITLE, self.parentWindow.INFO, self.CSV_INFO_FILE, self.PARAM_INFO_FILE) self.parentWindow.INFO['info'] = self.PARAM_INFO if self.parentWindow.init_Info(self.TMP_FILE_TITLE) == False: return #basic_stats.speed_and_dist(self.TMP_FILE_TITLE, self.parentWindow.INFO, self.CSV_INFO_FILE, self.PARAM_INFO_FILE) self.home.close() else: messages.send_warning( 'Column selection invalid: Check for empty fields, double indices and indices exceeding size of original file' )
def apply_smoothing(self): ''' If a filter is selected in the dropdown menu and the 'Apply Smoothing' button is pressed, the respective smoothing function is applied on the whole trajectory (not only the selected parts). ''' if self.DataLoaded == False: messages.send_warning('No File loaded') return smooth = str(self.selectSmoothing.currentText()) if smooth == None: pass else: csv_dict = json.load(open(self.CSV_INFO_FILE)) delim = csv_dict['write']['delim'] df = pd.read_csv(self.TMP_FILE, header=0, sep=delim) if smooth == 'MedFilter, k=5': for an in self.INFO['agent_names']: df[an + '_x'] = smoothing.medfilt(df[an + '_x'].values) df[an + '_y'] = smoothing.medfilt(df[an + '_y'].values) messages.send_info('Trajectory is now smooth !') self.INFO['filtered'] = True
def changeCoords(self): ''' when the 'Change'Button is clicked on the space Layout, this function changes the labels for x and y borders to Line edits where individula changes can be made. The label of the 'Change' Button is set to ok and when presed, the line edits are switched back to labels and the selected coordinate values are save d to the INFO dictionary.''' if self.DataLoaded == False: messages.send_warning('No File loaded') return for key in self.Border_sizes: self.INFO[key] = np.round(float(self.Border_sizes[key].text()), 2) self.spaceLayout.removeWidget(self.Border_sizes[key]) sip.delete(self.Border_sizes[key]) self.Border_sizes = {} if self.changeCoordsButton.text() == 'Change': for i, b in enumerate(self.WORLD_BORDERS): le = QtWidgets.QLineEdit(str(self.INFO[b])) le.setValidator(QtGui.QDoubleValidator()) self.Border_sizes[b] = le self.spaceLayout.addWidget(le, np.floor(i / 2.) + 1, (i % 2) * 2 + 1) self.changeCoordsButton.setText('Ok') elif self.changeCoordsButton.text() == 'Ok': self.checkSubregions() for i, b in enumerate(self.WORLD_BORDERS): l = QtWidgets.QLabel(str(self.INFO[b])) self.Border_sizes[b] = l self.spaceLayout.addWidget(l, np.floor(i / 2.) + 1, (i % 2) * 2 + 1) self.changeCoordsButton.setText('Change')
def on_browse_clicked(self): ''' When the 'Browse' Button is clicked this function opens a File selection dialog, checks wheteher the selected file is of correct type (csv) and writes the filepath to the upper line edit. Additionally the path to the selected datafile will be a added to the options dictionary, containing default paths. ''' options = json.load(open(self.OPTIONS_INFO_FILE)) data_folder = options['data_folder'] if len(data_folder) == 0: data_folder = os.getenv("HOME") data_name = QtWidgets.QFileDialog.getOpenFileName( self, 'Open File', data_folder)[0] options['data_folder'] = data_name[:data_name.rfind('/')] with open(self.OPTIONS_INFO_FILE, 'w') as op: json.dump(options, op) if data_name.find('csv') < 0: messages.send_warning( 'Not a valid file type. Please use a file of type .csv .') else: self.selectedFile.setText(data_name) self.INFO['data_file'] = data_name print(self.INFO['data_file'])
def pushed_ok(self): ''' the subregion borders from line edit elements, checks for validity (no empty fields, not exceeding world borders, min < max, ...) and dumps the results to the subregion file. Note that overlapping subregions are permitted. ''' self.subRegions = {} for l in self.labels: subreg = l.objectName() self.subRegions[subreg] = {} for x in self.Xs: child = self.home.findChild(QtWidgets.QLineEdit, subreg + x) self.subRegions[subreg][x] = child.text() print(self.subRegions) for key in self.subRegions: for x in self.Xs: try: self.subRegions[key][x]= float(self.subRegions[key][x]) # check if all selcted fields are selected except ValueError: messages.send_warning("{} has missing or invalid values".format(key)) return # check if selected subregions fall into world boundaries if (x.find('min') > -1) and (self.subRegions[key][x] < self.superRegion[x]): messages.send_warning("Borders of {} exceed world boundaries".format(key)) return elif (x.find('max') > -1) and (self.subRegions[key][x] > self.superRegion[x]): messages.send_warning("Borders of {} exceed world boundaries".format(key)) return # check if selected subregions are self consistent i.e. always min < max if (self.subRegions[key]['x_min'] > self.subRegions[key]['x_max']) or \ (self.subRegions[key]['y_min'] > self.subRegions[key]['y_max']): messages.send_warning("Borders of {} are inconsistent".format(key)) return # save results to json and close application with open(self.subregions_file, 'w') as sf: json.dump(self.subRegions, sf) self.home.close()
def open_data(self): ''' uses the parameters from self.CSV_INFO_FILE and tries to open the datafile with pandas. If NaN values are detected, the user is informed and the corresponding rows will be ignored in processing. If an error occurs while opening with pandas a warning is sent''' csv_dict = json.load(open(self.CSV_INFO_FILE)) try: delim = csv_dict['read']['delim'] skip_rows = csv_dict['read']['skip_rows'] comment = csv_dict['read']['comment'] df = pd.read_csv(self.fileName, sep=delim, comment=comment, skiprows=skip_rows) vals = df.count(axis=1).values # number of non-NaN values per row maxx = df.count( axis=1).max() # maximum number of non-NaN values per row delete_rows = len(np.where(vals < maxx)[0]) messages.send_info_detail( '(1)To speed up the display process only the first 100 rows of data will be shown. \n(2)NaN values have been detected in {} rows. These rows will be ignored in the following process.' .format(str(delete_rows)), detail='Indices of ignored rows: {}'.format( np.where(vals < maxx)[0])) df = df.dropna(how='any') df = df.head(100) except pd.errors.ParserError: messages.send_warning( "There seems to be a problem with the file you're trying to open.\n\n \ This is usually due to missing values. Please delete incomplete rows and try again." ) return self.close() return df
def stats_and_save(self): ''' when clickig the 'Stats and Save' Button this function calles the stats and save method from basic stats which creates two csv files: one with timelines and another with single values (like mean, var, borders, etc.). Then a File selection dialog is opened and at the chosen location a folder is created where the results are saved. Finally a 'goodbye' dialog is sent informing the user where results were saved and asking wheter they want to continue or close the application''' if self.DataLoaded == False: messages.send_warning('No File loaded') return # This is simply to show the bar # self.progressBar.setGeometry(30, 40, 200, 25) # self.progressBar.setValue(0) # self.statusBar().addPermanentWidget(self.progressBar) #------------------------------------------------------------------------------------------- # saving preparation section #------------------------------------------------------------------------------------------- #check if subregions are in agreement with current world_borders if self.checkSubregions() == False: return options = json.load(open(self.OPTIONS_INFO_FILE)) # read prefered save folder from options file, default is HOME save_folder = options['save_folder'] if len(save_folder) == 0: save_folder = os.getenv("HOME") # open prefered save folder (or HOME) and let user confirm results_folder_super = \ str(QtWidgets.QFileDialog.getExistingDirectory(self, "Select Directory", save_folder, QtWidgets.QFileDialog.ShowDirsOnly)) if results_folder_super == None or len(results_folder_super) == 0: print('saving procedure interrupted') return options['save_folder'] = results_folder_super # save new prefered save folder with open(self.OPTIONS_INFO_FILE, 'w') as of: json.dump(options, of) # create results folder within save folder results_folder = self.makeResultsDir(results_folder_super + '/') # read parameters for saving from options file csv_dict = json.load(open(self.CSV_INFO_FILE)) delim = csv_dict['write']['delim'] time_file = options['timeline_file'] #------------------------------------------------------------------------------------------- # create timelines.csv and info.csv #------------------------------------------------------------------------------------------- # cut data dataframe according to current time and space settings df = basic_stats.cut_timelines(self.TMP_FILE, self.INFO, self.CSV_INFO_FILE) # sort the columns of dataframe indiv_stats = ['_vx', '_vy', '_speed'] coll_stats = ['_dist'] time = ['frames', 'time', 'seconds'] agents = self.INFO['agent_names'] specs = ['_x', '_y', '_angle'] cols = df.columns new_order = [] for t in time: new_order.append(t) for a in agents: for sp in specs: new_order.append(a + sp) for in_st in indiv_stats: new_order.append(a + in_st) for c in coll_stats: for col in cols: if col.find(c) > 0: new_order.append(col) df = df.reindex(new_order, axis=1) # save ordered dataframe and create info.csv df.to_csv(results_folder + '/' + time_file, sep=delim) #genStats.makeFile(results_folder, results_folder + '/' + time_file, self.INFO, self.CSV_INFO_FILE, self.OPTIONS_INFO_FILE) genStats.makeFile(results_folder, df, self.INFO, self.CSV_INFO_FILE, self.OPTIONS_INFO_FILE) #self.progressBar.setValue(20) #------------------------------------------------------------------------------------------- # plotting section #------------------------------------------------------------------------------------------- plot_instructions = options['plot_selection'] pf.plot_things(df, results_folder, self.INFO['agent_names'], plot_instructions) #self.progressBar.setValue(30) #------------------------------------------------------------------------------------------- # TE section #------------------------------------------------------------------------------------------- if options['TE']: if len(self.INFO['agent_names']) == 2: te_params = json.load(open(self.TE_PARAMS_FILE)) TE_done = TE.TE(results_folder + '/' + time_file, results_folder, '/TE.csv', '/TE.jpg', start_frame=te_params['start_frame'], maxtime=te_params['max_time'], k_te=te_params['k_te'], frame_step=te_params['frame_step']) else: messages.send_warning( 'Cannot calculate transfer entropy for agent number other than 2.' ) #self.progressBar.setValue(70) #------------------------------------------------------------------------------------------- # Subregions section #------------------------------------------------------------------------------------------- sub_info = {'start_frame':self.INFO['start_frame'], 'stop_frame':self.INFO['stop_frame'],\ 'start_time':self.INFO['start_time'], 'stop_time':self.INFO['stop_time'],\ 'agent_names':self.INFO['agent_names'], 'data_file':self.INFO['data_file'], 'filtered':self.INFO['filtered']} subregions = json.load(open(self.SUBREGIONS_INFO_FILE)) for sub in subregions.keys(): for wb in self.WORLD_BORDERS: sub_info[wb] = subregions[sub][wb] sub_df = basic_stats.cut_timelines(self.TMP_FILE, sub_info, self.CSV_INFO_FILE) genStats.makeFile(os.getcwd(), sub_df, sub_info, self.CSV_INFO_FILE, self.OPTIONS_INFO_FILE, region=sub) info1 = pd.read_csv(results_folder + '/info.csv') info2 = pd.read_csv('info.csv') infofiles = [info1, info2] combined = pd.concat(infofiles) combined = combined[info1.columns] combined = combined.dropna(axis=1) combined.to_csv(results_folder + '/info.csv') #self.progressBar.setValue(100) # from scipy.stats import spearmanr # from scipy.stats import pearsonr # for i in range(len(self.INFO['agent_names'])): # for j in range(i+1, len(self.INFO['agent_names'])): # a0 = self.INFO['agent_names'][i] # a1 = self.INFO['agent_names'][j] # # # print(a0 + ''+ a1 ) # print('Pearson Corr Speed: ', pearsonr(df[a0 + '_speed'].values, df[a1 + '_speed'].values)) # print('Pearson Corr vx: ', pearsonr(df[a0 + '_vx'].values, df[a1 + '_vx'].values)) # print('Pearson Corr vy: ', pearsonr(df[a0 + '_vy'].values, df[a1 + '_vy'].values)) # print('Np.Corr Speed: ', np.correlate(df[a0 + '_speed'].values, df[a1 + '_speed'].values)) # # print('Pearson x: ', pearsonr(df[a0 + '_x'].values, df[a1 + '_x'].values)) # print('Np.Corr x: ', np.correlate(df[a0 + '_x'].values, df[a1 + '_x'].values)) messages.send_goodbye(self, results_folder)
def on_load_clicked(self): ''' When the 'Load' Button is clicked the data file specified in the line edit nest to the Browse Buttonis loaded for further processing. If view is checked this is done via the display in table window. Else the loading is done internally. In any case csv parameters are read from the respective line edits in mainWindow and and saved to json for further use.''' # check if file was selected if len(self.selectedFile.text()) == 0: messages.send_warning('No File loaded') return # read csv parameters and save to json csv_dict = json.load(open(self.CSV_INFO_FILE)) csv_dict['read']['delim'] = self.setDelim.text() csv_dict['read']['skip_rows'] = int(self.setSkipRows.text()) with open(self.CSV_INFO_FILE, 'w') as fp: json.dump(csv_dict, fp) # save the state of the view button options = json.load(open(self.OPTIONS_INFO_FILE)) options['view'] = self.openInView.checkState() print(options) with open(self.OPTIONS_INFO_FILE, 'w') as fp: json.dump(options, fp) # if view is checked open tableWindow if self.openInView.checkState() == 2: self.table = tableWindow(self, self.INFO['data_file']) self.table.show() else: #------------------------------------------------------------------------------------------- # in this section the file is loaded using settings from previous uses. The procedure is # similar to tableWindow's build_csv()-function. #------------------------------------------------------------------------------------- ----- # use column labels from previos session columns = json.load(open(self.PARAM_INFO_FILE)) time_labels = columns['time_labels'] agent_names = columns['agent_names'] agent_specs = columns['agent_specifications'] other = columns['other'] # use csv params from mainWindow to load selected csv into pandas try: df = pd.read_csv(self.selectedFile.text(), header=0, sep=csv_dict['read']['delim'], skiprows=csv_dict['read']['skip_rows'], comment=csv_dict['read']['comment']) except ValueError: messages.send_warning( "Something went wrong ! \nPlease enable 'view' and try again." ) return # define columns df_columns = { 'TIME': { time_labels[i]: columns[time_labels[i]] for i in range(len(time_labels)) }, 'AGENTS': {}, 'OTHER': {other[i]: columns[other[i]] for i in range(len(other))} } for k in range(len(agent_names)): for j in range(len(agent_specs)): key = agent_names[k] + agent_specs[j] df_columns['AGENTS'][key] = columns[key] # read the indices and names of columns of interest into a list real_indices = [] titles = [] for key1 in df_columns.keys(): for key2 in df_columns[key1].keys(): titles.append(key2) real_indices.append(int(df_columns[key1][key2]) - 1) # try to select the columns of interest in the loaded pandas try: df_new = df.iloc[:, real_indices] except IndexError: messages.send_warning( "Something went wrong ! \nPlease enable 'view' and try again." ) return # check for NaN values and delete the respective rows df_new.columns = titles df_new = df_new.dropna(how='any') # save pandas to temporary csv file df_new.to_csv(self.TMP_FILE, sep=csv_dict['write']['delim']) print('temporary file saved to', self.TMP_FILE) # perform basic_stats i.e convert time to seconds and calc speed and dist self.INFO['agent_names'] = agent_names try: basic_stats.speed_and_dist(self.TMP_FILE, self.INFO, self.CSV_INFO_FILE, self.PARAM_INFO_FILE) except TypeError: messages.send_warning( "Something went wrong ! \nPlease enable 'view' and try again." ) return # init the display of mainWindow and self.INFO with values read from datafile self.INFO['info'] = columns['info'] if self.init_Info(self.TMP_FILE) == False: messages.send_warning( "Something went wrong ! \nPlease enable 'view' and try again." ) return
def build_csv(self, fileName): ''' uses the selected columns to build a temporary pandas frame which is saved to .csv under a default name.''' header_dict = {} csv_dict = json.load(open(self.CSV_INFO_FILE)) delim = csv_dict['read']['delim'] skip_rows = csv_dict['read']['skip_rows'] comment = csv_dict['read']['comment'] for key in self.checkLabels.keys(): for k in self.checkLabels[key].keys(): header_dict[k] = self.checkLabels[key][k] df = pd.read_csv(fileName, header=None, sep=delim, skiprows=skip_rows, comment=comment, names=[str(i) for i in range(self.nColumns)]) real_indices = [str(int(cl) - 1) for cl in header_dict.values()] df_new = df.loc[:, real_indices] df_new.columns = list(header_dict.keys()) df_new = df_new.dropna(how='any') #print(header_dict) if type(df_new['frames'].values[0]) == str: #drop original header df_new = df_new.drop(df.index[0]) # check for invalid values in selcted columns should_be_floats = ['_x', '_y', '_angle'] for an in self.AGENT_NAMES: for sbf in should_be_floats: if not self.is_float(df_new[an + sbf].values[3]): messages.send_warning( 'Data type of column {} not understood'.format(an + sbf)) return False if not self.is_float(df_new['frames'].values[3]): messages.send_warning('Data type of column frames not understood') return False # save to tmp delim = csv_dict['write']['delim'] df_new.to_csv(self.TMP_FILE_TITLE, sep=delim) print('temporary file saved to', self.TMP_FILE_TITLE) #save paramter settings to json param_dict = {} param_dict['agent_names'] = self.AGENT_NAMES param_dict['time_labels'] = self.TIME_LABELS param_dict['agent_specifications'] = self.AGENT_DATA param_dict['other'] = self.OTHER param_dict['info'] = self.PARAM_INFO for key1 in self.checkLabels: for key2 in self.checkLabels[key1]: param_dict[key2] = self.checkLabels[key1][key2] with open(self.PARAM_INFO_FILE, 'w') as fp: json.dump(param_dict, fp)