def get(self): self.response.headers['Content-Type'] = 'text/plain' minutes = stats.generate_stats() if minutes is not None: msg = 'Processed %d minutes' % minutes logging.info(msg) self.response.write(msg)
def _test_handler(self, url, added_data): stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now) self.assertEqual('Yay', self.app.get(url).body) self.assertEqual(1, len(list(stats_framework.yield_entries(None, None)))) self.mock_now(self.now, 60) self.assertEqual(10, stats.generate_stats()) actual = stats_framework.get_stats(stats.STATS_HANDLER, 'minutes', self.now, 1, True) expected = [ { 'contains_lookups': 0, 'contains_requests': 0, 'downloads': 0, 'downloads_bytes': 0, 'failures': 0, 'key': datetime.datetime(2010, 1, 2, 3, 4), 'other_requests': 0, 'requests': 1, 'uploads': 0, 'uploads_bytes': 0, }, ] expected[0].update(added_data) self.assertEqual(expected, actual)
def build_dataset(self, msg_queue): save_func = save.get_save_func(self.final["DATASET_FILE_EXTENSION"]) msg_queue.put("Loading datasets. This may take a while. \n") # Load individual datasests datasets = loader.load_data(self.final["CONFIGS"], msg_queue) msg_queue.put("Finished loading datasets! \n" + "-" * 50 + "\n") msg_queue.put("Constructing new dataset \n") # Construct new dataset training_set, test_set = construct.get_subset(datasets, self.final) msg_queue.put("Finished Constructing datasets! \n" + "-" * 50 + "\n") msg_queue.put("Saving dataset \n") save_path = self.get_save_path(self.final["DATASET_NAME"] + "_training_set") save_func(save_path, training_set) save_path = self.get_save_path(self.final["DATASET_NAME"] + "_test_set") save.save_multiple(test_set, save_path, save_func) msg_queue.put("Finished saving dataset \n" + "-" * 50 + "\n") msg = "Do you want to save dataset statistics ?" if messagebox.askyesno("SAVE", msg): msg_queue.put("Saving dataset statistics \n") # Save statistics of each individual dataset datasets_stats = stats.generate_stats(test_set) save_path = self.get_save_path("test_set_stats") save_func(save_path, datasets_stats, fieldnames=datasets_stats[0].keys()) msg_queue.put("Finished saving statistics \n" + "-" * 50 + "\n") self.manager.update()
def _test_handler(self, url, added_data): stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now) self.assertEqual('Yay', self.app.get(url).body) self.assertEqual(1, len(list(stats_framework.yield_entries(None, None)))) self.mock_now(self.now, 60) self.assertEqual(10, stats.generate_stats()) actual = stats_framework.get_stats( stats.STATS_HANDLER, 'minutes', self.now, 1, True) expected = [ { 'contains_lookups': 0, 'contains_requests': 0, 'downloads': 0, 'downloads_bytes': 0, 'failures': 0, 'key': datetime.datetime(2010, 1, 2, 3, 4), 'other_requests': 0, 'requests': 1, 'uploads': 0, 'uploads_bytes': 0, }, ] expected[0].update(added_data) self.assertEqual(expected, actual)
def parse_dir(root_dir, fields_list, exclusion_list, sorting_order): counter = 0 for root, dirs, files in os.walk(root_dir, topdown=True): dirs = [d for d in dirs if re.match(r'0\d - ', d)] folders = dirs for dir in folders: counter += 1 os.chdir(os.path.join(root, dir)) cd = os.getcwd() print('Dir:' + str(counter)) print(cd) contents = os.listdir(cd) if not os.path.exists(cd + '\Reporting'): os.mkdir(cd + '\Reporting') folder_path = cd + '\Reporting\\' files = [f for f in contents if re.match(r'aggregate', f)] aggfiles = [] for file in files: print('Reading file: ' + str(file)) inputFile = pd.DataFrame(pd.read_csv(file)) aggfiles.append(inputFile) aggfile = '' if len(aggfiles) > 1: aggfile = st.combine_data(aggfiles) else: aggfile = aggfiles[0] # MODIFYING DATAFRAME print ('Removing unwanted fields') st.unwanted_fields(aggfile) # STATS print('Creating stats') aggfile = st.generate_stats(aggfile, folder_path, fields_list) # FILTER print('Filtering data') aggfile = st.filter_agg_stats(aggfile, fields_list, exclusion_list) cols = aggfile.columns.tolist() while cols[0] != 'label': cols = cols[-1:] + cols[:-1] aggfile = aggfile[cols] aggfile['label'] = pd.Categorical(aggfile['label'], sorting_order, ordered=True) aggfile = aggfile.sort_values('label') aggfile.to_csv(folder_path + '\stats.csv', index=False) if 'median' in fields_list and 'samples' in fields_list and '90th' in fields_list and '95th' in fields_list: # HISTOGRAM print('Creating histogram') labelword = aggfile['label'] ninetyline = aggfile['90th'] ninetyfiveline = aggfile['95th'] labelword = labelword.tolist() hg.histogram(labelword, ninetyline, ninetyfiveline, folder_path) else: print('Histogram could not be generated because the required values weren\'t requested')