def get(self):
self.response.headers['Content-Type'] = 'text/plain'
minutes = stats.generate_stats()
if minutes is not None:
msg = 'Processed %d minutes' % minutes
logging.info(msg)
self.response.write(msg)
def _test_handler(self, url, added_data):
stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)
self.assertEqual('Yay', self.app.get(url).body)
self.assertEqual(1, len(list(stats_framework.yield_entries(None,
None))))
self.mock_now(self.now, 60)
self.assertEqual(10, stats.generate_stats())
actual = stats_framework.get_stats(stats.STATS_HANDLER, 'minutes',
self.now, 1, True)
expected = [
{
'contains_lookups': 0,
'contains_requests': 0,
'downloads': 0,
'downloads_bytes': 0,
'failures': 0,
'key': datetime.datetime(2010, 1, 2, 3, 4),
'other_requests': 0,
'requests': 1,
'uploads': 0,
'uploads_bytes': 0,
},
]
expected[0].update(added_data)
self.assertEqual(expected, actual)
def get(self):
self.response.headers['Content-Type'] = 'text/plain'
minutes = stats.generate_stats()
if minutes is not None:
msg = 'Processed %d minutes' % minutes
logging.info(msg)
self.response.write(msg)
def build_dataset(self, msg_queue):
save_func = save.get_save_func(self.final["DATASET_FILE_EXTENSION"])
msg_queue.put("Loading datasets. This may take a while. \n")
# Load individual datasests
datasets = loader.load_data(self.final["CONFIGS"], msg_queue)
msg_queue.put("Finished loading datasets! \n" + "-" * 50 + "\n")
msg_queue.put("Constructing new dataset \n")
# Construct new dataset
training_set, test_set = construct.get_subset(datasets, self.final)
msg_queue.put("Finished Constructing datasets! \n" + "-" * 50 + "\n")
msg_queue.put("Saving dataset \n")
save_path = self.get_save_path(self.final["DATASET_NAME"] +
"_training_set")
save_func(save_path, training_set)
save_path = self.get_save_path(self.final["DATASET_NAME"] +
"_test_set")
save.save_multiple(test_set, save_path, save_func)
msg_queue.put("Finished saving dataset \n" + "-" * 50 + "\n")
msg = "Do you want to save dataset statistics ?"
if messagebox.askyesno("SAVE", msg):
msg_queue.put("Saving dataset statistics \n")
# Save statistics of each individual dataset
datasets_stats = stats.generate_stats(test_set)
save_path = self.get_save_path("test_set_stats")
save_func(save_path,
datasets_stats,
fieldnames=datasets_stats[0].keys())
msg_queue.put("Finished saving statistics \n" + "-" * 50 + "\n")
self.manager.update()
def _test_handler(self, url, added_data):
stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)
self.assertEqual('Yay', self.app.get(url).body)
self.assertEqual(1, len(list(stats_framework.yield_entries(None, None))))
self.mock_now(self.now, 60)
self.assertEqual(10, stats.generate_stats())
actual = stats_framework.get_stats(
stats.STATS_HANDLER, 'minutes', self.now, 1, True)
expected = [
{
'contains_lookups': 0,
'contains_requests': 0,
'downloads': 0,
'downloads_bytes': 0,
'failures': 0,
'key': datetime.datetime(2010, 1, 2, 3, 4),
'other_requests': 0,
'requests': 1,
'uploads': 0,
'uploads_bytes': 0,
},
]
expected[0].update(added_data)
self.assertEqual(expected, actual)
def parse_dir(root_dir, fields_list, exclusion_list, sorting_order):
counter = 0
for root, dirs, files in os.walk(root_dir, topdown=True):
dirs = [d for d in dirs if re.match(r'0\d - ', d)]
folders = dirs
for dir in folders:
counter += 1
os.chdir(os.path.join(root, dir))
cd = os.getcwd()
print('Dir:' + str(counter))
print(cd)
contents = os.listdir(cd)
if not os.path.exists(cd + '\Reporting'):
os.mkdir(cd + '\Reporting')
folder_path = cd + '\Reporting\\'
files = [f for f in contents if re.match(r'aggregate', f)]
aggfiles = []
for file in files:
print('Reading file: ' + str(file))
inputFile = pd.DataFrame(pd.read_csv(file))
aggfiles.append(inputFile)
aggfile = ''
if len(aggfiles) > 1:
aggfile = st.combine_data(aggfiles)
else:
aggfile = aggfiles[0]
# MODIFYING DATAFRAME
print ('Removing unwanted fields')
st.unwanted_fields(aggfile)
# STATS
print('Creating stats')
aggfile = st.generate_stats(aggfile, folder_path, fields_list)
# FILTER
print('Filtering data')
aggfile = st.filter_agg_stats(aggfile, fields_list, exclusion_list)
cols = aggfile.columns.tolist()
while cols[0] != 'label':
cols = cols[-1:] + cols[:-1]
aggfile = aggfile[cols]
aggfile['label'] = pd.Categorical(aggfile['label'], sorting_order, ordered=True)
aggfile = aggfile.sort_values('label')
aggfile.to_csv(folder_path + '\stats.csv', index=False)
if 'median' in fields_list and 'samples' in fields_list and '90th' in fields_list and '95th' in fields_list:
# HISTOGRAM
print('Creating histogram')
labelword = aggfile['label']
ninetyline = aggfile['90th']
ninetyfiveline = aggfile['95th']
labelword = labelword.tolist()
hg.histogram(labelword, ninetyline, ninetyfiveline, folder_path)
else:
print('Histogram could not be generated because the required values weren\'t requested')