コード例 #1
0
ファイル: handlers_backend.py プロジェクト: nodirt/luci-py
 def get(self):
   self.response.headers['Content-Type'] = 'text/plain'
   minutes = stats.generate_stats()
   if minutes is not None:
     msg = 'Processed %d minutes' % minutes
     logging.info(msg)
     self.response.write(msg)
コード例 #2
0
ファイル: stats_test.py プロジェクト: stefb965/luci-py
    def _test_handler(self, url, added_data):
        stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)

        self.assertEqual('Yay', self.app.get(url).body)
        self.assertEqual(1, len(list(stats_framework.yield_entries(None,
                                                                   None))))

        self.mock_now(self.now, 60)
        self.assertEqual(10, stats.generate_stats())

        actual = stats_framework.get_stats(stats.STATS_HANDLER, 'minutes',
                                           self.now, 1, True)
        expected = [
            {
                'contains_lookups': 0,
                'contains_requests': 0,
                'downloads': 0,
                'downloads_bytes': 0,
                'failures': 0,
                'key': datetime.datetime(2010, 1, 2, 3, 4),
                'other_requests': 0,
                'requests': 1,
                'uploads': 0,
                'uploads_bytes': 0,
            },
        ]
        expected[0].update(added_data)
        self.assertEqual(expected, actual)
コード例 #3
0
 def get(self):
     self.response.headers['Content-Type'] = 'text/plain'
     minutes = stats.generate_stats()
     if minutes is not None:
         msg = 'Processed %d minutes' % minutes
         logging.info(msg)
         self.response.write(msg)
コード例 #4
0
    def build_dataset(self, msg_queue):
        save_func = save.get_save_func(self.final["DATASET_FILE_EXTENSION"])

        msg_queue.put("Loading datasets. This may take a while. \n")
        # Load individual datasests
        datasets = loader.load_data(self.final["CONFIGS"], msg_queue)
        msg_queue.put("Finished loading datasets! \n" + "-" * 50 + "\n")
        msg_queue.put("Constructing new dataset \n")
        # Construct new dataset
        training_set, test_set = construct.get_subset(datasets, self.final)
        msg_queue.put("Finished Constructing datasets! \n" + "-" * 50 + "\n")

        msg_queue.put("Saving dataset \n")
        save_path = self.get_save_path(self.final["DATASET_NAME"] +
                                       "_training_set")
        save_func(save_path, training_set)

        save_path = self.get_save_path(self.final["DATASET_NAME"] +
                                       "_test_set")
        save.save_multiple(test_set, save_path, save_func)
        msg_queue.put("Finished saving dataset \n" + "-" * 50 + "\n")

        msg = "Do you want to save dataset statistics ?"
        if messagebox.askyesno("SAVE", msg):
            msg_queue.put("Saving dataset statistics \n")
            # Save statistics of each individual dataset
            datasets_stats = stats.generate_stats(test_set)
            save_path = self.get_save_path("test_set_stats")
            save_func(save_path,
                      datasets_stats,
                      fieldnames=datasets_stats[0].keys())
            msg_queue.put("Finished saving statistics \n" + "-" * 50 + "\n")

        self.manager.update()
コード例 #5
0
ファイル: stats_test.py プロジェクト: misscache/luci-py
  def _test_handler(self, url, added_data):
    stats_framework_mock.reset_timestamp(stats.STATS_HANDLER, self.now)

    self.assertEqual('Yay', self.app.get(url).body)
    self.assertEqual(1, len(list(stats_framework.yield_entries(None, None))))

    self.mock_now(self.now, 60)
    self.assertEqual(10, stats.generate_stats())

    actual = stats_framework.get_stats(
        stats.STATS_HANDLER, 'minutes', self.now, 1, True)
    expected = [
      {
        'contains_lookups': 0,
        'contains_requests': 0,
        'downloads': 0,
        'downloads_bytes': 0,
        'failures': 0,
        'key': datetime.datetime(2010, 1, 2, 3, 4),
        'other_requests': 0,
        'requests': 1,
        'uploads': 0,
        'uploads_bytes': 0,
      },
    ]
    expected[0].update(added_data)
    self.assertEqual(expected, actual)
コード例 #6
0
def parse_dir(root_dir, fields_list, exclusion_list, sorting_order):
    counter = 0
    for root, dirs, files in os.walk(root_dir, topdown=True):
        dirs = [d for d in dirs if re.match(r'0\d - ', d)]
        folders = dirs
        for dir in folders:
            counter += 1
            os.chdir(os.path.join(root, dir))
            cd = os.getcwd()
            print('Dir:' + str(counter))
            print(cd)
            contents = os.listdir(cd)
            if not os.path.exists(cd + '\Reporting'):
                os.mkdir(cd + '\Reporting')
            folder_path = cd + '\Reporting\\'
            files = [f for f in contents if re.match(r'aggregate', f)]
            aggfiles = []
            for file in files:
                print('Reading file: ' + str(file))
                inputFile = pd.DataFrame(pd.read_csv(file))
                aggfiles.append(inputFile)
            aggfile = ''
            if len(aggfiles) > 1:
                aggfile = st.combine_data(aggfiles)
            else:
                aggfile = aggfiles[0]
            # MODIFYING DATAFRAME
            print ('Removing unwanted fields')
            st.unwanted_fields(aggfile)
            # STATS
            print('Creating stats')
            aggfile = st.generate_stats(aggfile, folder_path, fields_list)
            # FILTER
            print('Filtering data')
            aggfile = st.filter_agg_stats(aggfile, fields_list, exclusion_list)
            cols = aggfile.columns.tolist()
            while cols[0] != 'label':
                cols = cols[-1:] + cols[:-1]
            aggfile = aggfile[cols]
            aggfile['label'] = pd.Categorical(aggfile['label'], sorting_order, ordered=True)
            aggfile = aggfile.sort_values('label')
            aggfile.to_csv(folder_path + '\stats.csv', index=False)
            if 'median' in fields_list and 'samples' in fields_list and '90th' in fields_list and '95th' in fields_list:
                # HISTOGRAM
                print('Creating histogram')
                labelword = aggfile['label']
                ninetyline = aggfile['90th']
                ninetyfiveline = aggfile['95th']
                labelword = labelword.tolist()
                hg.histogram(labelword, ninetyline, ninetyfiveline, folder_path)
            else:
                print('Histogram could not be generated because the required values weren\'t requested')