class TestGatherCrawlOutliers(unittest.TestCase): def create_file_path(self, file_name): path = os.path.join(self.data_file_path, file_name) return path def setUp(self): arguments = parse_arguments(['-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800', '-s', '10']) self.overwatch = Overwatch(arguments) self.data_file_path = os.path.join(os.getcwd(), 'test_data') self.json_file_name = 'scrapyd_list_jobs_for_loop_json.json' self.json_path = self.create_file_path(self.json_file_name) self.scrapyd_json = open(self.json_path, 'rb').read() json_dict = json.loads(self.scrapyd_json) self.session = requests.Session() self.adapter = requests_mock.Adapter() self.session.mount('mock', self.adapter) self.adapter.register_uri( 'GET', 'mock://0.0.0.1:6800/listjobs.json?project=harvestman', json=json_dict, status_code=200, ) self.overwatch.response = self.session.get( 'mock://0.0.0.1:6800/listjobs.json?project=harvestman') self.outliers = self.overwatch.gather_crawl_outliers() def test_gather_crawl_outliers(self): expted = {'strt': datetime.datetime(2016, 4, 1, 1, 1, 59, 999999), 'end': datetime.datetime(2016, 4, 1, 23, 59, 59, 999999)} self.assertEqual(self.outliers, expted)
class TestGatherCrawlOutliers(unittest.TestCase): def create_file_path(self, file_name): path = os.path.join(self.data_file_path, file_name) return path def setUp(self): arguments = parse_arguments([ '-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800', '-s', '10' ]) self.overwatch = Overwatch(arguments) self.data_file_path = os.path.join(os.getcwd(), 'test_data') self.json_file_name = 'scrapyd_list_jobs_for_loop_json.json' self.json_path = self.create_file_path(self.json_file_name) self.scrapyd_json = open(self.json_path, 'rb').read() json_dict = json.loads(self.scrapyd_json) self.session = requests.Session() self.adapter = requests_mock.Adapter() self.session.mount('mock', self.adapter) self.adapter.register_uri( 'GET', 'mock://0.0.0.1:6800/listjobs.json?project=harvestman', json=json_dict, status_code=200, ) self.overwatch.response = self.session.get( 'mock://0.0.0.1:6800/listjobs.json?project=harvestman') self.outliers = self.overwatch.gather_crawl_outliers() def test_gather_crawl_outliers(self): expted = { 'strt': datetime.datetime(2016, 4, 1, 1, 1, 59, 999999), 'end': datetime.datetime(2016, 4, 1, 23, 59, 59, 999999) } self.assertEqual(self.outliers, expted)
class TestOverwatch(unittest.TestCase): def create_file_path(self, file_name): path = os.path.join(self.data_file_path, file_name) return path def setUp(self): today = datetime.datetime.today().strftime('%d-%m-%Y') arguments = parse_arguments(['-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800', '-s', '50']) self.overwatch = Overwatch(arguments) self.exp_url = 'http://192.168.124.30:6800/listjobs.json?project=harvestman' filename = '{}_{}.csv'.format(today, self.overwatch.arguments.project_name[0]) self.output_file = os.path.join(settings.OUTPUT_PATH, filename) self.temp_dir = tempfile.mkdtemp(prefix='temp_test_dir') self.data_file_path = os.path.join(os.getcwd(), 'test_data') self.json_outliers_file_name = 'scrapyd_list_jobs_outliers_json.json' self.scrapyd_outliers_json_path = self.create_file_path( self.json_outliers_file_name) self.scrapyd_outliers_json = open(self.scrapyd_outliers_json_path, 'rb').read() outliers_json_dict = json.loads(self.scrapyd_outliers_json) self.session = requests.Session() self.adapter = requests_mock.Adapter() self.session.mount('mock', self.adapter) self.adapter.register_uri( 'GET', 'mock://0.0.0.1:6800/listjobs.json?project=harvestman', json=outliers_json_dict, status_code=200, ) self.overwatch.response = self.session.get( 'mock://0.0.0.1:6800/listjobs.json?project=harvestman') def test__init__(self): self.assertEqual(self.overwatch.query_url, self.exp_url) self.assertEqual(self.overwatch.con_spiders, 50) self.assertEqual(self.overwatch.output_file, self.output_file) def test_gather_crawl_outliers(self): expected = {'strt': datetime.datetime(2016, 4, 29, 10, 28, 8, 4732), 'end': datetime.datetime(2016, 4, 29, 10, 33, 51, 420786) } self.assertEqual(self.overwatch.gather_crawl_outliers(), expected) def test_gather_completed_crawl_count(self): self.assertEqual(self.overwatch.gather_completed_crawl_count(), 3) def test_calculate_total_duration(self): expected = 343.416054 self.assertEqual(self.overwatch.calculate_total_duration(), expected) def test_gather_crawl_durations(self): expected = [241.547793, 233.489018, 258.652448] self.assertEqual(self.overwatch.gather_crawl_durations(), expected) def test_calculate_av_crawl_duration(self): expected = 244.56 self.assertEqual(self.overwatch.calculate_av_crawl_duration(), expected) def test_calculate_single_crawls_per_hour(self): expected = 14.72 self.assertEqual(self.overwatch.calculate_single_crawls_per_hour(), expected) def test_calculate_est_total_crawls_per_hour(self): expected = 736 self.assertEqual(self.overwatch.calculate_est_total_crawls_per_hour(), expected) def test_calculate_single_crawls_per_day(self): expected = 353.28 self.assertEqual(self.overwatch.calculate_single_crawls_per_day(), expected) def test_calculate_est_total_crawls_per_day(self): expected = 17664 self.assertEqual(self.overwatch.calculate_est_total_crawls_per_day(), expected) def test_calculate_single_crawls_per_week(self): expected = 2472.96 self.assertEqual(self.overwatch.calculate_single_crawls_per_week(), expected) def test_calculate_est_total_crawls_per_week(self): expected = 123648 self.assertEqual(self.overwatch.calculate_est_total_crawls_per_week(), expected) def test_gather_scrapy_metrics(self): expected = { 'Av CR (S)': 244.56, 'Longest CR (S)': 258.652448, 'Shortest CR (S)': 233.489018, 'Total Duration': 343.416054, 'Single CR p/h': 14.72, 'Max CR p/h': 736, 'Single CR p/d': 353.28, 'Max CR p/d': 17664, 'Single CR p/7d': 2472.96, 'Max CR p/7d': 123648, 'Completed crawls': 3 } self.assertEqual(self.overwatch.gather_scrapy_metrics(), expected) def test_write_to_csv(self): filename = 'test_ouput.csv' self.overwatch.output_file = os.path.join(self.temp_dir, filename) os.mknod(self.overwatch.output_file) self.overwatch.write_to_csv() with open(self.overwatch.output_file, 'rb') as csvfile: reader = csv.DictReader(csvfile) csv_data = [row for row in reader] self.assertEqual(csv_data[0]['Total Duration'], '343.416054') self.assertEqual(csv_data[0]['Completed crawls'], '3')
class TestOverwatch(unittest.TestCase): def create_file_path(self, file_name): path = os.path.join(self.data_file_path, file_name) return path def setUp(self): today = datetime.datetime.today().strftime('%d-%m-%Y') arguments = parse_arguments([ '-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800', '-s', '50' ]) self.overwatch = Overwatch(arguments) self.exp_url = 'http://192.168.124.30:6800/listjobs.json?project=harvestman' filename = '{}_{}.csv'.format(today, self.overwatch.arguments.project_name[0]) self.output_file = os.path.join(settings.OUTPUT_PATH, filename) self.temp_dir = tempfile.mkdtemp(prefix='temp_test_dir') self.data_file_path = os.path.join(os.getcwd(), 'test_data') self.json_outliers_file_name = 'scrapyd_list_jobs_outliers_json.json' self.scrapyd_outliers_json_path = self.create_file_path( self.json_outliers_file_name) self.scrapyd_outliers_json = open(self.scrapyd_outliers_json_path, 'rb').read() outliers_json_dict = json.loads(self.scrapyd_outliers_json) self.session = requests.Session() self.adapter = requests_mock.Adapter() self.session.mount('mock', self.adapter) self.adapter.register_uri( 'GET', 'mock://0.0.0.1:6800/listjobs.json?project=harvestman', json=outliers_json_dict, status_code=200, ) self.overwatch.response = self.session.get( 'mock://0.0.0.1:6800/listjobs.json?project=harvestman') def test__init__(self): self.assertEqual(self.overwatch.query_url, self.exp_url) self.assertEqual(self.overwatch.con_spiders, 50) self.assertEqual(self.overwatch.output_file, self.output_file) def test_gather_crawl_outliers(self): expected = { 'strt': datetime.datetime(2016, 4, 29, 10, 28, 8, 4732), 'end': datetime.datetime(2016, 4, 29, 10, 33, 51, 420786) } self.assertEqual(self.overwatch.gather_crawl_outliers(), expected) def test_gather_completed_crawl_count(self): self.assertEqual(self.overwatch.gather_completed_crawl_count(), 3) def test_calculate_total_duration(self): expected = 343.416054 self.assertEqual(self.overwatch.calculate_total_duration(), expected) def test_gather_crawl_durations(self): expected = [241.547793, 233.489018, 258.652448] self.assertEqual(self.overwatch.gather_crawl_durations(), expected) def test_calculate_av_crawl_duration(self): expected = 244.56 self.assertEqual(self.overwatch.calculate_av_crawl_duration(), expected) def test_calculate_single_crawls_per_hour(self): expected = 14.72 self.assertEqual(self.overwatch.calculate_single_crawls_per_hour(), expected) def test_calculate_est_total_crawls_per_hour(self): expected = 736 self.assertEqual(self.overwatch.calculate_est_total_crawls_per_hour(), expected) def test_calculate_single_crawls_per_day(self): expected = 353.28 self.assertEqual(self.overwatch.calculate_single_crawls_per_day(), expected) def test_calculate_est_total_crawls_per_day(self): expected = 17664 self.assertEqual(self.overwatch.calculate_est_total_crawls_per_day(), expected) def test_calculate_single_crawls_per_week(self): expected = 2472.96 self.assertEqual(self.overwatch.calculate_single_crawls_per_week(), expected) def test_calculate_est_total_crawls_per_week(self): expected = 123648 self.assertEqual(self.overwatch.calculate_est_total_crawls_per_week(), expected) def test_gather_scrapy_metrics(self): expected = { 'Av CR (S)': 244.56, 'Longest CR (S)': 258.652448, 'Shortest CR (S)': 233.489018, 'Total Duration': 343.416054, 'Single CR p/h': 14.72, 'Max CR p/h': 736, 'Single CR p/d': 353.28, 'Max CR p/d': 17664, 'Single CR p/7d': 2472.96, 'Max CR p/7d': 123648, 'Completed crawls': 3 } self.assertEqual(self.overwatch.gather_scrapy_metrics(), expected) def test_write_to_csv(self): filename = 'test_ouput.csv' self.overwatch.output_file = os.path.join(self.temp_dir, filename) os.mknod(self.overwatch.output_file) self.overwatch.write_to_csv() with open(self.overwatch.output_file, 'rb') as csvfile: reader = csv.DictReader(csvfile) csv_data = [row for row in reader] self.assertEqual(csv_data[0]['Total Duration'], '343.416054') self.assertEqual(csv_data[0]['Completed crawls'], '3')