def test_retrieve_raw_with_interval_check_all_data_points_retrieved(self):
with temporary_directory() as tem_dir:
store = Store(directory=tem_dir, bucket_size=600)
for i in range(20):
store.record(name='foo',
timestamp=self.start + timedelta(minutes=60 * i),
data='pending:' + str(i))
store.record(name='foo',
timestamp=self.start + timedelta(minutes=60 * i),
data='pending:' + str(i))
records = list(
store.retrieve_raw('foo',
interval=Interval(
start=self.start,
delta=timedelta(hours=10))))
self.assertEqual(len(records), 11)
self.assertListEqual([
'1520080200.0 pending:0\n1520080200.0 pending:0\n',
'1520083800.0 pending:1\n1520083800.0 pending:1\n',
'1520087400.0 pending:2\n1520087400.0 pending:2\n',
'1520091000.0 pending:3\n1520091000.0 pending:3\n',
'1520094600.0 pending:4\n1520094600.0 pending:4\n',
'1520098200.0 pending:5\n1520098200.0 pending:5\n',
'1520101800.0 pending:6\n1520101800.0 pending:6\n',
'1520105400.0 pending:7\n1520105400.0 pending:7\n',
'1520109000.0 pending:8\n1520109000.0 pending:8\n',
'1520112600.0 pending:9\n1520112600.0 pending:9\n',
'1520116200.0 pending:10\n1520116200.0 pending:10\n'
], records)
def test_append_with_no_file_check_created(self):
with temporary_directory() as temp_dir:
buck, _ = self.make_buck(base=temp_dir)
self.assertFalse(os.path.exists(buck.full_path))
buck.append(123456, 'cpu:7,some:8.4')
self.assertTrue(os.path.exists(buck.full_path))
self.assertFalse(os.path.exists(buck.full_path))
def test_forget_interval_with_not_all_in_it_check_we_leave_stuff_from_days_not_in_interval(
self):
with temporary_directory() as tem_dir:
self.make_three_buckets(tem_dir=tem_dir)
store = Store(directory=tem_dir, bucket_size=600)
ts = self.start + timedelta(days=2)
store.record(name='some', timestamp=ts, data='pending:1')
ts += timedelta(seconds=1200)
store.record(name='some', timestamp=ts, data='pending:2')
ts += timedelta(hours=5)
store.record(name='some', timestamp=ts, data='pending:3')
base = os.path.join(tem_dir, 'some', '600', '2018_03_05')
files = list(os.listdir(base))
self.assertEqual(3, len(files))
content = [file_content(base, file) for file in files]
print(content)
self.assertListEqual(content, [
'1520272200.0 pending:3\n', '1520253000.0 pending:1\n',
'1520254200.0 pending:2\n'
])
to_remove = Interval(start=self.start, delta=timedelta(days=1))
store.forget(name='some', interval=to_remove)
self.assertTrue(os.path.exists(base))
def test_forget_interval_including_all_check_namespace_deleted(self):
with temporary_directory() as tem_dir:
self.make_three_buckets(tem_dir=tem_dir)
store = Store(directory=tem_dir, bucket_size=600)
store.forget(name='some',
interval=Interval(start=self.start,
delta=timedelta(days=3)))
self.assertEqual([], os.listdir(tem_dir))
def test_I_can_save_and_retrieve_1_week_of_one_metric(self):
with temporary_directory() as tem_dir:
store = Store(directory=tem_dir, bucket_size=600)
self.save_days_of_metric(store=store, days=7)
records = self.retrieve(store)
start = time()
records.sort()
print('Sorting took {}'.format(time() - start))
self.assertEqual(len(records), 3600 * 24 * 7)
def test_read_raw_with_file_existing_check_return_value(self):
with temporary_directory() as temp_dir:
buck, _ = self.make_buck(base=temp_dir)
self.assertFalse(os.path.exists(buck.full_path))
buck.append(123456, 'cpu:7,some:8.4')
buck.append(123457, 'cpu:8,some:8.4')
buck.append(123456, 'cpu:8,some:8.4')
self.assertEqual([
'123456 cpu:7,some:8.4\n123457 cpu:8,some:8.4\n123456 cpu:8,some:8.4\n',
], list(buck.raw()))
self.assertFalse(os.path.exists(buck.full_path))
def test_bucket_iteration(self):
with temporary_directory() as temp_dir:
buck, _ = self.make_buck(base=temp_dir)
self.assertFalse(os.path.exists(buck.full_path))
buck.append(123456, 'cpu:7,some:8.4')
buck.append(123457, 'cpu:8,some:8.4')
self.assertEqual([
(123456, 'cpu:7,some:8.4'),
(123457, 'cpu:8,some:8.4'),
], list(buck))
self.assertFalse(os.path.exists(buck.full_path))
def test_append_with_file_existing_check_appended(self):
with temporary_directory() as temp_dir:
buck, _ = self.make_buck(base=temp_dir)
self.assertFalse(os.path.exists(buck.full_path))
buck.append(123456, 'cpu:7,some:8.4')
self.assertEqual('123456 cpu:7,some:8.4\n',
file_content(buck.full_path))
buck.append(123457, 'cpu:8,some:8.4')
self.assertEqual('123456 cpu:7,some:8.4\n123457 cpu:8,some:8.4\n',
file_content(buck.full_path))
self.assertFalse(os.path.exists(buck.full_path))
def test_delete_check_deleted(self):
with temporary_directory() as temp_dir:
buck, _ = self.make_buck(base=temp_dir)
self.assertFalse(os.path.exists(buck.full_path))
buck.append(123456, 'cpu:7,some:8.4')
buck.append(123457, 'cpu:8,some:8.4')
self.assertListEqual([
(123456.0, 'cpu:7,some:8.4'),
(123457.0, 'cpu:8,some:8.4'),
], list(buck))
self.assertEqual('123456 cpu:7,some:8.4\n123457 cpu:8,some:8.4\n',
file_content(buck.full_path))
buck.delete()
self.assertFalse(os.path.exists(buck.full_path))
def test_I_can_save_and_retrieve_2_week_of_one_metric(self):
with temporary_directory() as tem_dir:
store = Store(directory=tem_dir, bucket_size=600)
self.save_days_of_metric(store=store, days=14)
start = time()
records = list(
store.retrieve(name='temp',
interval=Interval(start=self.start,
delta=seconds(3600 * 24 *
14))))
print('Retrieving {} records took {}'.format(
len(records),
time() - start))
start = time()
records.sort()
print('Sorting took {}'.format(time() - start))
self.assertEqual(len(records), 3600 * 24 * 14)
def test_record_with_existing_buckets_check_correct_one_updated(self):
with temporary_directory() as tem_dir:
self.make_three_buckets(tem_dir=tem_dir)
store = Store(directory=tem_dir, bucket_size=600)
store.record(name='some',
timestamp=self.start + timedelta(seconds=20),
data='pending:10;cpu:11.6')
ts = self.start + timedelta(seconds=1220)
store.record(name='some', timestamp=ts, data='pending:17;cpu:11.8')
ts = ts + timedelta(hours=5)
store.record(name='some', timestamp=ts, data='pending:4;cpu:11.8')
base = os.path.join(tem_dir, 'some', '600', '2018_03_03')
files = list(os.listdir(base))
self.assertEqual(3, len(files))
content = [file_content(base, file) for file in files]
self.assertListEqual(content, [
'1520099400.0 pending:3;cpu:11.8\n1520099420.0 pending:4;cpu:11.8\n',
'1520080200.0 pending:7;cpu:11.6\n1520080220.0 pending:10;cpu:11.6\n',
'1520081400.0 pending:7;cpu:11.8\n1520081420.0 pending:17;cpu:11.8\n'
])
def test_read_with_no_file_check_return_value(self):
with temporary_directory() as temp_dir:
buck, _ = self.make_buck(base=temp_dir)
self.assertEqual([], list(buck.read()))
def test_record_metric_with_no_buckets_check_buckets_created(self):
with temporary_directory() as tem_dir:
self.make_three_buckets(tem_dir=tem_dir)
def test_get_bucket_not_existing_check_return_and_not_created(self):
with temporary_directory() as tem_dir:
store = Store(directory=tem_dir, bucket_size=600)
buck = store._get_bucket('some', dt('2017-03-03T10:25:11'))
self.assertFalse(os.path.exists(buck.full_path))