def test_diff_with_empty(self):
testdb = "test-%s" % self.filename
now = time.time()
self.addCleanup(self._remove, testdb)
whisper.create(testdb, self.retention)
whisper.create(self.filename, self.retention)
whisper.update(testdb, 1.0, now)
whisper.update(self.filename, 2.0, now)
# Purposefully insert nulls to strip out
previous = now - self.retention[0][0]
whisper.update(testdb, float('NaN'), previous)
results = whisper.diff(testdb, self.filename, ignore_empty=True)
self.assertEqual(
results,
[(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)],
)
results_empties = whisper.diff(testdb,
self.filename,
ignore_empty=False)
expected = [(0, [(int(previous), float('NaN'), None),
(int(now), 1.0, 2.0)], 2), (1, [], 0)]
# Stupidly, float('NaN') != float('NaN'), so assert that the
# repr() results are the same :/
#
# See this thread:
# https://mail.python.org/pipermail/python-ideas/2010-March/006945.html
self.assertEqual(
repr(results_empties),
repr(expected),
)
# Since the above test is somewhat of a sham, ensure that there
# is a nan where there should be.
self.assertTrue(math.isnan(results_empties[0][1][0][1]))
def main():
archive_diffs = whisper.diff(path_a,path_b,ignore_empty=options.ignore_empty,until_time=until_time)
if options.summary:
if options.json:
print_summary_json(archive_diffs,path_a,path_b)
else:
print_summary(archive_diffs,pretty=(not options.columns),headers=(not options.no_headers))
else:
if options.json:
print_diffs_json(archive_diffs,path_a,path_b)
else:
print_diffs(archive_diffs,pretty=(not options.columns),headers=(not options.no_headers))
def test_diff_with_empty(self):
testdb = "test-%s" % self.filename
now = time.time()
self.addCleanup(self._remove, testdb)
whisper.create(testdb, self.retention)
whisper.create(self.filename, self.retention)
whisper.update(testdb, 1.0, now)
whisper.update(self.filename, 2.0, now)
# Purposefully insert nulls to strip out
previous = now - self.retention[0][0]
whisper.update(testdb, float('NaN'), previous)
results = whisper.diff(testdb, self.filename, ignore_empty=True)
self.assertEqual(
results,
[(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)],
)
results_empties = whisper.diff(testdb, self.filename, ignore_empty=False)
expected = [(0, [(int(previous), float('NaN'), None), (int(now), 1.0, 2.0)], 2), (1, [], 0)]
# Stupidly, float('NaN') != float('NaN'), so assert that the
# repr() results are the same :/
#
# See this thread:
# https://mail.python.org/pipermail/python-ideas/2010-March/006945.html
self.assertEqual(
repr(results_empties),
repr(expected),
)
# Since the above test is somewhat of a sham, ensure that there
# is a nan where there should be.
self.assertTrue(
math.isnan(results_empties[0][1][0][1])
)
def test_diff(self):
testdb = "test-%s" % self.filename
now = time.time()
whisper.create(testdb, self.retention)
whisper.create(self.filename, self.retention)
whisper.update(testdb, 1.0, now)
whisper.update(self.filename, 2.0, now)
results = whisper.diff(testdb, self.filename)
self._remove(testdb)
expected = [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)]
self.assertEqual(results, expected)
def test_diff(self):
testdb = "test-%s" % self.filename
now = time.time()
whisper.create(testdb, self.retention)
whisper.create(self.filename, self.retention)
whisper.update(testdb, 1.0, now)
whisper.update(self.filename, 2.0, now)
results = whisper.diff(testdb, self.filename)
self._remove(testdb)
expected = [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)]
self.assertEqual(results, expected)
sys.stdout.write(h % ('', 'timestamp', 'value_a', 'value_b'))
for p in points:
if pretty:
sys.stdout.write(f % ('', p[0], p[1], p[2]))
else:
sys.stdout.write(f % (archive, p[0], p[1], p[2]))
def print_summary(diffs, pretty=True, headers=True):
if pretty:
f = "%7s %9s %9s\n"
else:
f = "%s %s %s\n"
if headers:
sys.stdout.write(f % ('archive', 'total', 'differing'))
for archive, points, total in diffs:
sys.stdout.write(f % (archive, total, points.__len__()))
archive_diffs = whisper.diff(path_a,
path_b,
ignore_empty=options.ignore_empty,
until_time=until_time)
if options.summary:
print_summary(archive_diffs,
pretty=(not options.columns),
headers=(not options.no_headers))
else:
print_diffs(archive_diffs,
pretty=(not options.columns),
headers=(not options.no_headers))
def test_heal(self):
staging_dir = tempfile.mkdtemp(prefix='staging')
storage_dir = tempfile.mkdtemp(prefix='storage')
carbonate_sync.STORAGE_DIR = storage_dir
remote = os.path.join(staging_dir, 'foo.wsp')
local = os.path.join(storage_dir, 'foo.wsp')
resolution = [(1, 10)]
now = int(time.time())
whisper.create(local, resolution)
whisper.create(remote, resolution)
# N, N, N, 6, 5, 4, 3, N, N, N
whisper.update(local, 6.0, now - 6)
whisper.update(local, 5.0, now - 5)
whisper.update(local, 4.0, now - 4)
whisper.update(local, 3.0, now - 3)
# N, N, N, 6, 6, N, 3, 2, 1, N
whisper.update(remote, 6.0, now - 6)
whisper.update(remote, 6.0, now - 5)
whisper.update(remote, 3.0, now - 3)
whisper.update(remote, 2.0, now - 2)
whisper.update(remote, 1.0, now - 1)
results = whisper.diff(local, remote)
expected = [
(0, [
(now - 5, 5.0, 6.0),
(now - 4, 4.0, None),
(now - 2, None, 2.0),
(now - 1, None, 1.0),
], 6)
]
self.assertEqual(results, expected)
metrics_fs = ['foo.wsp']
start_time = now - 10
end_time = now
attr = {
'staging_dir': staging_dir,
'metrics_fs': metrics_fs,
'start_time': start_time,
'end_time': end_time,
'remote_user': '******',
'remote_node': 'foo',
'rsync_options': [],
'ssh_options': [],
'overwrite': False,
}
batch = carbonate_sync._Batch(**attr)
carbonate_sync._heal(batch)
# Check that we add missing points.
expected = [(0, [(now - 5, 5.0, 6.0), (now - 4, 4.0, None)], 6)]
results = whisper.diff(local, remote)
self.assertEqual(results, expected)
attr['overwrite'] = True
batch = carbonate_sync._Batch(**attr)
carbonate_sync._heal(batch)
expected = [(0, [(now - 4, 4.0, None)], 6)]
results = whisper.diff(local, remote)
self.assertEqual(results, expected)
shutil.rmtree(staging_dir)
shutil.rmtree(storage_dir)
else:
h = "%s %s %s %s\n"
f = "%s %d %s %s\n"
if headers:
sys.stdout.write(h%('archive','timestamp','value_a','value_b'))
for archive, points, total in diffs:
count = count=points.__len__()
if pretty:
sys.stdout.write('Archive %d (%d of %d datapoints differ)\n'%(archive,points.__len__(),total))
sys.stdout.write(h%('','timestamp','value_a','value_b'))
for p in points:
if pretty:
sys.stdout.write(f%('',p[0],p[1],p[2]))
else:
sys.stdout.write(f%(archive,p[0],p[1],p[2]))
def print_summary(diffs,pretty=True,headers=True):
if pretty:
f = "%7s %9s %9s\n"
else:
f = "%s %s %s\n"
if headers:
sys.stdout.write(f%('archive','total','differing'))
for archive, points, total in diffs:
sys.stdout.write(f%(archive,total,points.__len__()))
archive_diffs = whisper.diff(path_a,path_b,ignore_empty=options.ignore_empty)
if options.summary:
print_summary(archive_diffs,pretty=(not options.columns),headers=(not options.no_headers))
else:
print_diffs(archive_diffs,pretty=(not options.columns),headers=(not options.no_headers))