Example #1
0
    def test_diff_with_empty(self):
        testdb = "test-%s" % self.filename

        now = time.time()

        self.addCleanup(self._remove, testdb)
        whisper.create(testdb, self.retention)

        whisper.create(self.filename, self.retention)

        whisper.update(testdb, 1.0, now)
        whisper.update(self.filename, 2.0, now)

        # Purposefully insert nulls to strip out
        previous = now - self.retention[0][0]
        whisper.update(testdb, float('NaN'), previous)

        results = whisper.diff(testdb, self.filename, ignore_empty=True)
        self.assertEqual(
            results,
            [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)],
        )

        results_empties = whisper.diff(testdb,
                                       self.filename,
                                       ignore_empty=False)
        expected = [(0, [(int(previous), float('NaN'), None),
                         (int(now), 1.0, 2.0)], 2), (1, [], 0)]

        # Stupidly, float('NaN') != float('NaN'), so assert that the
        # repr() results are the same :/
        #
        # See this thread:
        #    https://mail.python.org/pipermail/python-ideas/2010-March/006945.html
        self.assertEqual(
            repr(results_empties),
            repr(expected),
        )
        # Since the above test is somewhat of a sham, ensure that there
        # is a nan where there should be.
        self.assertTrue(math.isnan(results_empties[0][1][0][1]))
Example #2
0
def main():
  archive_diffs = whisper.diff(path_a,path_b,ignore_empty=options.ignore_empty,until_time=until_time)
  if options.summary:
    if options.json:
      print_summary_json(archive_diffs,path_a,path_b)
    else:
      print_summary(archive_diffs,pretty=(not options.columns),headers=(not options.no_headers))
  else:
    if options.json:
      print_diffs_json(archive_diffs,path_a,path_b)
    else:
      print_diffs(archive_diffs,pretty=(not options.columns),headers=(not options.no_headers))
Example #3
0
    def test_diff_with_empty(self):
        testdb = "test-%s" % self.filename

        now = time.time()

        self.addCleanup(self._remove, testdb)
        whisper.create(testdb, self.retention)

        whisper.create(self.filename, self.retention)

        whisper.update(testdb, 1.0, now)
        whisper.update(self.filename, 2.0, now)

        # Purposefully insert nulls to strip out
        previous = now - self.retention[0][0]
        whisper.update(testdb, float('NaN'), previous)

        results = whisper.diff(testdb, self.filename, ignore_empty=True)
        self.assertEqual(
            results,
            [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)],
        )

        results_empties = whisper.diff(testdb, self.filename, ignore_empty=False)
        expected = [(0, [(int(previous), float('NaN'), None), (int(now), 1.0, 2.0)], 2), (1, [], 0)]

        # Stupidly, float('NaN') != float('NaN'), so assert that the
        # repr() results are the same :/
        #
        # See this thread:
        #    https://mail.python.org/pipermail/python-ideas/2010-March/006945.html
        self.assertEqual(
            repr(results_empties),
            repr(expected),
        )
        # Since the above test is somewhat of a sham, ensure that there
        # is a nan where there should be.
        self.assertTrue(
            math.isnan(results_empties[0][1][0][1])
        )
Example #4
0
    def test_diff(self):
        testdb = "test-%s" % self.filename

        now = time.time()

        whisper.create(testdb, self.retention)
        whisper.create(self.filename, self.retention)
        whisper.update(testdb, 1.0, now)
        whisper.update(self.filename, 2.0, now)

        results = whisper.diff(testdb, self.filename)
        self._remove(testdb)

        expected = [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)]

        self.assertEqual(results, expected)
Example #5
0
    def test_diff(self):
        testdb = "test-%s" % self.filename

        now = time.time()

        whisper.create(testdb, self.retention)
        whisper.create(self.filename, self.retention)
        whisper.update(testdb, 1.0, now)
        whisper.update(self.filename, 2.0, now)

        results = whisper.diff(testdb, self.filename)
        self._remove(testdb)

        expected = [(0, [(int(now), 1.0, 2.0)], 1), (1, [], 0)]

        self.assertEqual(results, expected)
Example #6
0
            sys.stdout.write(h % ('', 'timestamp', 'value_a', 'value_b'))
        for p in points:
            if pretty:
                sys.stdout.write(f % ('', p[0], p[1], p[2]))
            else:
                sys.stdout.write(f % (archive, p[0], p[1], p[2]))


def print_summary(diffs, pretty=True, headers=True):
    if pretty:
        f = "%7s %9s %9s\n"
    else:
        f = "%s %s %s\n"
    if headers:
        sys.stdout.write(f % ('archive', 'total', 'differing'))
    for archive, points, total in diffs:
        sys.stdout.write(f % (archive, total, points.__len__()))


archive_diffs = whisper.diff(path_a,
                             path_b,
                             ignore_empty=options.ignore_empty,
                             until_time=until_time)
if options.summary:
    print_summary(archive_diffs,
                  pretty=(not options.columns),
                  headers=(not options.no_headers))
else:
    print_diffs(archive_diffs,
                pretty=(not options.columns),
                headers=(not options.no_headers))
    def test_heal(self):
        staging_dir = tempfile.mkdtemp(prefix='staging')
        storage_dir = tempfile.mkdtemp(prefix='storage')
        carbonate_sync.STORAGE_DIR = storage_dir
        remote = os.path.join(staging_dir, 'foo.wsp')
        local = os.path.join(storage_dir, 'foo.wsp')
        resolution = [(1, 10)]

        now = int(time.time())

        whisper.create(local, resolution)
        whisper.create(remote, resolution)

        # N, N, N, 6, 5, 4, 3, N, N, N
        whisper.update(local, 6.0, now - 6)
        whisper.update(local, 5.0, now - 5)
        whisper.update(local, 4.0, now - 4)
        whisper.update(local, 3.0, now - 3)

        # N, N, N, 6, 6, N, 3, 2, 1, N
        whisper.update(remote, 6.0, now - 6)
        whisper.update(remote, 6.0, now - 5)
        whisper.update(remote, 3.0, now - 3)
        whisper.update(remote, 2.0, now - 2)
        whisper.update(remote, 1.0, now - 1)

        results = whisper.diff(local, remote)
        expected = [
            (0, [
                (now - 5, 5.0, 6.0),
                (now - 4, 4.0, None),
                (now - 2, None, 2.0),
                (now - 1, None, 1.0),
            ], 6)
        ]
        self.assertEqual(results, expected)

        metrics_fs = ['foo.wsp']
        start_time = now - 10
        end_time = now

        attr = {
            'staging_dir': staging_dir,
            'metrics_fs': metrics_fs,
            'start_time': start_time,
            'end_time': end_time,
            'remote_user': '******',
            'remote_node': 'foo',
            'rsync_options': [],
            'ssh_options': [],
            'overwrite': False,
        }
        batch = carbonate_sync._Batch(**attr)

        carbonate_sync._heal(batch)

        # Check that we add missing points.
        expected = [(0, [(now - 5, 5.0, 6.0), (now - 4, 4.0, None)], 6)]
        results = whisper.diff(local, remote)
        self.assertEqual(results, expected)

        attr['overwrite'] = True
        batch = carbonate_sync._Batch(**attr)
        carbonate_sync._heal(batch)

        expected = [(0, [(now - 4, 4.0, None)], 6)]
        results = whisper.diff(local, remote)
        self.assertEqual(results, expected)

        shutil.rmtree(staging_dir)
        shutil.rmtree(storage_dir)
  else:
    h = "%s %s %s %s\n"
    f = "%s %d %s %s\n"
  if headers:
    sys.stdout.write(h%('archive','timestamp','value_a','value_b'))
  for archive, points, total in diffs:
    count = count=points.__len__()
    if pretty:
      sys.stdout.write('Archive %d (%d of %d datapoints differ)\n'%(archive,points.__len__(),total))
      sys.stdout.write(h%('','timestamp','value_a','value_b'))
    for p in points:
      if pretty:
        sys.stdout.write(f%('',p[0],p[1],p[2]))
      else:
        sys.stdout.write(f%(archive,p[0],p[1],p[2]))

def print_summary(diffs,pretty=True,headers=True):
  if pretty:
    f = "%7s %9s %9s\n"
  else:
    f = "%s %s %s\n"
  if headers:
    sys.stdout.write(f%('archive','total','differing'))
  for archive, points, total in diffs:
    sys.stdout.write(f%(archive,total,points.__len__()))
    
archive_diffs = whisper.diff(path_a,path_b,ignore_empty=options.ignore_empty)
if options.summary:
  print_summary(archive_diffs,pretty=(not options.columns),headers=(not options.no_headers))
else:
  print_diffs(archive_diffs,pretty=(not options.columns),headers=(not options.no_headers))