def test_called_back_error(self): """Test that the hasher signals error when no file.""" # create the hasher mark = object() queue = hash_queue.UniqueQueue() d = defer.Deferred() class Helper(object): """Helper class.""" def push(self, event, **kwargs): """Callback.""" d.callback((event, kwargs)) receiver = Helper() hasher = hash_queue._Hasher(queue, mark, receiver) # send what to hash item = (("not_to_be_found", "foo"), FAKE_TIMESTAMP) queue.put(item) # start the hasher after putting the work items hasher.start() def check_info((event, kwargs)): """Check the info pushed by the hasher.""" hasher.stop() hasher.join(timeout=5) self.assertEqual(event, "HQ_HASH_ERROR") self.assertEqual(kwargs['mdid'], "foo") d.addCallback(check_info) # release the processor and check return d
def test_open_file_with_rb(self): """Check that the file to hash is opened with 'rb' mode.""" called = [] orig = hash_queue.open_file def faked_open_file(*a): called.append(a) return orig(*a) self.patch(hash_queue, 'open_file', faked_open_file) queue = hash_queue.UniqueQueue() testfile = os.path.join(self.test_dir, "testfile") with open_file(testfile, "wb") as fh: fh.write("foobar") item = ((testfile, "mdid"), FAKE_TIMESTAMP) queue.put(item) d = defer.Deferred() eq = FakeEventQueue(d) hasher = hash_queue._Hasher(queue=queue, end_mark='end-mark', event_queue=eq) # start the hasher after putting the work items hasher.start() yield d hasher.stop() self.assertEqual(called, [(testfile, 'rb')])
def test_unique(self): """The hasher should return in order.""" # calculate what we should receive should_be = [] for i in range(10): hasher = content_hash_factory() text = "supercalifragilistico"+str(i) hasher.hash_object.update(text) tfile = os.path.join(self.test_dir, "tfile"+str(i)) with open_file(tfile, "wb") as fh: fh.write("supercalifragilistico"+str(i)) d = dict(path=tfile, hash=hasher.content_hash(), crc32=crc32(text), size=len(text), stat=stat_path(tfile)) should_be.append(("HQ_HASH_NEW", d)) d = defer.Deferred() class Helper(object): """Helper class.""" # class-closure, cannot use self, pylint: disable-msg=E0213 def __init__(innerself): innerself.store = [] def push(innerself, event, **kwargs): """Callback.""" innerself.store.append((event, kwargs)) if len(innerself.store) == 10: if innerself.store == should_be: d.callback(True) else: d.errback(Exception("are different!")) receiver = Helper() hq = hash_queue.HashQueue(receiver) self.addCleanup(hq.shutdown) # stop the hasher so we can test the unique items in the queue hq.hasher.stop() self.log.debug('Hasher stopped (forced)') # allow the hasher to fully stop time.sleep(0.1) # create a new hasher just like the HashQueue creates it hq.hasher = hash_queue._Hasher(hq._queue, hq._end_mark, receiver) hq.hasher.setDaemon(True) # send to hash twice for i in range(10): tfile = os.path.join(self.test_dir, "tfile"+str(i)) hq.insert(tfile, "mdid") hq.insert(tfile, "mdid") # start the hasher self.log.debug('Hasher started (forced)') hq.hasher.start() # insert the last item to check the uniqueness in the queue while # the hasher is running for i in range(9, 10): tfile = os.path.join(self.test_dir, "tfile"+str(i)) hq.insert(tfile, "mdid") return d
def test_order(self): """The hasher should return in order.""" # calculate what we should receive should_be = [] for i in range(10): hasher = content_hash_factory() text = "supercalifragilistico"+str(i) hasher.hash_object.update(text) tfile = os.path.join(self.test_dir, "tfile"+str(i)) with open_file(tfile, "wb") as fh: fh.write("supercalifragilistico"+str(i)) d = dict(path=tfile, hash=hasher.content_hash(), crc32=crc32(text), size=len(text), stat=stat_path(tfile)) should_be.append(("HQ_HASH_NEW", d)) # create the hasher mark = object() queue = hash_queue.UniqueQueue() d = defer.Deferred() class Helper(object): """Helper class.""" # class-closure, cannot use self, pylint: disable-msg=E0213 def __init__(innerself): innerself.store = [] def push(innerself, event, **kwargs): """Callback.""" innerself.store.append((event, kwargs)) if len(innerself.store) == 10: hasher.stop() hasher.join(timeout=5) if innerself.store == should_be: d.callback(True) else: d.errback(Exception("are different!")) receiver = Helper() hasher = hash_queue._Hasher(queue, mark, receiver) # send what to hash for i in range(10): tfile = os.path.join(self.test_dir, "tfile"+str(i)) item = ((tfile, "mdid"), FAKE_TIMESTAMP) queue.put(item) # start the hasher after putting the work items hasher.start() return d
def test_called_back_ok(self): """Test that the hasher produces correct info.""" # create the hasher mark = object() queue = hash_queue.UniqueQueue() d = defer.Deferred() class Helper(object): """Helper class.""" def push(self, event, **kwargs): """Callback.""" d.callback((event, kwargs)) receiver = Helper() hasher = hash_queue._Hasher(queue, mark, receiver) # send what to hash testfile = os.path.join(self.test_dir, "testfile") with open_file(testfile, "wb") as fh: fh.write("foobar") item = ((testfile, "mdid"), FAKE_TIMESTAMP) queue.put(item) # start the hasher after putting the work items hasher.start() def check_info((event, kwargs)): """check the info pushed by the hasher""" # pylint: disable-msg=W0612 hasher.stop() hasher.join(timeout=5) self.assertEqual(event, "HQ_HASH_NEW") # calculate what we should receive realh = content_hash_factory() realh.hash_object.update("foobar") should_be = realh.content_hash() curr_stat = stat_path(testfile) self.assertEquals(should_be, kwargs['hash']) for attr in ('st_mode', 'st_ino', 'st_dev', 'st_nlink', 'st_uid', 'st_gid', 'st_size', 'st_ctime', 'st_mtime'): self.assertEquals(getattr(curr_stat, attr), getattr(kwargs['stat'], attr)) d.addCallback(check_info) # release the processor and check return d
def test_called_back_log_ok(self): """Test that the hasher produces correct info.""" # create the hasher mark = object() queue = hash_queue.UniqueQueue() d = defer.Deferred() class Helper(object): """Helper class.""" def push(self, event, **kwargs): """Callback.""" d.callback(kwargs) receiver = Helper() hasher = hash_queue._Hasher(queue, mark, receiver) # log config handler = MementoHandler() handler.setLevel(logging.DEBUG) hasher.logger.addHandler(handler) # send what to hash testfile = os.path.join(self.test_dir, "testfile") with open_file(testfile, "wb") as fh: fh.write("foobar") item = ((testfile, "mdid"), FAKE_TIMESTAMP) queue.put(item) # start the hasher after putting the work items hasher.start() # wait event and stop hasher yield d hasher.stop() hasher.join(timeout=5) # check log log_msg = [r.message for r in handler.records if "path hash pushed" in r.message][0] self.assertTrue("path" in log_msg) self.assertTrue("hash" in log_msg) self.assertTrue("crc" in log_msg) self.assertTrue("size" in log_msg) self.assertTrue("st_ino" in log_msg) self.assertTrue("st_size" in log_msg) self.assertTrue("st_mtime" in log_msg) hasher.logger.removeHandler(handler)
def test_large_content(self): """The hasher works ok for a lot of info.""" # calculate what we should receive testinfo = "".join(chr(random.randint(0, 255)) for i in range(100000)) hasher = content_hash_factory() hasher.hash_object.update(testinfo) testfile = os.path.join(self.test_dir, "testfile") testhash = hasher.content_hash() # create the hasher mark = object() queue = hash_queue.UniqueQueue() d = defer.Deferred() class Helper(object): """Helper class.""" def push(self, event, path, hash, crc32, size, stat): """callback""" hasher.stop() hasher.join(timeout=5) if event != "HQ_HASH_NEW": d.errback(Exception("envent is not HQ_HASH_NEW")) elif path != testfile: d.errback(Exception("path is not the original one")) elif hash != testhash: d.errback(Exception("the hashes are different!")) else: d.callback(True) receiver = Helper() hasher = hash_queue._Hasher(queue, mark, receiver) # send what to hash with open_file(testfile, "wb") as fh: fh.write(testinfo) item = ((testfile, "mdid"), FAKE_TIMESTAMP) queue.put(item) # start the hasher after putting the work items hasher.start() return d
def setUpWithCount(self, expected_events=1): """Initialize this test instance.""" self.test_dir = self.mktemp('test_dir') self.fake_time = FakeTimeModule() self.patch(hash_queue, "time", self.fake_time) self.queue = hash_queue.UniqueQueue() self.testfile = os.path.join(self.test_dir, "testfile") with open_file(self.testfile, "wb") as fh: fh.write("foobar") self.event_d = defer.Deferred() self.eq = FakeEventQueue(self.event_d, expected_events) self.hasher = hash_queue._Hasher(queue=self.queue, end_mark='end-mark', event_queue=self.eq) def stop_hasher(): """Safely stop the hasher.""" self.hasher.stop() self.hasher.join(timeout=5) self.assertFalse(self.hasher.isAlive()) self.addCleanup(stop_hasher)
def test_live_process(self): """Check that the hasher lives and dies.""" # create the hasher mark = object() queue = hash_queue.UniqueQueue() class Helper(object): """Helper class.""" def push(self, event, **kwargs): """Callback.""" receiver = Helper() hasher = hash_queue._Hasher(queue, mark, receiver) hasher.start() # it's aliveeeeeeee! self.assertTrue(hasher.isAlive()) # stop it, and release the processor to let the other thread run hasher.stop() time.sleep(.1) # "I see dead threads" self.assertFalse(hasher.isAlive()) hasher.join(timeout=5)