def test_skip_listener_read_failure(self): """ Listens for a file that's skipped because we lack read permissions. """ if system.is_windows(): test.runner.skip(self, "(chmod not functional)") test_path = test.runner.get_runner().get_test_dir("secret_file") try: test_file = open(test_path, "w") test_file.write("test data for test_skip_listener_unrecognized_type()") test_file.close() os.chmod(test_path, 0077) # remove read permissions skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.ReadFailed)) self.assertTrue(isinstance(skip_exception.exception, IOError)) finally: if os.path.exists(test_path): os.remove(test_path)
def test_skip_listener_already_read(self): """ Checks that calling set_processed_files() prior to reading makes us skip those files. This also doubles for testing that skip listeners are notified of files that we've already read. """ # path that we want the DescriptorReader to skip test_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor") initial_processed_files = {test_path: sys.maxint} skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) reader.set_processed_files(initial_processed_files) self.assertEquals(initial_processed_files, reader.get_processed_files()) with reader: list(reader) # iterates over all of the descriptors self.assertEquals(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.AlreadyRead)) self.assertEqual(sys.maxint, skip_exception.last_modified_when_read)
def test_skip_listener_unrecognized_type(self): """ Listens for a file that's skipped because its file type isn't recognized. """ # types are solely based on file extensions so making something that looks # like an png image test_path = test.runner.get_runner().get_test_dir("test.png") try: test_file = open(test_path, "w") test_file.write("test data for test_skip_listener_unrecognized_type()") test_file.close() skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType)) self.assertTrue(skip_exception.mime_type in (("image/png", None), ("image/x-png", None))) finally: if os.path.exists(test_path): os.remove(test_path)
def test_skip_nondescriptor_contents(self): """ Checks that the reader properly reports when it skips both binary and plaintext non-descriptor files. """ skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(4, len(skip_listener.results)) for skip_path, skip_exception in skip_listener.results: if skip_path.endswith(".swp"): continue # skip vim temp files if not os.path.basename(skip_path) in ("riddle", "tiny.png", "vote", "new_metrics_type"): self.fail("Unexpected non-descriptor content: %s" % skip_path) self.assertTrue( isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
def test_skip_listener_unrecognized_type(self): """ Listens for a file that's skipped because its file type isn't recognized. """ # types are solely based on file extensions so making something that looks # like an png image test_path = os.path.join(self.temp_directory, 'test.png') try: test_file = open(test_path, 'w') test_file.write('test data for test_skip_listener_unrecognized_type()') test_file.close() skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType)) self.assertTrue(skip_exception.mime_type in (('image/png', None), ('image/x-png', None))) finally: if os.path.exists(test_path): os.remove(test_path)
def test_skip_nondescriptor_contents(self): """ Checks that the reader properly reports when it skips both binary and plaintext non-descriptor files. """ skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA) reader.register_skip_listener(skip_listener.listener) expected_skip_files = ("riddle", "tiny.png", "vote", "new_metrics_type") with reader: list(reader) # iterates over all of the descriptors # strip anything with a .swp suffix (vim tmp files) skip_listener.results = [(path, exc) for (path, exc) in skip_listener.results if not path.endswith(".swp")] if len(skip_listener.results) != len(expected_skip_files): expected_label = ",\n ".join(expected_skip_files) results_label = ",\n ".join(["%s (%s)" % (path, exc) for (path, exc) in skip_listener.results]) self.fail("Skipped files that we should have been able to parse.\n\nExpected:\n %s\n\nResult:\n %s" % (expected_label, results_label)) for skip_path, skip_exception in skip_listener.results: if not os.path.basename(skip_path) in expected_skip_files: self.fail("Unexpected non-descriptor content: %s" % skip_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
def test_skip_listener_read_failure(self): """ Listens for a file that's skipped because we lack read permissions. """ if system.is_windows(): test.runner.skip(self, "(chmod not functional)") test_path = test.runner.get_runner().get_test_dir("secret_file") try: test_file = open(test_path, "w") test_file.write( "test data for test_skip_listener_unrecognized_type()") test_file.close() os.chmod(test_path, 0077) # remove read permissions skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue( isinstance(skip_exception, stem.descriptor.reader.ReadFailed)) self.assertTrue(isinstance(skip_exception.exception, IOError)) finally: if os.path.exists(test_path): os.remove(test_path)
def test_skip_nondescriptor_contents(self): """ Checks that the reader properly reports when it skips both binary and plaintext non-descriptor files. """ skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(os.path.join(DESCRIPTOR_TEST_DATA, 'unparseable')) reader.register_skip_listener(skip_listener.listener) expected_skip_files = ('riddle', 'tiny.png', 'vote', 'new_metrics_type', 'cached-microdesc-consensus_with_carriage_returns', 'extrainfo_nonascii_v3_reqs') with reader: list(reader) # iterates over all of the descriptors # strip anything with a .swp suffix (vim tmp files) skip_listener.results = [(path, exc) for (path, exc) in skip_listener.results if not path.endswith('.swp')] if len(skip_listener.results) != len(expected_skip_files): expected_label = ',\n '.join(expected_skip_files) results_label = ',\n '.join(['%s (%s)' % (path, exc) for (path, exc) in skip_listener.results]) self.fail('Skipped files that we should have been able to parse.\n\nExpected:\n %s\n\nResult:\n %s' % (expected_label, results_label)) for skip_path, skip_exception in skip_listener.results: if not os.path.basename(skip_path) in expected_skip_files: self.fail('Unexpected non-descriptor content: %s' % skip_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
def test_skip_listener_already_read(self): """ Checks that calling set_processed_files() prior to reading makes us skip those files. This also doubles for testing that skip listeners are notified of files that we've already read. """ # path that we want the DescriptorReader to skip test_path = os.path.join(DESCRIPTOR_TEST_DATA, 'example_descriptor') initial_processed_files = {test_path: sys.maxsize} skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) reader.set_processed_files(initial_processed_files) self.assertEqual(initial_processed_files, reader.get_processed_files()) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue( isinstance(skip_exception, stem.descriptor.reader.AlreadyRead)) self.assertEqual(sys.maxsize, skip_exception.last_modified_when_read)
def read_descriptors(descriptors, end_of_month_desc, descriptor_dir, skip_listener): """Add to descriptors contents of descriptor archive in descriptor_dir.""" num_descriptors = 0 num_relays = 0 print('Reading descriptors from: {0}'.format(descriptor_dir)) reader = stem.descriptor.reader.DescriptorReader(descriptor_dir, \ validate=False) reader.register_skip_listener(skip_listener) # use read li$stener to store metrics type annotation, whione_annotation = [None] cur_type_annotation = [None] def read_listener(path): f = open(path) # store initial metrics type annotation initial_position = f.tell() first_line = f.readline() f.seek(initial_position) if (first_line[0:5] == '@type'): cur_type_annotation[0] = first_line else: cur_type_annotation[0] = None f.close() reader.register_read_listener(read_listener) with reader: for desc in reader: if (num_descriptors % 10000 == 0): print('{0} descriptors processed.'.format(num_descriptors)) num_descriptors += 1 if (desc.fingerprint not in descriptors): descriptors[desc.fingerprint] = {} num_relays += 1 # stuff type annotation into stem object desc.type_annotation = cur_type_annotation[0] if desc.published is not None: t_published = timestamp( desc.published.replace(tzinfo=pytz.UTC)) if (desc.published.day > 27): #february ends the 28th and I am lazy if (desc.fingerprint not in end_of_month_desc): end_of_month_desc[desc.fingerprint] = {} end_of_month_desc[desc.fingerprint][t_published] = desc descriptors[desc.fingerprint]\ [t_published] = desc print('#descriptors: {0}; #relays:{1}'.\ format(num_descriptors,num_relays))
def test_skip_listener_file_missing(self): """ Listens for a file that's skipped because the file doesn't exist. """ test_path = "/non-existant/path" skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.FileMissing))
def test_unrecognized_metrics_type(self): """ Parses a file that has a valid metrics header, but an unrecognized type. """ test_path = test.integ.descriptor.get_resource("new_metrics_type") skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType)) self.assertEqual((None, None), skip_exception.mime_type)
def test_skip_nondescriptor_contents(self): """ Checks that the reader properly reports when it skips both binary and plaintext non-descriptor files. """ test.mocking.mock_method( stem.descriptor.server_descriptor.RelayDescriptor, '_validate_content', test.mocking.no_op()) skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA) reader.register_skip_listener(skip_listener.listener) expected_skip_files = ("riddle", "tiny.png", "vote", "new_metrics_type") with reader: list(reader) # iterates over all of the descriptors # strip anything with a .swp suffix (vim tmp files) skip_listener.results = [(path, exc) for (path, exc) in skip_listener.results if not path.endswith(".swp")] if len(skip_listener.results) != len(expected_skip_files): expected_label = ",\n ".join(expected_skip_files) results_label = ",\n ".join([ "%s (%s)" % (path, exc) for (path, exc) in skip_listener.results ]) self.fail( "Skipped files that we should have been able to parse.\n\nExpected:\n %s\n\nResult:\n %s" % (expected_label, results_label)) for skip_path, skip_exception in skip_listener.results: if not os.path.basename(skip_path) in expected_skip_files: self.fail("Unexpected non-descriptor content: %s" % skip_path) self.assertTrue( isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
def test_skip_listener_file_missing(self): """ Listens for a file that's skipped because the file doesn't exist. """ test_path = '/non-existant/path' skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.FileMissing))
def test_unrecognized_metrics_type(self): """ Parses a file that has a valid metrics header, but an unrecognized type. """ test_path = test.unit.descriptor.get_resource('unparseable/new_metrics_type') skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType)) self.assertEqual((None, None), skip_exception.mime_type)
def read_descriptors(descriptors, descriptor_dir, skip_listener): """Add to descriptors contents of descriptor archive in descriptor_dir.""" num_descriptors = 0 num_relays = 0 print('Reading descriptors from: {0}'.format(descriptor_dir)) reader = stem.descriptor.reader.DescriptorReader(descriptor_dir,validate=True) reader.register_skip_listener(skip_listener) with reader: for desc in reader: if (num_descriptors % 10000 == 0): print('{0} descriptors processed.'.format(num_descriptors)) num_descriptors += 1 if (desc.fingerprint not in descriptors): descriptors[desc.fingerprint] = {} num_relays += 1 descriptors[desc.fingerprint]\ [pathsim.timestamp(desc.published)] = desc print('#descriptors: {0}; #relays:{1}'.\ format(num_descriptors,num_relays))
def test_skip_listener_read_failure(self): """ Listens for a file that's skipped because we lack read permissions. """ # test relies on being unable to read a file if getpass.getuser() == 'root': test.runner.skip(self, '(running as root)') return elif system.is_windows(): test.runner.skip(self, '(chmod not functional)') return test_path = os.path.join(self.temp_directory, 'secret_file') try: test_file = open(test_path, 'w') test_file.write( 'test data for test_skip_listener_unrecognized_type()') test_file.close() os.chmod(test_path, 0o077) # remove read permissions skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue( isinstance(skip_exception, stem.descriptor.reader.ReadFailed)) self.assertTrue(isinstance(skip_exception.exception, IOError)) finally: if os.path.exists(test_path): os.remove(test_path)
def read_descriptors(descriptors, descriptor_dir, skip_listener): """Add to descriptors contents of descriptor archive in descriptor_dir.""" num_descriptors = 0 num_relays = 0 print('Reading descriptors from: {0}'.format(descriptor_dir)) reader = stem.descriptor.reader.DescriptorReader(descriptor_dir, validate=True) reader.register_skip_listener(skip_listener) with reader: for desc in reader: if (num_descriptors % 10000 == 0): print('{0} descriptors processed.'.format(num_descriptors)) num_descriptors += 1 if (desc.fingerprint not in descriptors): descriptors[desc.fingerprint] = {} num_relays += 1 descriptors[desc.fingerprint]\ [pathsim.timestamp(desc.published)] = desc print('#descriptors: {0}; #relays:{1}'.\ format(num_descriptors,num_relays))
def test_skip_nondescriptor_contents(self): """ Checks that the reader properly reports when it skips both binary and plaintext non-descriptor files. """ skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(4, len(skip_listener.results)) for skip_path, skip_exception in skip_listener.results: if skip_path.endswith(".swp"): continue # skip vim temp files if not os.path.basename(skip_path) in ("riddle", "tiny.png", "vote", "new_metrics_type"): self.fail("Unexpected non-descriptor content: %s" % skip_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
def read_descriptors(descriptors, descriptor_dir, skip_listener): """Add to descriptors contents of descriptor archive in descriptor_dir.""" num_descriptors = 0 num_relays = 0 print('Reading descriptors from: {0}'.format(descriptor_dir)) reader = stem.descriptor.reader.DescriptorReader(descriptor_dir, validate=True) reader.register_skip_listener(skip_listener) # use read listener to store metrics type annotation, which is otherwise discarded cur_type_annotation = [None] def read_listener(path): f = open(path) # store initial metrics type annotation initial_position = f.tell() first_line = f.readline() f.seek(initial_position) if (first_line[0:5] == '@type'): cur_type_annotation[0] = first_line else: cur_type_annotation[0] = None f.close() reader.register_read_listener(read_listener) with reader: for desc in reader: if (num_descriptors % 10000 == 0): print('{0} descriptors processed.'.format(num_descriptors)) num_descriptors += 1 if (desc.fingerprint not in descriptors): descriptors[desc.fingerprint] = {} num_relays += 1 # stuff type annotation into stem object desc.type_annotation = cur_type_annotation[0] descriptors[desc.fingerprint]\ [pathsim.timestamp(desc.published)] = desc print('#descriptors: {0}; #relays:{1}'.\ format(num_descriptors,num_relays))
def test_skip_listener_read_failure(self): """ Listens for a file that's skipped because we lack read permissions. """ # test relies on being unable to read a file if getpass.getuser() == 'root': self.skipTest('(running as root)') return elif system.is_windows(): self.skipTest('(chmod not functional)') return test_path = os.path.join(self.temp_directory, 'secret_file') try: test_file = open(test_path, 'w') test_file.write('test data for test_skip_listener_unrecognized_type()') test_file.close() os.chmod(test_path, 0o077) # remove read permissions skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.ReadFailed)) self.assertTrue(isinstance(skip_exception.exception, IOError)) finally: if os.path.exists(test_path): os.remove(test_path)