def test_skip_listener_read_failure(self):
"""
Listens for a file that's skipped because we lack read permissions.
"""
if system.is_windows():
test.runner.skip(self, "(chmod not functional)")
test_path = test.runner.get_runner().get_test_dir("secret_file")
try:
test_file = open(test_path, "w")
test_file.write("test data for test_skip_listener_unrecognized_type()")
test_file.close()
os.chmod(test_path, 0077) # remove read permissions
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader: list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.ReadFailed))
self.assertTrue(isinstance(skip_exception.exception, IOError))
finally:
if os.path.exists(test_path):
os.remove(test_path)
def test_skip_listener_already_read(self):
"""
Checks that calling set_processed_files() prior to reading makes us skip
those files. This also doubles for testing that skip listeners are notified
of files that we've already read.
"""
# path that we want the DescriptorReader to skip
test_path = os.path.join(DESCRIPTOR_TEST_DATA, "example_descriptor")
initial_processed_files = {test_path: sys.maxint}
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
reader.set_processed_files(initial_processed_files)
self.assertEquals(initial_processed_files, reader.get_processed_files())
with reader: list(reader) # iterates over all of the descriptors
self.assertEquals(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.AlreadyRead))
self.assertEqual(sys.maxint, skip_exception.last_modified_when_read)
def test_skip_listener_unrecognized_type(self):
"""
Listens for a file that's skipped because its file type isn't recognized.
"""
# types are solely based on file extensions so making something that looks
# like an png image
test_path = test.runner.get_runner().get_test_dir("test.png")
try:
test_file = open(test_path, "w")
test_file.write("test data for test_skip_listener_unrecognized_type()")
test_file.close()
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader: list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
self.assertTrue(skip_exception.mime_type in (("image/png", None), ("image/x-png", None)))
finally:
if os.path.exists(test_path):
os.remove(test_path)
def test_skip_nondescriptor_contents(self):
"""
Checks that the reader properly reports when it skips both binary and
plaintext non-descriptor files.
"""
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA)
reader.register_skip_listener(skip_listener.listener)
with reader:
list(reader) # iterates over all of the descriptors
self.assertEqual(4, len(skip_listener.results))
for skip_path, skip_exception in skip_listener.results:
if skip_path.endswith(".swp"):
continue # skip vim temp files
if not os.path.basename(skip_path) in ("riddle", "tiny.png",
"vote", "new_metrics_type"):
self.fail("Unexpected non-descriptor content: %s" % skip_path)
self.assertTrue(
isinstance(skip_exception,
stem.descriptor.reader.UnrecognizedType))
def test_skip_listener_unrecognized_type(self):
"""
Listens for a file that's skipped because its file type isn't recognized.
"""
# types are solely based on file extensions so making something that looks
# like an png image
test_path = os.path.join(self.temp_directory, 'test.png')
try:
test_file = open(test_path, 'w')
test_file.write('test data for test_skip_listener_unrecognized_type()')
test_file.close()
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader:
list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
self.assertTrue(skip_exception.mime_type in (('image/png', None), ('image/x-png', None)))
finally:
if os.path.exists(test_path):
os.remove(test_path)
def test_skip_nondescriptor_contents(self):
"""
Checks that the reader properly reports when it skips both binary and
plaintext non-descriptor files.
"""
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA)
reader.register_skip_listener(skip_listener.listener)
expected_skip_files = ("riddle", "tiny.png", "vote", "new_metrics_type")
with reader:
list(reader) # iterates over all of the descriptors
# strip anything with a .swp suffix (vim tmp files)
skip_listener.results = [(path, exc) for (path, exc) in skip_listener.results if not path.endswith(".swp")]
if len(skip_listener.results) != len(expected_skip_files):
expected_label = ",\n ".join(expected_skip_files)
results_label = ",\n ".join(["%s (%s)" % (path, exc) for (path, exc) in skip_listener.results])
self.fail("Skipped files that we should have been able to parse.\n\nExpected:\n %s\n\nResult:\n %s" % (expected_label, results_label))
for skip_path, skip_exception in skip_listener.results:
if not os.path.basename(skip_path) in expected_skip_files:
self.fail("Unexpected non-descriptor content: %s" % skip_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
def test_skip_listener_read_failure(self):
"""
Listens for a file that's skipped because we lack read permissions.
"""
if system.is_windows():
test.runner.skip(self, "(chmod not functional)")
test_path = test.runner.get_runner().get_test_dir("secret_file")
try:
test_file = open(test_path, "w")
test_file.write(
"test data for test_skip_listener_unrecognized_type()")
test_file.close()
os.chmod(test_path, 0077) # remove read permissions
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader:
list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(
isinstance(skip_exception, stem.descriptor.reader.ReadFailed))
self.assertTrue(isinstance(skip_exception.exception, IOError))
finally:
if os.path.exists(test_path):
os.remove(test_path)
def test_skip_nondescriptor_contents(self):
"""
Checks that the reader properly reports when it skips both binary and
plaintext non-descriptor files.
"""
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(os.path.join(DESCRIPTOR_TEST_DATA, 'unparseable'))
reader.register_skip_listener(skip_listener.listener)
expected_skip_files = ('riddle', 'tiny.png', 'vote', 'new_metrics_type', 'cached-microdesc-consensus_with_carriage_returns', 'extrainfo_nonascii_v3_reqs')
with reader:
list(reader) # iterates over all of the descriptors
# strip anything with a .swp suffix (vim tmp files)
skip_listener.results = [(path, exc) for (path, exc) in skip_listener.results if not path.endswith('.swp')]
if len(skip_listener.results) != len(expected_skip_files):
expected_label = ',\n '.join(expected_skip_files)
results_label = ',\n '.join(['%s (%s)' % (path, exc) for (path, exc) in skip_listener.results])
self.fail('Skipped files that we should have been able to parse.\n\nExpected:\n %s\n\nResult:\n %s' % (expected_label, results_label))
for skip_path, skip_exception in skip_listener.results:
if not os.path.basename(skip_path) in expected_skip_files:
self.fail('Unexpected non-descriptor content: %s' % skip_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
def test_skip_listener_already_read(self):
"""
Checks that calling set_processed_files() prior to reading makes us skip
those files. This also doubles for testing that skip listeners are notified
of files that we've already read.
"""
# path that we want the DescriptorReader to skip
test_path = os.path.join(DESCRIPTOR_TEST_DATA, 'example_descriptor')
initial_processed_files = {test_path: sys.maxsize}
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
reader.set_processed_files(initial_processed_files)
self.assertEqual(initial_processed_files, reader.get_processed_files())
with reader:
list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(
isinstance(skip_exception, stem.descriptor.reader.AlreadyRead))
self.assertEqual(sys.maxsize, skip_exception.last_modified_when_read)
def read_descriptors(descriptors, end_of_month_desc, descriptor_dir,
skip_listener):
"""Add to descriptors contents of descriptor archive in descriptor_dir."""
num_descriptors = 0
num_relays = 0
print('Reading descriptors from: {0}'.format(descriptor_dir))
reader = stem.descriptor.reader.DescriptorReader(descriptor_dir, \
validate=False)
reader.register_skip_listener(skip_listener)
# use read li$stener to store metrics type annotation, whione_annotation = [None]
cur_type_annotation = [None]
def read_listener(path):
f = open(path)
# store initial metrics type annotation
initial_position = f.tell()
first_line = f.readline()
f.seek(initial_position)
if (first_line[0:5] == '@type'):
cur_type_annotation[0] = first_line
else:
cur_type_annotation[0] = None
f.close()
reader.register_read_listener(read_listener)
with reader:
for desc in reader:
if (num_descriptors % 10000 == 0):
print('{0} descriptors processed.'.format(num_descriptors))
num_descriptors += 1
if (desc.fingerprint not in descriptors):
descriptors[desc.fingerprint] = {}
num_relays += 1
# stuff type annotation into stem object
desc.type_annotation = cur_type_annotation[0]
if desc.published is not None:
t_published = timestamp(
desc.published.replace(tzinfo=pytz.UTC))
if (desc.published.day >
27): #february ends the 28th and I am lazy
if (desc.fingerprint not in end_of_month_desc):
end_of_month_desc[desc.fingerprint] = {}
end_of_month_desc[desc.fingerprint][t_published] = desc
descriptors[desc.fingerprint]\
[t_published] = desc
print('#descriptors: {0}; #relays:{1}'.\
format(num_descriptors,num_relays))
def test_skip_listener_file_missing(self): """ Listens for a file that's skipped because the file doesn't exist. """ test_path = "/non-existant/path" skip_listener = SkipListener() reader = stem.descriptor.reader.DescriptorReader(test_path) reader.register_skip_listener(skip_listener.listener) with reader: list(reader) # iterates over all of the descriptors self.assertEqual(1, len(skip_listener.results)) skipped_path, skip_exception = skip_listener.results[0] self.assertEqual(test_path, skipped_path) self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.FileMissing))
def test_unrecognized_metrics_type(self):
"""
Parses a file that has a valid metrics header, but an unrecognized type.
"""
test_path = test.integ.descriptor.get_resource("new_metrics_type")
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader: list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
self.assertEqual((None, None), skip_exception.mime_type)
def test_skip_nondescriptor_contents(self):
"""
Checks that the reader properly reports when it skips both binary and
plaintext non-descriptor files.
"""
test.mocking.mock_method(
stem.descriptor.server_descriptor.RelayDescriptor,
'_validate_content', test.mocking.no_op())
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA)
reader.register_skip_listener(skip_listener.listener)
expected_skip_files = ("riddle", "tiny.png", "vote",
"new_metrics_type")
with reader:
list(reader) # iterates over all of the descriptors
# strip anything with a .swp suffix (vim tmp files)
skip_listener.results = [(path, exc)
for (path, exc) in skip_listener.results
if not path.endswith(".swp")]
if len(skip_listener.results) != len(expected_skip_files):
expected_label = ",\n ".join(expected_skip_files)
results_label = ",\n ".join([
"%s (%s)" % (path, exc)
for (path, exc) in skip_listener.results
])
self.fail(
"Skipped files that we should have been able to parse.\n\nExpected:\n %s\n\nResult:\n %s"
% (expected_label, results_label))
for skip_path, skip_exception in skip_listener.results:
if not os.path.basename(skip_path) in expected_skip_files:
self.fail("Unexpected non-descriptor content: %s" % skip_path)
self.assertTrue(
isinstance(skip_exception,
stem.descriptor.reader.UnrecognizedType))
def test_skip_listener_file_missing(self):
"""
Listens for a file that's skipped because the file doesn't exist.
"""
test_path = '/non-existant/path'
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader:
list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.FileMissing))
def test_unrecognized_metrics_type(self):
"""
Parses a file that has a valid metrics header, but an unrecognized type.
"""
test_path = test.unit.descriptor.get_resource('unparseable/new_metrics_type')
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader:
list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
self.assertEqual((None, None), skip_exception.mime_type)
def read_descriptors(descriptors, descriptor_dir, skip_listener):
"""Add to descriptors contents of descriptor archive in descriptor_dir."""
num_descriptors = 0
num_relays = 0
print('Reading descriptors from: {0}'.format(descriptor_dir))
reader = stem.descriptor.reader.DescriptorReader(descriptor_dir,validate=True)
reader.register_skip_listener(skip_listener)
with reader:
for desc in reader:
if (num_descriptors % 10000 == 0):
print('{0} descriptors processed.'.format(num_descriptors))
num_descriptors += 1
if (desc.fingerprint not in descriptors):
descriptors[desc.fingerprint] = {}
num_relays += 1
descriptors[desc.fingerprint]\
[pathsim.timestamp(desc.published)] = desc
print('#descriptors: {0}; #relays:{1}'.\
format(num_descriptors,num_relays))
def test_skip_listener_read_failure(self):
"""
Listens for a file that's skipped because we lack read permissions.
"""
# test relies on being unable to read a file
if getpass.getuser() == 'root':
test.runner.skip(self, '(running as root)')
return
elif system.is_windows():
test.runner.skip(self, '(chmod not functional)')
return
test_path = os.path.join(self.temp_directory, 'secret_file')
try:
test_file = open(test_path, 'w')
test_file.write(
'test data for test_skip_listener_unrecognized_type()')
test_file.close()
os.chmod(test_path, 0o077) # remove read permissions
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader:
list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(
isinstance(skip_exception, stem.descriptor.reader.ReadFailed))
self.assertTrue(isinstance(skip_exception.exception, IOError))
finally:
if os.path.exists(test_path):
os.remove(test_path)
def read_descriptors(descriptors, descriptor_dir, skip_listener):
"""Add to descriptors contents of descriptor archive in descriptor_dir."""
num_descriptors = 0
num_relays = 0
print('Reading descriptors from: {0}'.format(descriptor_dir))
reader = stem.descriptor.reader.DescriptorReader(descriptor_dir,
validate=True)
reader.register_skip_listener(skip_listener)
with reader:
for desc in reader:
if (num_descriptors % 10000 == 0):
print('{0} descriptors processed.'.format(num_descriptors))
num_descriptors += 1
if (desc.fingerprint not in descriptors):
descriptors[desc.fingerprint] = {}
num_relays += 1
descriptors[desc.fingerprint]\
[pathsim.timestamp(desc.published)] = desc
print('#descriptors: {0}; #relays:{1}'.\
format(num_descriptors,num_relays))
def test_skip_nondescriptor_contents(self):
"""
Checks that the reader properly reports when it skips both binary and
plaintext non-descriptor files.
"""
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(DESCRIPTOR_TEST_DATA)
reader.register_skip_listener(skip_listener.listener)
with reader: list(reader) # iterates over all of the descriptors
self.assertEqual(4, len(skip_listener.results))
for skip_path, skip_exception in skip_listener.results:
if skip_path.endswith(".swp"): continue # skip vim temp files
if not os.path.basename(skip_path) in ("riddle", "tiny.png", "vote", "new_metrics_type"):
self.fail("Unexpected non-descriptor content: %s" % skip_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.UnrecognizedType))
def read_descriptors(descriptors, descriptor_dir, skip_listener):
"""Add to descriptors contents of descriptor archive in descriptor_dir."""
num_descriptors = 0
num_relays = 0
print('Reading descriptors from: {0}'.format(descriptor_dir))
reader = stem.descriptor.reader.DescriptorReader(descriptor_dir,
validate=True)
reader.register_skip_listener(skip_listener)
# use read listener to store metrics type annotation, which is otherwise discarded
cur_type_annotation = [None]
def read_listener(path):
f = open(path)
# store initial metrics type annotation
initial_position = f.tell()
first_line = f.readline()
f.seek(initial_position)
if (first_line[0:5] == '@type'):
cur_type_annotation[0] = first_line
else:
cur_type_annotation[0] = None
f.close()
reader.register_read_listener(read_listener)
with reader:
for desc in reader:
if (num_descriptors % 10000 == 0):
print('{0} descriptors processed.'.format(num_descriptors))
num_descriptors += 1
if (desc.fingerprint not in descriptors):
descriptors[desc.fingerprint] = {}
num_relays += 1
# stuff type annotation into stem object
desc.type_annotation = cur_type_annotation[0]
descriptors[desc.fingerprint]\
[pathsim.timestamp(desc.published)] = desc
print('#descriptors: {0}; #relays:{1}'.\
format(num_descriptors,num_relays))
def test_skip_listener_read_failure(self):
"""
Listens for a file that's skipped because we lack read permissions.
"""
# test relies on being unable to read a file
if getpass.getuser() == 'root':
self.skipTest('(running as root)')
return
elif system.is_windows():
self.skipTest('(chmod not functional)')
return
test_path = os.path.join(self.temp_directory, 'secret_file')
try:
test_file = open(test_path, 'w')
test_file.write('test data for test_skip_listener_unrecognized_type()')
test_file.close()
os.chmod(test_path, 0o077) # remove read permissions
skip_listener = SkipListener()
reader = stem.descriptor.reader.DescriptorReader(test_path)
reader.register_skip_listener(skip_listener.listener)
with reader:
list(reader) # iterates over all of the descriptors
self.assertEqual(1, len(skip_listener.results))
skipped_path, skip_exception = skip_listener.results[0]
self.assertEqual(test_path, skipped_path)
self.assertTrue(isinstance(skip_exception, stem.descriptor.reader.ReadFailed))
self.assertTrue(isinstance(skip_exception.exception, IOError))
finally:
if os.path.exists(test_path):
os.remove(test_path)
def read_descriptors(descriptors, descriptor_dir, skip_listener):
"""Add to descriptors contents of descriptor archive in descriptor_dir."""
num_descriptors = 0
num_relays = 0
print('Reading descriptors from: {0}'.format(descriptor_dir))
reader = stem.descriptor.reader.DescriptorReader(descriptor_dir,
validate=True)
reader.register_skip_listener(skip_listener)
# use read listener to store metrics type annotation, which is otherwise discarded
cur_type_annotation = [None]
def read_listener(path):
f = open(path)
# store initial metrics type annotation
initial_position = f.tell()
first_line = f.readline()
f.seek(initial_position)
if (first_line[0:5] == '@type'):
cur_type_annotation[0] = first_line
else:
cur_type_annotation[0] = None
f.close()
reader.register_read_listener(read_listener)
with reader:
for desc in reader:
if (num_descriptors % 10000 == 0):
print('{0} descriptors processed.'.format(num_descriptors))
num_descriptors += 1
if (desc.fingerprint not in descriptors):
descriptors[desc.fingerprint] = {}
num_relays += 1
# stuff type annotation into stem object
desc.type_annotation = cur_type_annotation[0]
descriptors[desc.fingerprint]\
[pathsim.timestamp(desc.published)] = desc
print('#descriptors: {0}; #relays:{1}'.\
format(num_descriptors,num_relays))