def test_index_retries(self, urlopen_mock): urlopen_mock.side_effect = IOError('boom') collector = CollecTor(retries = 0) self.assertRaisesRegexp(IOError, 'boom', collector.index) self.assertEqual(1, urlopen_mock.call_count) urlopen_mock.reset_mock() collector = CollecTor(retries = 4) self.assertRaisesRegexp(IOError, 'boom', collector.index) self.assertEqual(5, urlopen_mock.call_count)
def test_index_malformed_json(self): collector = CollecTor() if stem.prereq.is_python_3(): self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index, Compression.PLAINTEXT) else: self.assertRaisesRegexp(ValueError, 'No JSON object could be decoded', collector.index, Compression.PLAINTEXT)
def test_index_plaintext(self, urlopen_mock): urlopen_mock.return_value = io.BytesIO(EXAMPLE_INDEX_JSON) collector = CollecTor() self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.PLAINTEXT)) urlopen_mock.assert_called_with( 'https://collector.torproject.org/index/index.json', timeout=None)
def test_file_by_date(self): collector = CollecTor() self.assertEqual([ 'recent/relay-descriptors/server-descriptors/2019-07-03-02-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-03-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-07-03-04-05-00-server-descriptors', ], [ f.path for f in collector.files(descriptor_type='server-descriptor', start=datetime.datetime(2007, 1, 1)) ]) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', ], [ f.path for f in collector.files(descriptor_type='server-descriptor', end=datetime.datetime(2007, 1, 1)) ]) self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', ], [ f.path for f in collector.files(descriptor_type='server-descriptor', start=datetime.datetime(2006, 2, 10), end=datetime.datetime(2007, 1, 1)) ])
def test_index_malformed_compression(self): for compression in (Compression.GZIP, Compression.BZ2, Compression.LZMA): if not compression.available: continue with patch(URL_OPEN, Mock(return_value = io.BytesIO(b'not compressed'))): collector = CollecTor() self.assertRaisesRegexp(IOError, 'Failed to decompress as %s' % compression, collector.index, compression)
def test_files(self): collector = CollecTor() files = collector.files() self.assertEqual(96, len(files)) extrainfo_file = list(filter(lambda x: x.path.endswith('extra-infos-2007-09.tar.xz'), files))[0] self.assertEqual('archive/relay-descriptors/extra-infos/extra-infos-2007-09.tar.xz', extrainfo_file.path) self.assertEqual(Compression.LZMA, extrainfo_file.compression) self.assertEqual(6459884, extrainfo_file.size) self.assertEqual(datetime.datetime(2016, 6, 23, 9, 54), extrainfo_file.last_modified)
def test_index_lzma(self, urlopen_mock): if not Compression.LZMA.available: self.skipTest('(lzma compression unavailable)') return import lzma urlopen_mock.return_value = io.BytesIO(lzma.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.LZMA)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.xz', timeout = None)
def test_index_gzip(self, urlopen_mock): if not Compression.GZIP.available: self.skipTest('(gzip compression unavailable)') return import zlib urlopen_mock.return_value = io.BytesIO(zlib.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.GZIP)) urlopen_mock.assert_called_with('https://collector.torproject.org/index/index.json.gz', timeout = None)
def test_files_by_descriptor_type(self): collector = CollecTor() self.assertEqual([ 'archive/relay-descriptors/server-descriptors/server-descriptors-2005-12.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-02.tar.xz', 'archive/relay-descriptors/server-descriptors/server-descriptors-2006-03.tar.xz', 'recent/relay-descriptors/server-descriptors/2019-11-28-01-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-11-28-00-05-00-server-descriptors', 'recent/relay-descriptors/server-descriptors/2019-11-27-23-05-00-server-descriptors', ], [f.path for f in collector.files(descriptor_type = 'server-descriptor')])
def test_index_bz2(self, urlopen_mock): if not Compression.BZ2.available: self.skipTest('(bz2 compression unavailable)') import bz2 urlopen_mock.return_value = io.BytesIO( bz2.compress(EXAMPLE_INDEX_JSON)) collector = CollecTor() self.assertEqual(EXAMPLE_INDEX, collector.index(Compression.BZ2)) urlopen_mock.assert_called_with( 'https://collector.torproject.org/index/index.json.bz2', timeout=None)
def test_index_malformed_json(self): collector = CollecTor() self.assertRaisesRegexp(ValueError, 'Expecting value: line 1 column 1', collector.index, Compression.PLAINTEXT)