def open(self, request, data=None, timeout=None): if self.response.code != 200: raise urllib2.URLError(self.response.code) self.request = request self.data = data return self.response
def test_download_retries(self, urlopen_mock): urlopen_mock.side_effect = urllib.URLError('boom') self.assertRaisesRegexp(IOError, 'boom', stem.util.connection.download, URL) self.assertEqual(1, urlopen_mock.call_count) urlopen_mock.reset_mock() self.assertRaisesRegexp(IOError, 'boom', stem.util.connection.download, URL, retries = 4) self.assertEqual(5, urlopen_mock.call_count)
def _open(self, req, ssl, sslContext): if not req.host: raise request.URLError('packrat error: no host given') method = getattr(req, 'method', 'GET') if method == 'GET': return self._open_get(req, ssl, sslContext) elif method == 'POST': return self._open_post(req, ssl, sslContext) else: raise ValueError('Invalid Method "{0}"'.format(method))
def test_download_failure(self, urlopen_mock): urlopen_mock.side_effect = urllib.URLError('boom') try: stem.util.connection.download(URL) self.fail('expected a stem.DownloadFailed to be raised') except stem.DownloadFailed as exc: self.assertEqual('Failed to download from https://example.unit.test.url (URLError): boom', str(exc)) self.assertEqual(URL, exc.url) self.assertEqual('boom', exc.error.reason) self.assertEqual(urllib.URLError, type(exc.error)) self.assertTrue('return urllib.urlopen(url, timeout = timeout).read()' in exc.stacktrace_str)
def test_failure_general(self, mock_logging, mock_post): mock_post.side_effect = urllib2.URLError('something') # test infrastructure endpoint url creation a = netuitive.Client(api_key='apikey') e = netuitive.Event( 'test', 'INFO', 'test event', 'big old test message', 'INFO') resp = a.post_event(e) self.assertNotEqual(resp, True) self.assertEqual(mock_logging.exception.call_args_list[0][0][ 0], 'error posting payload to api ingest endpoint (%s): %s')
def test_failure_general(self, mock_logging, mock_post): mock_post.return_value = MockResponse(code=500) # test infrastructure endpoint url creation a = netuitive.Client(api_key='apikey') mock_post.side_effect = urllib2.URLError('something') e = netuitive.Element() e.add_sample( 'nonsparseDataStrategy', 1434110794, 1, 'COUNTER', host='hostname') resp = a.post(e) self.assertNotEqual(resp, True) self.assertEqual(mock_logging.exception.call_args_list[0][0][ 0], 'error posting payload to api ingest endpoint (%s): %s')
def get_langs(url: str = "https://www.gitignore.io/api/list") -> T.Set[str]: try: with open(TMP_CACHE, "r") as tmp_file: return set(l.strip() for l in tmp_file.readlines()) except: # fallback on the url resp = http.urlopen(url) if resp.getcode() != 200: raise http.URLError(f"Error: {resp.getcode()} {resp.reason}") res = set(LIST_SEPERATOR.split(resp.read().decode().strip())) try: with open(TMP_CACHE, "w") as tmp_file: for r in res: print(r, file=tmp_file) except: pass return res
def get_gitignore(langs: T.Iterable[str], base_url="https://www.gitignore.io/api/"): langs = list(langs) supported_langs = get_langs() for l in langs: if l not in supported_langs: raise Exception(f"language '{l}' is not supported by gitignore.io") # make sure that they're all supported query = quote(",".join(l.lower().strip() for l in langs)) if query == "": return "" url = base_url + ("" if base_url[-1] == "" else "/") + query try: resp = http.urlopen(url) except: print(f"Failed to get url: {url}") raise if resp.getcode() != 200: raise http.URLError(f"Error: {resp.getcode()} {resp.reason}") return resp.read().decode()
class TestManual(unittest.TestCase): def test_has_all_summaries(self): """ Check that we have brief, human readable summaries for all of tor's configuration options. If you add a new config entry then please take a sec to write a little summary. They're located in 'stem/settings.cfg'. """ manual = _cached_manual() present = set(manual.config_options.keys()) expected = set([ key[15:] for key in stem.manual._config(lowercase=False) if key.startswith('manual.summary.') ]) missing_options = present.difference(expected) extra_options = expected.difference(present) if missing_options: self.fail( "Ran cache_manual.py? Please update Stem's settings.cfg with summaries of the following config options: %s" % ', '.join(missing_options)) elif extra_options: self.fail( "Ran cache_manual.py? Please remove the following summaries from Stem's settings.cfg: %s" % ', '.join(extra_options)) def test_is_important(self): self.assertTrue(stem.manual.is_important('ExitPolicy')) self.assertTrue(stem.manual.is_important('exitpolicy')) self.assertTrue(stem.manual.is_important('EXITPOLICY')) self.assertFalse(stem.manual.is_important('ConstrainedSockSize')) def test_minimal_config_option(self): blank = stem.manual.ConfigOption('UnknownOption') self.assertEqual(stem.manual.Category.UNKNOWN, blank.category) self.assertEqual('UnknownOption', blank.name) self.assertEqual('', blank.usage) self.assertEqual('', blank.summary) self.assertEqual('', blank.description) @test.require.command('man') def test_parsing_with_example(self): """ Read a trimmed copy of tor's man page. This gives a good exercise of our parser with static content. As new oddball man oddities appear feel free to expand our example (or add another). """ if stem.util.system.is_mac(): self.skipTest('(man lacks --encoding arg on OSX, #18660)') return manual = stem.manual.Manual.from_man(EXAMPLE_MAN_PATH) self.assertEqual('tor - The second-generation onion router', manual.name) self.assertEqual('tor [OPTION value]...', manual.synopsis) self.assertEqual(EXPECTED_DESCRIPTION, manual.description) self.assertEqual(EXPECTED_CLI_OPTIONS, manual.commandline_options) self.assertEqual(EXPECTED_SIGNALS, manual.signals) self.assertEqual(EXPECTED_FILES, manual.files) self.assertEqual(EXPECTED_CONFIG_OPTIONS, manual.config_options) @test.require.command('man') def test_parsing_with_unknown_options(self): """ Check that we can read a local mock man page that contains unrecognized options. Unlike most other tests this doesn't require network access. """ if stem.util.system.is_mac(): self.skipTest('(man lacks --encoding arg on OSX, #18660)') return manual = stem.manual.Manual.from_man(UNKNOWN_OPTIONS_MAN_PATH) self.assertEqual('tor - The second-generation onion router', manual.name) self.assertEqual('', manual.synopsis) self.assertEqual('', manual.description) self.assertEqual({}, manual.commandline_options) self.assertEqual({}, manual.signals) self.assertEqual({}, manual.files) self.assertEqual(2, len(manual.config_options)) option = [ entry for entry in manual.config_options.values() if entry.category == stem.manual.Category.UNKNOWN ][0] self.assertEqual(stem.manual.Category.UNKNOWN, option.category) self.assertEqual('SpiffyNewOption', option.name) self.assertEqual('transport exec path-to-binary [options]', option.usage) self.assertEqual('', option.summary) self.assertEqual('Description of this new option.', option.description) @test.require.command('man') def test_saving_manual(self): """ Check that we can save and reload manuals. """ manual = stem.manual.Manual.from_man(EXAMPLE_MAN_PATH) with tempfile.NamedTemporaryFile(prefix='saved_test_manual.') as tmp: manual.save(tmp.name) loaded_manual = stem.manual.Manual.from_cache(tmp.name) self.assertEqual(manual, loaded_manual) def test_cached_manual(self): manual = _cached_manual() self.assertEqual('tor - The second-generation onion router', manual.name) self.assertEqual('tor [OPTION value]...', manual.synopsis) self.assertTrue(manual.description.startswith(EXPECTED_DESCRIPTION)) self.assertTrue(len(manual.commandline_options) > 10) self.assertTrue(len(manual.signals) > 5) self.assertTrue(len(manual.files) > 20) self.assertTrue(len(manual.config_options) > 200) def test_download_man_page_without_arguments(self): exc_msg = "Either the path or file_handle we're saving to must be provided" self.assertRaisesRegexp(ValueError, exc_msg, stem.manual.download_man_page) @patch('stem.util.system.is_available', Mock(return_value=False)) def test_download_man_page_requires_a2x(self): exc_msg = 'We require a2x from asciidoc to provide a man page' self.assertRaisesRegexp(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file') @patch('tempfile.mkdtemp', Mock(return_value='/no/such/path')) @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(side_effect=IOError('unable to write to file')), create=True) @patch('stem.util.system.is_available', Mock(return_value=True)) def test_download_man_page_when_unable_to_write(self): exc_msg = "Unable to download tor's manual from https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt to /no/such/path/tor.1.txt: unable to write to file" self.assertRaisesRegexp(IOError, re.escape(exc_msg), stem.manual.download_man_page, '/tmp/no_such_file') @patch('tempfile.mkdtemp', Mock(return_value='/no/such/path')) @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(return_value=io.BytesIO()), create=True) @patch('stem.util.system.is_available', Mock(return_value=True)) @patch(URL_OPEN, Mock(side_effect=urllib.URLError( '<urlopen error [Errno -2] Name or service not known>'))) def test_download_man_page_when_download_fails(self): exc_msg = "Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>" self.assertRaisesRegexp(IOError, re.escape(exc_msg), stem.manual.download_man_page, '/tmp/no_such_file', url='https://www.atagar.com/foo/bar') @patch('tempfile.mkdtemp', Mock(return_value='/no/such/path')) @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(return_value=io.BytesIO()), create=True) @patch('stem.util.system.call', Mock(side_effect=stem.util.system.CallError( 'call failed', 'a2x -f manpage /no/such/path/tor.1.txt', 1, None, None, 'call failed'))) @patch('stem.util.system.is_available', Mock(return_value=True)) @patch(URL_OPEN, Mock(return_value=io.BytesIO(b'test content'))) def test_download_man_page_when_a2x_fails(self): exc_msg = "Unable to run 'a2x -f manpage /no/such/path/tor.1.txt': call failed" self.assertRaisesRegexp(IOError, exc_msg, stem.manual.download_man_page, '/tmp/no_such_file', url='https://www.atagar.com/foo/bar') @patch('tempfile.mkdtemp', Mock(return_value='/no/such/path')) @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', create=True) @patch('stem.util.system.call') @patch('stem.util.system.is_available', Mock(return_value=True)) @patch('os.path.exists', Mock(return_value=True)) @patch(URL_OPEN, Mock(return_value=io.BytesIO(b'test content'))) def test_download_man_page_when_successful(self, call_mock, open_mock): open_mock.side_effect = lambda path, *args: { '/no/such/path/tor.1.txt': io.BytesIO(), '/no/such/path/tor.1': io.BytesIO(b'a2x output'), }[path] call_mock.return_value = Mock() output = io.BytesIO() stem.manual.download_man_page(file_handle=output) self.assertEqual(b'a2x output', output.getvalue()) call_mock.assert_called_once_with( 'a2x -f manpage /no/such/path/tor.1.txt') @patch('stem.util.system.is_mac', Mock(return_value=False)) @patch('stem.util.system.call', Mock(side_effect=OSError( 'man --encoding=ascii -P cat tor returned exit status 16'))) def test_from_man_when_manual_is_unavailable(self): exc_msg = "Unable to run 'man --encoding=ascii -P cat tor': man --encoding=ascii -P cat tor returned exit status 16" self.assertRaisesRegexp(IOError, exc_msg, stem.manual.Manual.from_man) @patch('stem.util.system.call', Mock(return_value=[])) def test_when_man_is_empty(self): manual = stem.manual.Manual.from_man() self.assertEqual('', manual.name) self.assertEqual('', manual.synopsis) self.assertEqual('', manual.description) self.assertEqual({}, manual.commandline_options) self.assertEqual({}, manual.signals) self.assertEqual({}, manual.files) self.assertEqual(OrderedDict(), manual.config_options)
def urlopen_error_response(): return request.URLError('from test')
class TestManual(unittest.TestCase): def test_is_important(self): self.assertTrue(stem.manual.is_important('ExitPolicy')) self.assertTrue(stem.manual.is_important('exitpolicy')) self.assertTrue(stem.manual.is_important('EXITPOLICY')) self.assertFalse(stem.manual.is_important('ConstrainedSockSize')) def test_minimal_config_option(self): blank = stem.manual.ConfigOption('UnknownOption') self.assertEqual(stem.manual.Category.UNKNOWN, blank.category) self.assertEqual('UnknownOption', blank.name) self.assertEqual('', blank.usage) self.assertEqual('', blank.summary) self.assertEqual('', blank.description) def test_parsing_with_example(self): """ Read a trimmed copy of tor's man page. This gives a good exercise of our parser with static content. As new oddball man oddities appear feel free to expand our example (or add another). """ if not stem.util.system.is_available('man'): test.runner.skip(self, '(require man command)') return elif stem.util.system.is_mac(): test.runner.skip(self, '(man lacks --encoding arg on OSX, #18660)') return manual = stem.manual.Manual.from_man(EXAMPLE_MAN_PATH) self.assertEqual('tor - The second-generation onion router', manual.name) self.assertEqual('tor [OPTION value]...', manual.synopsis) self.assertEqual(EXPECTED_DESCRIPTION, manual.description) self.assertEqual(EXPECTED_CLI_OPTIONS, manual.commandline_options) self.assertEqual(EXPECTED_SIGNALS, manual.signals) self.assertEqual(EXPECTED_FILES, manual.files) self.assertEqual(EXPECTED_CONFIG_OPTIONS, manual.config_options) def test_parsing_with_unknown_options(self): """ Check that we can read a local mock man page that contains unrecognized options. Unlike most other tests this doesn't require network access. """ if not stem.util.system.is_available('man'): test.runner.skip(self, '(require man command)') return elif stem.util.system.is_mac(): test.runner.skip(self, '(man lacks --encoding arg on OSX, #18660)') return manual = stem.manual.Manual.from_man(UNKNOWN_OPTIONS_MAN_PATH) self.assertEqual('tor - The second-generation onion router', manual.name) self.assertEqual('', manual.synopsis) self.assertEqual('', manual.description) self.assertEqual({}, manual.commandline_options) self.assertEqual({}, manual.signals) self.assertEqual({}, manual.files) self.assertEqual(2, len(manual.config_options)) option = [ entry for entry in manual.config_options.values() if entry.category == stem.manual.Category.UNKNOWN ][0] self.assertEqual(stem.manual.Category.UNKNOWN, option.category) self.assertEqual('SpiffyNewOption', option.name) self.assertEqual('transport exec path-to-binary [options]', option.usage) self.assertEqual('', option.summary) self.assertEqual('Description of this new option.', option.description) def test_saving_manual(self): """ Check that we can save and reload manuals. """ if not stem.util.system.is_available('man'): test.runner.skip(self, '(require man command)') return manual = stem.manual.Manual.from_man(EXAMPLE_MAN_PATH) with tempfile.NamedTemporaryFile(prefix='saved_test_manual.') as tmp: manual.save(tmp.name) loaded_manual = stem.manual.Manual.from_cache(tmp.name) self.assertEqual(manual, loaded_manual) def test_cached_manual(self): manual = stem.manual.Manual.from_cache() self.assertEqual('tor - The second-generation onion router', manual.name) self.assertEqual('tor [OPTION value]...', manual.synopsis) self.assertTrue(manual.description.startswith(EXPECTED_DESCRIPTION)) self.assertTrue(len(manual.commandline_options) > 10) self.assertTrue(len(manual.signals) > 5) self.assertTrue(len(manual.files) > 20) self.assertTrue(len(manual.config_options) > 200) def test_download_man_page_without_arguments(self): try: stem.manual.download_man_page() self.fail('we should fail without a path or file handler') except ValueError as exc: self.assertEqual( "Either the path or file_handle we're saving to must be provided", str(exc)) @patch('stem.util.system.is_available', Mock(return_value=False)) def test_download_man_page_requires_a2x(self): try: stem.manual.download_man_page('/tmp/no_such_file') self.fail('we should require a2x to be available') except IOError as exc: self.assertEqual( 'We require a2x from asciidoc to provide a man page', str(exc)) @patch('tempfile.mkdtemp', Mock(return_value='/no/such/path')) @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(side_effect=IOError('unable to write to file')), create=True) @patch('stem.util.system.is_available', Mock(return_value=True)) def test_download_man_page_when_unable_to_write(self): try: stem.manual.download_man_page('/tmp/no_such_file') self.fail("we shouldn't be able to write to /no/such/path") except IOError as exc: self.assertEqual( "Unable to download tor's manual from https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt to /no/such/path/tor.1.txt: unable to write to file", str(exc)) @patch('tempfile.mkdtemp', Mock(return_value='/no/such/path')) @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(return_value=io.BytesIO()), create=True) @patch('stem.util.system.is_available', Mock(return_value=True)) @patch(URL_OPEN, Mock(side_effect=urllib.URLError( '<urlopen error [Errno -2] Name or service not known>'))) def test_download_man_page_when_download_fails(self): try: stem.manual.download_man_page('/tmp/no_such_file', url='https://www.atagar.com/foo/bar') self.fail("downloading from test_invalid_url.org shouldn't work") except IOError as exc: self.assertEqual( "Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>", str(exc)) @patch('tempfile.mkdtemp', Mock(return_value='/no/such/path')) @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', Mock(return_value=io.BytesIO()), create=True) @patch('stem.util.system.call', Mock(side_effect=stem.util.system.CallError( 'call failed', 'a2x -f manpage /no/such/path/tor.1.txt', 1, None, None, 'call failed'))) @patch('stem.util.system.is_available', Mock(return_value=True)) @patch(URL_OPEN, Mock(return_value=io.BytesIO(b'test content'))) def test_download_man_page_when_a2x_fails(self): try: stem.manual.download_man_page('/tmp/no_such_file', url='https://www.atagar.com/foo/bar') self.fail("downloading from test_invalid_url.org shouldn't work") except IOError as exc: self.assertEqual( "Unable to run 'a2x -f manpage /no/such/path/tor.1.txt': call failed", str(exc)) @patch('tempfile.mkdtemp', Mock(return_value='/no/such/path')) @patch('shutil.rmtree', Mock()) @patch('stem.manual.open', create=True) @patch('stem.util.system.call') @patch('stem.util.system.is_available', Mock(return_value=True)) @patch('os.path.exists', Mock(return_value=True)) @patch(URL_OPEN, Mock(return_value=io.BytesIO(b'test content'))) def test_download_man_page_when_successful(self, call_mock, open_mock): open_mock.side_effect = lambda path, *args: { '/no/such/path/tor.1.txt': io.BytesIO(), '/no/such/path/tor.1': io.BytesIO(b'a2x output'), }[path] call_mock.return_value = Mock() output = io.BytesIO() stem.manual.download_man_page(file_handle=output) self.assertEqual(b'a2x output', output.getvalue()) call_mock.assert_called_once_with( 'a2x -f manpage /no/such/path/tor.1.txt') @patch('stem.util.system.is_mac', Mock(return_value=False)) @patch('stem.util.system.call', Mock(side_effect=OSError( 'man --encoding=ascii -P cat tor returned exit status 16'))) def test_from_man_when_manual_is_unavailable(self): try: stem.manual.Manual.from_man() self.fail("fetching the manual should fail when it's unavailable") except IOError as exc: self.assertEqual( "Unable to run 'man --encoding=ascii -P cat tor': man --encoding=ascii -P cat tor returned exit status 16", str(exc)) @patch('stem.util.system.call', Mock(return_value=[])) def test_when_man_is_empty(self): manual = stem.manual.Manual.from_man() self.assertEqual('', manual.name) self.assertEqual('', manual.synopsis) self.assertEqual('', manual.description) self.assertEqual({}, manual.commandline_options) self.assertEqual({}, manual.signals) self.assertEqual({}, manual.files) self.assertEqual(OrderedDict(), manual.config_options)
def test_get_urlopen_with_urlerror(self, mock_urlopen): mock_urlopen.side_effect = urllib2.URLError('Error Reason') ret_val = tvdb._get('http://url.com') self.assertEqual(ret_val, '')