def open(self, request, data=None, timeout=None):
if self.response.code != 200:
raise urllib2.URLError(self.response.code)
self.request = request
self.data = data
return self.response
def test_download_retries(self, urlopen_mock):
urlopen_mock.side_effect = urllib.URLError('boom')
self.assertRaisesRegexp(IOError, 'boom', stem.util.connection.download, URL)
self.assertEqual(1, urlopen_mock.call_count)
urlopen_mock.reset_mock()
self.assertRaisesRegexp(IOError, 'boom', stem.util.connection.download, URL, retries = 4)
self.assertEqual(5, urlopen_mock.call_count)
def _open(self, req, ssl, sslContext):
if not req.host:
raise request.URLError('packrat error: no host given')
method = getattr(req, 'method', 'GET')
if method == 'GET':
return self._open_get(req, ssl, sslContext)
elif method == 'POST':
return self._open_post(req, ssl, sslContext)
else:
raise ValueError('Invalid Method "{0}"'.format(method))
def test_download_failure(self, urlopen_mock):
urlopen_mock.side_effect = urllib.URLError('boom')
try:
stem.util.connection.download(URL)
self.fail('expected a stem.DownloadFailed to be raised')
except stem.DownloadFailed as exc:
self.assertEqual('Failed to download from https://example.unit.test.url (URLError): boom', str(exc))
self.assertEqual(URL, exc.url)
self.assertEqual('boom', exc.error.reason)
self.assertEqual(urllib.URLError, type(exc.error))
self.assertTrue('return urllib.urlopen(url, timeout = timeout).read()' in exc.stacktrace_str)
def test_failure_general(self, mock_logging, mock_post):
mock_post.side_effect = urllib2.URLError('something')
# test infrastructure endpoint url creation
a = netuitive.Client(api_key='apikey')
e = netuitive.Event(
'test', 'INFO', 'test event', 'big old test message', 'INFO')
resp = a.post_event(e)
self.assertNotEqual(resp, True)
self.assertEqual(mock_logging.exception.call_args_list[0][0][
0], 'error posting payload to api ingest endpoint (%s): %s')
def test_failure_general(self, mock_logging, mock_post):
mock_post.return_value = MockResponse(code=500)
# test infrastructure endpoint url creation
a = netuitive.Client(api_key='apikey')
mock_post.side_effect = urllib2.URLError('something')
e = netuitive.Element()
e.add_sample(
'nonsparseDataStrategy', 1434110794, 1, 'COUNTER', host='hostname')
resp = a.post(e)
self.assertNotEqual(resp, True)
self.assertEqual(mock_logging.exception.call_args_list[0][0][
0], 'error posting payload to api ingest endpoint (%s): %s')
def get_langs(url: str = "https://www.gitignore.io/api/list") -> T.Set[str]:
try:
with open(TMP_CACHE, "r") as tmp_file:
return set(l.strip() for l in tmp_file.readlines())
except:
# fallback on the url
resp = http.urlopen(url)
if resp.getcode() != 200:
raise http.URLError(f"Error: {resp.getcode()} {resp.reason}")
res = set(LIST_SEPERATOR.split(resp.read().decode().strip()))
try:
with open(TMP_CACHE, "w") as tmp_file:
for r in res:
print(r, file=tmp_file)
except:
pass
return res
def get_gitignore(langs: T.Iterable[str],
base_url="https://www.gitignore.io/api/"):
langs = list(langs)
supported_langs = get_langs()
for l in langs:
if l not in supported_langs:
raise Exception(f"language '{l}' is not supported by gitignore.io")
# make sure that they're all supported
query = quote(",".join(l.lower().strip() for l in langs))
if query == "":
return ""
url = base_url + ("" if base_url[-1] == "" else "/") + query
try:
resp = http.urlopen(url)
except:
print(f"Failed to get url: {url}")
raise
if resp.getcode() != 200:
raise http.URLError(f"Error: {resp.getcode()} {resp.reason}")
return resp.read().decode()
class TestManual(unittest.TestCase):
def test_has_all_summaries(self):
"""
Check that we have brief, human readable summaries for all of tor's
configuration options. If you add a new config entry then please take a sec
to write a little summary. They're located in 'stem/settings.cfg'.
"""
manual = _cached_manual()
present = set(manual.config_options.keys())
expected = set([
key[15:] for key in stem.manual._config(lowercase=False)
if key.startswith('manual.summary.')
])
missing_options = present.difference(expected)
extra_options = expected.difference(present)
if missing_options:
self.fail(
"Ran cache_manual.py? Please update Stem's settings.cfg with summaries of the following config options: %s"
% ', '.join(missing_options))
elif extra_options:
self.fail(
"Ran cache_manual.py? Please remove the following summaries from Stem's settings.cfg: %s"
% ', '.join(extra_options))
def test_is_important(self):
self.assertTrue(stem.manual.is_important('ExitPolicy'))
self.assertTrue(stem.manual.is_important('exitpolicy'))
self.assertTrue(stem.manual.is_important('EXITPOLICY'))
self.assertFalse(stem.manual.is_important('ConstrainedSockSize'))
def test_minimal_config_option(self):
blank = stem.manual.ConfigOption('UnknownOption')
self.assertEqual(stem.manual.Category.UNKNOWN, blank.category)
self.assertEqual('UnknownOption', blank.name)
self.assertEqual('', blank.usage)
self.assertEqual('', blank.summary)
self.assertEqual('', blank.description)
@test.require.command('man')
def test_parsing_with_example(self):
"""
Read a trimmed copy of tor's man page. This gives a good exercise of our
parser with static content. As new oddball man oddities appear feel free to
expand our example (or add another).
"""
if stem.util.system.is_mac():
self.skipTest('(man lacks --encoding arg on OSX, #18660)')
return
manual = stem.manual.Manual.from_man(EXAMPLE_MAN_PATH)
self.assertEqual('tor - The second-generation onion router',
manual.name)
self.assertEqual('tor [OPTION value]...', manual.synopsis)
self.assertEqual(EXPECTED_DESCRIPTION, manual.description)
self.assertEqual(EXPECTED_CLI_OPTIONS, manual.commandline_options)
self.assertEqual(EXPECTED_SIGNALS, manual.signals)
self.assertEqual(EXPECTED_FILES, manual.files)
self.assertEqual(EXPECTED_CONFIG_OPTIONS, manual.config_options)
@test.require.command('man')
def test_parsing_with_unknown_options(self):
"""
Check that we can read a local mock man page that contains unrecognized
options. Unlike most other tests this doesn't require network access.
"""
if stem.util.system.is_mac():
self.skipTest('(man lacks --encoding arg on OSX, #18660)')
return
manual = stem.manual.Manual.from_man(UNKNOWN_OPTIONS_MAN_PATH)
self.assertEqual('tor - The second-generation onion router',
manual.name)
self.assertEqual('', manual.synopsis)
self.assertEqual('', manual.description)
self.assertEqual({}, manual.commandline_options)
self.assertEqual({}, manual.signals)
self.assertEqual({}, manual.files)
self.assertEqual(2, len(manual.config_options))
option = [
entry for entry in manual.config_options.values()
if entry.category == stem.manual.Category.UNKNOWN
][0]
self.assertEqual(stem.manual.Category.UNKNOWN, option.category)
self.assertEqual('SpiffyNewOption', option.name)
self.assertEqual('transport exec path-to-binary [options]',
option.usage)
self.assertEqual('', option.summary)
self.assertEqual('Description of this new option.', option.description)
@test.require.command('man')
def test_saving_manual(self):
"""
Check that we can save and reload manuals.
"""
manual = stem.manual.Manual.from_man(EXAMPLE_MAN_PATH)
with tempfile.NamedTemporaryFile(prefix='saved_test_manual.') as tmp:
manual.save(tmp.name)
loaded_manual = stem.manual.Manual.from_cache(tmp.name)
self.assertEqual(manual, loaded_manual)
def test_cached_manual(self):
manual = _cached_manual()
self.assertEqual('tor - The second-generation onion router',
manual.name)
self.assertEqual('tor [OPTION value]...', manual.synopsis)
self.assertTrue(manual.description.startswith(EXPECTED_DESCRIPTION))
self.assertTrue(len(manual.commandline_options) > 10)
self.assertTrue(len(manual.signals) > 5)
self.assertTrue(len(manual.files) > 20)
self.assertTrue(len(manual.config_options) > 200)
def test_download_man_page_without_arguments(self):
exc_msg = "Either the path or file_handle we're saving to must be provided"
self.assertRaisesRegexp(ValueError, exc_msg,
stem.manual.download_man_page)
@patch('stem.util.system.is_available', Mock(return_value=False))
def test_download_man_page_requires_a2x(self):
exc_msg = 'We require a2x from asciidoc to provide a man page'
self.assertRaisesRegexp(IOError, exc_msg,
stem.manual.download_man_page,
'/tmp/no_such_file')
@patch('tempfile.mkdtemp', Mock(return_value='/no/such/path'))
@patch('shutil.rmtree', Mock())
@patch('stem.manual.open',
Mock(side_effect=IOError('unable to write to file')),
create=True)
@patch('stem.util.system.is_available', Mock(return_value=True))
def test_download_man_page_when_unable_to_write(self):
exc_msg = "Unable to download tor's manual from https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt to /no/such/path/tor.1.txt: unable to write to file"
self.assertRaisesRegexp(IOError, re.escape(exc_msg),
stem.manual.download_man_page,
'/tmp/no_such_file')
@patch('tempfile.mkdtemp', Mock(return_value='/no/such/path'))
@patch('shutil.rmtree', Mock())
@patch('stem.manual.open', Mock(return_value=io.BytesIO()), create=True)
@patch('stem.util.system.is_available', Mock(return_value=True))
@patch(URL_OPEN,
Mock(side_effect=urllib.URLError(
'<urlopen error [Errno -2] Name or service not known>')))
def test_download_man_page_when_download_fails(self):
exc_msg = "Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>"
self.assertRaisesRegexp(IOError,
re.escape(exc_msg),
stem.manual.download_man_page,
'/tmp/no_such_file',
url='https://www.atagar.com/foo/bar')
@patch('tempfile.mkdtemp', Mock(return_value='/no/such/path'))
@patch('shutil.rmtree', Mock())
@patch('stem.manual.open', Mock(return_value=io.BytesIO()), create=True)
@patch('stem.util.system.call',
Mock(side_effect=stem.util.system.CallError(
'call failed', 'a2x -f manpage /no/such/path/tor.1.txt', 1,
None, None, 'call failed')))
@patch('stem.util.system.is_available', Mock(return_value=True))
@patch(URL_OPEN, Mock(return_value=io.BytesIO(b'test content')))
def test_download_man_page_when_a2x_fails(self):
exc_msg = "Unable to run 'a2x -f manpage /no/such/path/tor.1.txt': call failed"
self.assertRaisesRegexp(IOError,
exc_msg,
stem.manual.download_man_page,
'/tmp/no_such_file',
url='https://www.atagar.com/foo/bar')
@patch('tempfile.mkdtemp', Mock(return_value='/no/such/path'))
@patch('shutil.rmtree', Mock())
@patch('stem.manual.open', create=True)
@patch('stem.util.system.call')
@patch('stem.util.system.is_available', Mock(return_value=True))
@patch('os.path.exists', Mock(return_value=True))
@patch(URL_OPEN, Mock(return_value=io.BytesIO(b'test content')))
def test_download_man_page_when_successful(self, call_mock, open_mock):
open_mock.side_effect = lambda path, *args: {
'/no/such/path/tor.1.txt': io.BytesIO(),
'/no/such/path/tor.1': io.BytesIO(b'a2x output'),
}[path]
call_mock.return_value = Mock()
output = io.BytesIO()
stem.manual.download_man_page(file_handle=output)
self.assertEqual(b'a2x output', output.getvalue())
call_mock.assert_called_once_with(
'a2x -f manpage /no/such/path/tor.1.txt')
@patch('stem.util.system.is_mac', Mock(return_value=False))
@patch('stem.util.system.call',
Mock(side_effect=OSError(
'man --encoding=ascii -P cat tor returned exit status 16')))
def test_from_man_when_manual_is_unavailable(self):
exc_msg = "Unable to run 'man --encoding=ascii -P cat tor': man --encoding=ascii -P cat tor returned exit status 16"
self.assertRaisesRegexp(IOError, exc_msg, stem.manual.Manual.from_man)
@patch('stem.util.system.call', Mock(return_value=[]))
def test_when_man_is_empty(self):
manual = stem.manual.Manual.from_man()
self.assertEqual('', manual.name)
self.assertEqual('', manual.synopsis)
self.assertEqual('', manual.description)
self.assertEqual({}, manual.commandline_options)
self.assertEqual({}, manual.signals)
self.assertEqual({}, manual.files)
self.assertEqual(OrderedDict(), manual.config_options)
def urlopen_error_response():
return request.URLError('from test')
class TestManual(unittest.TestCase):
def test_is_important(self):
self.assertTrue(stem.manual.is_important('ExitPolicy'))
self.assertTrue(stem.manual.is_important('exitpolicy'))
self.assertTrue(stem.manual.is_important('EXITPOLICY'))
self.assertFalse(stem.manual.is_important('ConstrainedSockSize'))
def test_minimal_config_option(self):
blank = stem.manual.ConfigOption('UnknownOption')
self.assertEqual(stem.manual.Category.UNKNOWN, blank.category)
self.assertEqual('UnknownOption', blank.name)
self.assertEqual('', blank.usage)
self.assertEqual('', blank.summary)
self.assertEqual('', blank.description)
def test_parsing_with_example(self):
"""
Read a trimmed copy of tor's man page. This gives a good exercise of our
parser with static content. As new oddball man oddities appear feel free to
expand our example (or add another).
"""
if not stem.util.system.is_available('man'):
test.runner.skip(self, '(require man command)')
return
elif stem.util.system.is_mac():
test.runner.skip(self, '(man lacks --encoding arg on OSX, #18660)')
return
manual = stem.manual.Manual.from_man(EXAMPLE_MAN_PATH)
self.assertEqual('tor - The second-generation onion router',
manual.name)
self.assertEqual('tor [OPTION value]...', manual.synopsis)
self.assertEqual(EXPECTED_DESCRIPTION, manual.description)
self.assertEqual(EXPECTED_CLI_OPTIONS, manual.commandline_options)
self.assertEqual(EXPECTED_SIGNALS, manual.signals)
self.assertEqual(EXPECTED_FILES, manual.files)
self.assertEqual(EXPECTED_CONFIG_OPTIONS, manual.config_options)
def test_parsing_with_unknown_options(self):
"""
Check that we can read a local mock man page that contains unrecognized
options. Unlike most other tests this doesn't require network access.
"""
if not stem.util.system.is_available('man'):
test.runner.skip(self, '(require man command)')
return
elif stem.util.system.is_mac():
test.runner.skip(self, '(man lacks --encoding arg on OSX, #18660)')
return
manual = stem.manual.Manual.from_man(UNKNOWN_OPTIONS_MAN_PATH)
self.assertEqual('tor - The second-generation onion router',
manual.name)
self.assertEqual('', manual.synopsis)
self.assertEqual('', manual.description)
self.assertEqual({}, manual.commandline_options)
self.assertEqual({}, manual.signals)
self.assertEqual({}, manual.files)
self.assertEqual(2, len(manual.config_options))
option = [
entry for entry in manual.config_options.values()
if entry.category == stem.manual.Category.UNKNOWN
][0]
self.assertEqual(stem.manual.Category.UNKNOWN, option.category)
self.assertEqual('SpiffyNewOption', option.name)
self.assertEqual('transport exec path-to-binary [options]',
option.usage)
self.assertEqual('', option.summary)
self.assertEqual('Description of this new option.', option.description)
def test_saving_manual(self):
"""
Check that we can save and reload manuals.
"""
if not stem.util.system.is_available('man'):
test.runner.skip(self, '(require man command)')
return
manual = stem.manual.Manual.from_man(EXAMPLE_MAN_PATH)
with tempfile.NamedTemporaryFile(prefix='saved_test_manual.') as tmp:
manual.save(tmp.name)
loaded_manual = stem.manual.Manual.from_cache(tmp.name)
self.assertEqual(manual, loaded_manual)
def test_cached_manual(self):
manual = stem.manual.Manual.from_cache()
self.assertEqual('tor - The second-generation onion router',
manual.name)
self.assertEqual('tor [OPTION value]...', manual.synopsis)
self.assertTrue(manual.description.startswith(EXPECTED_DESCRIPTION))
self.assertTrue(len(manual.commandline_options) > 10)
self.assertTrue(len(manual.signals) > 5)
self.assertTrue(len(manual.files) > 20)
self.assertTrue(len(manual.config_options) > 200)
def test_download_man_page_without_arguments(self):
try:
stem.manual.download_man_page()
self.fail('we should fail without a path or file handler')
except ValueError as exc:
self.assertEqual(
"Either the path or file_handle we're saving to must be provided",
str(exc))
@patch('stem.util.system.is_available', Mock(return_value=False))
def test_download_man_page_requires_a2x(self):
try:
stem.manual.download_man_page('/tmp/no_such_file')
self.fail('we should require a2x to be available')
except IOError as exc:
self.assertEqual(
'We require a2x from asciidoc to provide a man page', str(exc))
@patch('tempfile.mkdtemp', Mock(return_value='/no/such/path'))
@patch('shutil.rmtree', Mock())
@patch('stem.manual.open',
Mock(side_effect=IOError('unable to write to file')),
create=True)
@patch('stem.util.system.is_available', Mock(return_value=True))
def test_download_man_page_when_unable_to_write(self):
try:
stem.manual.download_man_page('/tmp/no_such_file')
self.fail("we shouldn't be able to write to /no/such/path")
except IOError as exc:
self.assertEqual(
"Unable to download tor's manual from https://gitweb.torproject.org/tor.git/plain/doc/tor.1.txt to /no/such/path/tor.1.txt: unable to write to file",
str(exc))
@patch('tempfile.mkdtemp', Mock(return_value='/no/such/path'))
@patch('shutil.rmtree', Mock())
@patch('stem.manual.open', Mock(return_value=io.BytesIO()), create=True)
@patch('stem.util.system.is_available', Mock(return_value=True))
@patch(URL_OPEN,
Mock(side_effect=urllib.URLError(
'<urlopen error [Errno -2] Name or service not known>')))
def test_download_man_page_when_download_fails(self):
try:
stem.manual.download_man_page('/tmp/no_such_file',
url='https://www.atagar.com/foo/bar')
self.fail("downloading from test_invalid_url.org shouldn't work")
except IOError as exc:
self.assertEqual(
"Unable to download tor's manual from https://www.atagar.com/foo/bar to /no/such/path/tor.1.txt: <urlopen error <urlopen error [Errno -2] Name or service not known>>",
str(exc))
@patch('tempfile.mkdtemp', Mock(return_value='/no/such/path'))
@patch('shutil.rmtree', Mock())
@patch('stem.manual.open', Mock(return_value=io.BytesIO()), create=True)
@patch('stem.util.system.call',
Mock(side_effect=stem.util.system.CallError(
'call failed', 'a2x -f manpage /no/such/path/tor.1.txt', 1,
None, None, 'call failed')))
@patch('stem.util.system.is_available', Mock(return_value=True))
@patch(URL_OPEN, Mock(return_value=io.BytesIO(b'test content')))
def test_download_man_page_when_a2x_fails(self):
try:
stem.manual.download_man_page('/tmp/no_such_file',
url='https://www.atagar.com/foo/bar')
self.fail("downloading from test_invalid_url.org shouldn't work")
except IOError as exc:
self.assertEqual(
"Unable to run 'a2x -f manpage /no/such/path/tor.1.txt': call failed",
str(exc))
@patch('tempfile.mkdtemp', Mock(return_value='/no/such/path'))
@patch('shutil.rmtree', Mock())
@patch('stem.manual.open', create=True)
@patch('stem.util.system.call')
@patch('stem.util.system.is_available', Mock(return_value=True))
@patch('os.path.exists', Mock(return_value=True))
@patch(URL_OPEN, Mock(return_value=io.BytesIO(b'test content')))
def test_download_man_page_when_successful(self, call_mock, open_mock):
open_mock.side_effect = lambda path, *args: {
'/no/such/path/tor.1.txt': io.BytesIO(),
'/no/such/path/tor.1': io.BytesIO(b'a2x output'),
}[path]
call_mock.return_value = Mock()
output = io.BytesIO()
stem.manual.download_man_page(file_handle=output)
self.assertEqual(b'a2x output', output.getvalue())
call_mock.assert_called_once_with(
'a2x -f manpage /no/such/path/tor.1.txt')
@patch('stem.util.system.is_mac', Mock(return_value=False))
@patch('stem.util.system.call',
Mock(side_effect=OSError(
'man --encoding=ascii -P cat tor returned exit status 16')))
def test_from_man_when_manual_is_unavailable(self):
try:
stem.manual.Manual.from_man()
self.fail("fetching the manual should fail when it's unavailable")
except IOError as exc:
self.assertEqual(
"Unable to run 'man --encoding=ascii -P cat tor': man --encoding=ascii -P cat tor returned exit status 16",
str(exc))
@patch('stem.util.system.call', Mock(return_value=[]))
def test_when_man_is_empty(self):
manual = stem.manual.Manual.from_man()
self.assertEqual('', manual.name)
self.assertEqual('', manual.synopsis)
self.assertEqual('', manual.description)
self.assertEqual({}, manual.commandline_options)
self.assertEqual({}, manual.signals)
self.assertEqual({}, manual.files)
self.assertEqual(OrderedDict(), manual.config_options)
def test_get_urlopen_with_urlerror(self, mock_urlopen):
mock_urlopen.side_effect = urllib2.URLError('Error Reason')
ret_val = tvdb._get('http://url.com')
self.assertEqual(ret_val, '')
