async def test_when_fetch_fails_bad_repo(self, loop):
handler = WordPressRepository(loop=loop,
storage=MagicMock(),
repository_checker=MagicMock())
handler.current_plugins = lambda: {'hello-world', 'unknown-plugin'}
handler.enumerate_plugins = lambda: fake_future(
{'hello-world', 'a', 'b'}, loop)
handler.fetch_plugin = MagicMock()
handler.fetch_plugin.side_effect = PluginNotFound('A side effect!')
handler.checker.has_content.return_value = fake_future(False, loop)
await handler.perform_plugin_lookup()
handler.fetch_plugin.assert_has_calls([
call('a'),
call('b'),
],
any_order=True)
handler.storage.write_meta.assert_not_called()
handler.checker.has_content.assert_has_calls([
call(
Repository(type="subversion",
location="https://plugins.svn.wordpress.org/a/")),
call(
Repository(type="subversion",
location="https://plugins.svn.wordpress.org/b/")),
],
any_order=True)
handler.storage.append.assert_has_calls([
call("plugins-ignore.txt", "a"),
call("plugins-ignore.txt", "b"),
],
any_order=True)
async def test_switch_ignore_externals_if_any_recursive_external(
self, loop):
with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
proc = MagicMock()
proc.communicate.return_value = fake_future((b"out", b"err"),
loop=loop)
proc.returncode = 0
cse.return_value = proc
svn = Subversion(loop=loop)
svn.has_recursive_externals = MagicMock(
return_value=fake_future(True, loop))
await svn.switch("https://svn.example.com/tags/1.2.3",
workdir="/tmp/foobar")
cse.assert_called_with("svn",
"switch",
"--ignore-ancestry",
"--ignore-externals",
"https://svn.example.com/tags/1.2.3",
cwd="/tmp/foobar",
loop=loop,
stdout=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
proc.communicate.assert_called_with()
async def test_when_fetch_fails(self, loop):
handler = WordPressRepository(loop=loop,
storage=MagicMock(),
repository_checker=MagicMock())
handler.current_themes = lambda: {'hello-world', 'unknown-theme'}
handler.enumerate_themes = lambda: fake_future(
{'hello-world', 'a', 'b'}, loop)
handler.fetch_theme = MagicMock()
handler.fetch_theme.side_effect = PluginNotFound('A side effect!')
handler.checker.has_content.return_value = fake_future(True, loop)
await handler.perform_theme_lookup()
handler.fetch_theme.assert_has_calls([
call('a'),
call('b'),
],
any_order=True)
handler.storage.write_meta.assert_has_calls([
call(handler.theme_parser.create_meta(slug="a")),
call(handler.theme_parser.create_meta(slug="b")),
],
any_order=True)
handler.checker.has_content.assert_has_calls([
call(
Repository(type="subversion",
location="https://themes.svn.wordpress.org/a/")),
call(
Repository(type="subversion",
location="https://themes.svn.wordpress.org/b/")),
],
any_order=True)
async def test_execute_handle_error(self, loop):
with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
proc = MagicMock()
proc.communicate.return_value = fake_future((b"out", b"err"),
loop=loop)
proc.returncode = 1
cse.return_value = proc
svn = Subversion(loop=loop)
svn.has_recursive_externals = MagicMock(
return_value=fake_future(False, loop))
with self.assertRaises(ExecutionFailure):
await svn.checkout("https://svn.example.com/tags/1.2.3",
workdir="/tmp/foobar")
cse.assert_called_with("svn",
"checkout",
"https://svn.example.com/tags/1.2.3",
".",
cwd="/tmp/foobar",
loop=loop,
stdout=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
proc.communicate.assert_called_with()
async def test_check_repository_has_classic_structure_and_data(self, loop):
checker = RepositoryChecker(subversion=MagicMock())
checker.subversion.ls.side_effect = [
fake_future(['branches/', 'tags/', 'trunk/', 'somefile.txt'],
loop),
fake_future(['1.0/', '1.1/', '1.2/'], loop),
]
self.assertTrue(await checker.has_content(self.svnrepo))
checker.subversion.ls.assert_has_calls([
call("http://example.com/"),
call("http://example.com/tags"),
])
async def test_calls_made_when_new_plugins_arrive(self, loop):
handler = WordPressRepository(loop=loop, storage=MagicMock())
handler.current_plugins = lambda: {'hello-world', 'unknown-plugin'}
handler.enumerate_plugins = lambda: fake_future({'hello-world', 'a', 'b'}, loop)
handler.fetch_plugin = MagicMock()
handler.fetch_plugin.return_value = fake_future(Meta(key="a", name="A"), loop)
await handler.perform_plugin_lookup()
handler.fetch_plugin.assert_has_calls([
call('a'),
call('b'),
], any_order=True)
handler.storage.write_meta.assert_called_with(Meta(key="a", name="A"))
async def test_no_calls_made_when_nothing_new(self, loop):
handler = WordPressRepository(loop=loop)
handler.current_plugins = lambda: {'hello-world', 'unknown-plugin'}
handler.enumerate_plugins = lambda: fake_future({'hello-world'}, loop)
handler.fetch_plugin = MagicMock()
handler.fetch_plugin.assert_not_called()
await handler.perform_plugin_lookup()
async def test_classic_structure(self, loop):
svn = Subversion(loop=loop, svn_base_dir="/temp/")
svn.ls = lambda p: fake_future(['tags/', 'trunk/', 'branches/'],
loop=loop)
workspace = SubversionWorkspace(workdir="/tmp/foo",
subversion=svn,
repository="https://svn.example.com/")
self.assertEqual(workspace.repository, "https://svn.example.com/")
await workspace.prepare()
svn.ls = lambda p: fake_future(['1.2/', '2.0/', '2.1/'], loop=loop)
self.assertEqual(workspace.repository, "https://svn.example.com/tags/")
self.assertEqual(["1.2", "2.0", "2.1"], await
workspace.list_versions())
async def test_svn_get_last_release_date_of_components_replace_hours_with_current_year(
self, loop):
with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
proc = MagicMock()
proc.stdout.readline.return_value = \
fake_future(b"1749964 user1 Oct 20 11:15 plugin-1/\n", loop=loop)
proc.stdout.at_eof.side_effect = [False, True]
proc.wait.return_value = fake_future(0, loop=loop)
cse.return_value = proc
svn = Subversion(loop=loop)
plugins = await svn._get_last_release_date_of_components(
"plugins", "http://plugins.svn.wordpress.org/")
self.assertEqual(plugins, {
"plugins/plugin-1":
date(year=date.today().year, month=10, day=20)
})
async def test_straight_version_listing(self, loop):
checker = RepositoryChecker(subversion=MagicMock())
checker.subversion.ls.side_effect = [
fake_future(['1.0/', '1.1/', '1.2/'], loop),
]
self.assertTrue(await checker.has_content(self.svnrepo))
checker.subversion.ls.assert_has_calls([
call("http://example.com/"),
])
async def test_checkout_and_switch(self, fake_future):
svn = MagicMock()
svn.checkout.return_value = fake_future(None)
svn.switch.return_value = fake_future(None)
workspace = SubversionWorkspace(workdir="/tmp/foo",
subversion=svn,
repository="https://svn.example.com/")
await workspace.to_version("1.0")
svn.checkout.assert_called_with("https://svn.example.com/1.0",
workdir="/tmp/foo")
svn.reset_mock()
await workspace.to_version("1.1")
svn.switch.assert_called_with("https://svn.example.com/1.1",
workdir="/tmp/foo")
async def test_check_repository_content_empty_svn(self, loop):
checker = RepositoryChecker(subversion=MagicMock())
checker.subversion.ls.side_effect = [
fake_future([], loop),
]
self.assertFalse(await checker.has_content(self.svnrepo))
checker.subversion.ls.assert_has_calls([
call("http://example.com/"),
])
async def test_svn_info(self, loop):
with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
proc = MagicMock()
proc.communicate.side_effect = [
fake_future(
(b"https://plugins.svn.wordpress.org/plugin/tags/1.0\n",
b""),
loop=loop),
fake_future((b"https://plugins.svn.wordpress.org\n", b""),
loop=loop)
]
proc.returncode = 0
cse.return_value = proc
svn = Subversion(loop=loop)
info = await svn.info(
"https://plugins.svn.wordpress.org/plugin/tags/1.0",
workdir="/tmp/plugin")
cse.assert_has_calls([
call(*("svn", "info", "--show-item", "url",
"https://plugins.svn.wordpress.org/plugin/tags/1.0"),
cwd="/tmp/plugin",
loop=loop,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE),
call().communicate(),
call(*("svn", "info", "--show-item", "repos-root-url",
"https://plugins.svn.wordpress.org/plugin/tags/1.0"),
cwd="/tmp/plugin",
loop=loop,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE)
])
self.assertEqual(
info, {
"url": "https://plugins.svn.wordpress.org/plugin/tags/1.0",
"root": "https://plugins.svn.wordpress.org"
})
async def test_list_versions_skip_versions_without_digit(
self, fake_future):
svn = MagicMock()
svn.ls.return_value = fake_future(["1.0/", "1.1/", "trunk/"])
workspace = SubversionWorkspace(workdir="/tmp/foo",
subversion=svn,
repository="https://svn.test.com/")
versions = await workspace.list_versions()
self.assertEqual(versions, ["1.0", "1.1"])
async def test_list_externals_no_external(self, loop):
with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
proc = MagicMock()
proc.communicate.return_value = fake_future((b"", b""), loop=loop)
proc.returncode = 0
cse.return_value = fake_future(proc, loop=loop)
svn = Subversion(loop=loop)
externals = await svn.list_externals(
"https://plugins.svn.wordpress.org/plugin/tags/1.0",
workdir="/tmp/plugin")
cse.assert_called_once_with(
*("svn", "propget", "-R", "svn:externals",
"https://plugins.svn.wordpress.org/plugin/tags/1.0"),
cwd="/tmp/plugin",
loop=loop,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE)
self.assertEqual(externals, [])
async def test_obtain_list(self, loop):
handler = WordPressRepository(loop=loop, subversion=MagicMock())
handler.subversion.ls.return_value = fake_future(
['a-class-act-ny', 'skt-bakery', 'the-modern-accounting-firm'],
loop)
themes = await handler.enumerate_plugins()
handler.subversion.ls.assert_called_with(
"https://plugins.svn.wordpress.org/")
self.assertEqual(
themes,
{'a-class-act-ny', 'skt-bakery', 'the-modern-accounting-firm'})
async def test_flat_does_not_alter_repo(self, loop):
svn = Subversion(loop=loop, svn_base_dir="/temp/")
svn.ls = lambda p: fake_future(['1.0/', '1.1/'], loop=loop)
workspace = SubversionWorkspace(workdir="/tmp/foo",
subversion=svn,
repository="https://svn.example.com/")
self.assertEqual(workspace.repository, "https://svn.example.com/")
await workspace.prepare()
self.assertEqual(workspace.repository, "https://svn.example.com/")
async def test_has_recursive_externals(self, loop):
svn = Subversion(loop=loop)
svn.list_externals = MagicMock(side_effect=[
fake_future([{
"name": "plugin",
"url": "https://plugins.svn.wordpress.org/plugin"
}], loop),
fake_future([{
"name": "valid-external",
"url": "https://plugins.svn.wordpress.org/external"
}], loop)
])
self.assertTrue(await svn.has_recursive_externals(
"https://plugins.svn.wordpress.org/plugin/tags/1.0", workdir=None))
self.assertFalse(await svn.has_recursive_externals(
"https://plugins.svn.wordpress.org/plugin/tags/1.0", workdir=None))
svn.list_externals.assert_has_calls([
call("https://plugins.svn.wordpress.org/plugin/tags/1.0",
workdir=None),
call("https://plugins.svn.wordpress.org/plugin/tags/1.0",
workdir=None)
])
async def test_obtain_list(self, loop):
handler = WordPressRepository(loop=loop, subversion=MagicMock())
handler.subversion.ls.return_value = fake_future([
'aioseo-fix/',
'easy-book-reviews/',
'isd-wordpress-rss-feed-plugin/',
'picashow/',
'wp-auto-hotel-finder/',
], loop)
plugins = await handler.enumerate_plugins()
handler.subversion.ls.assert_called_with("https://plugins.svn.wordpress.org/")
self.assertEqual(plugins, {'aioseo-fix', 'easy-book-reviews', 'isd-wordpress-rss-feed-plugin', 'picashow',
'wp-auto-hotel-finder'})
async def test_list_externals(self, loop):
with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
proc = MagicMock()
out = b"https://plugins.svn.wordpress.org/plugin/tags/1.0 - https://www.some-external.example external\n\n" \
b"https://plugins.svn.wordpress.org/plugin/tags/1.0/class - https://www.some-external.example external\n\n" \
b"https://plugins.svn.wordpress.org/plugin/tags/1.0/css - external https://www.some-external.example\n\n" \
b"https://plugins.svn.wordpress.org/plugin/tags/1.0/languages - external https://www.some-external.example\n\n"
proc.communicate.return_value = fake_future((out, b""), loop=loop)
proc.returncode = 0
cse.return_value = proc
svn = Subversion(loop=loop)
svn.info = MagicMock(return_value=fake_future(
{
"url": "https://plugins.svn.wordpress.org/plugin/tags/1.0",
"root": "https://plugins.svn.wordpress.org/"
},
loop=loop))
externals = await svn.list_externals(
"https://plugins.svn.wordpress.org/plugin/tags/1.0",
workdir="/tmp/plugin")
cse.assert_called_once_with(
*("svn", "propget", "-R", "svn:externals",
"https://plugins.svn.wordpress.org/plugin/tags/1.0"),
cwd="/tmp/plugin",
loop=loop,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE)
self.assertEqual(externals,
[{
"name": "external",
"url": "https://www.some-external.example"
}] * 4)
async def test_request_files_set_expected_hash_and_return_code_for_requests(
self, loop):
self._setup_async_test(loop)
self.hammertime.request = MagicMock(
return_value=fake_future(MagicMock(), loop=loop))
await self.fetcher.request_files(self.plugin_key, self.files_to_fetch)
expected_arguments = {
"expected_hash": {"hash", "another-hash"},
"hash_algo": "SHA256",
"file_path": ANY,
"expected_status_code": 200
}
for args, kwargs in self.hammertime.request.call_args_list:
self.assertEqual(kwargs["arguments"], expected_arguments)
async def test_svn_get_components_with_new_release(self, loop):
themes = {
"themes/theme-0": date(year=2017, month=10, day=20),
"themes/theme-1": date(year=2016, month=4, day=4),
"themes/theme-2": date(year=2015, month=10, day=21),
"themes/theme-3": date(year=2017, month=10, day=6)
}
svn = Subversion(loop=None)
svn._get_last_release_date_of_components = MagicMock(
return_value=fake_future(themes, loop=loop))
recently_updated = await svn.get_components_with_new_release(
"themes", "http://themes.svn.wordpress.org/",
date(year=2017, day=6, month=10))
self.assertEqual(recently_updated,
{"themes/theme-0", "themes/theme-3"})
async def test_svn_get_last_release_date_of_components_return_last_modification_date_of_tags_folder(
self, loop):
with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
proc = MagicMock()
proc.stdout.readline.side_effect = [
fake_future(
b"svn: warning: W160013: URL 'http://themes.svn.wordpress.org/tags' non-existent in revision 83065",
loop=loop),
fake_future(b"1749964 user1 Oct 20 11:15 ./\n",
loop=loop),
fake_future(
b"1077807 user 2 Jan 28 2015 plugin-1/\n",
loop=loop),
fake_future(
b"1385952 user.3 Apr 04 2016 plugin-2/",
loop=loop),
fake_future(
b"svn: E200009: Could not list all targets because some targets don't exist",
loop=loop)
]
proc.stdout.at_eof.side_effect = [
False, False, False, False, False, True
]
proc.wait.return_value = fake_future(0, loop=loop)
cse.return_value = fake_future(proc, loop=loop)
svn = Subversion(loop=loop)
plugins = await svn._get_last_release_date_of_components(
"plugins", "http://plugins.svn.wordpress.org/")
cse.assert_has_calls([
call(*("svn", "ls", "-v", "^/tags",
"http://plugins.svn.wordpress.org/"),
loop=loop,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
stdin=asyncio.subprocess.DEVNULL)
])
self.assertEqual(
plugins, {
"plugins/plugin-1": date(year=2015, month=1, day=28),
"plugins/plugin-2": date(year=2016, month=4, day=4)
})
async def test_list_externals_with_relative_path(self, loop):
with patch('openwebvulndb.common.vcs.create_subprocess_exec') as cse:
proc = MagicMock()
out = b"https://svn.example.com/plugins/plugin/tags/1.0/subdir0 - //svn.example.com/external external\n\n" \
b"https://svn.example.com/plugins/plugin/tags/1.0/subdir1 - /external external\n\n" \
b"https://svn.example.com/plugins/plugin/tags/1.0/subdir2 - ^/external external\n\n" \
b"https://svn.example.com/plugins/plugin/tags/1.0/subdir3 - external ../external\n\n" \
b"https://svn.example.com/plugins/plugin/tags/1.0/subdir3 - external ../../external\n\n" \
b"https://svn.example.com/plugins/plugin/tags/1.0/subdir3 - ../../../../../external external\n\n" \
b"https://svn.example.com/plugins/plugin/tags/1.0/subdir4 - external ^/../repo/external\n\n" \
b"https://svn.example.com/plugins/plugin/tags/1.0/subdir4 - external ^/../../../repo/external\n\n"
proc.communicate.return_value = fake_future((out, b""), loop=loop)
proc.returncode = 0
cse.return_value = proc
svn = Subversion(loop=loop)
svn.info = MagicMock(return_value=fake_future(
{
"url": "https://svn.example.com/plugins/plugin/tags/1.0",
"root": "https://svn.example.com/plugins"
},
loop=loop))
externals = await svn.list_externals(
"https://svn.example.com/plugins/plugin/tags/1.0",
workdir="/tmp/plugin")
cse.assert_called_once_with(
*("svn", "propget", "-R", "svn:externals",
"https://svn.example.com/plugins/plugin/tags/1.0"),
cwd="/tmp/plugin",
loop=loop,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
stdin=asyncio.subprocess.PIPE)
self.assertEqual(
externals,
[{
"name": "external",
"url": "https://svn.example.com/external"
}, {
"name": "external",
"url": "https://svn.example.com/external"
}, {
"name": "external",
"url": "https://svn.example.com/plugins/external"
}, {
"name": "external",
"url":
"https://svn.example.com/plugins/plugin/tags/external"
}, {
"name": "external",
"url": "https://svn.example.com/plugins/plugin/external"
}, {
"name": "external",
"url": "https://svn.example.com/external"
}, {
"name": "external",
"url": "https://svn.example.com/repo/external"
}, {
"name": "external",
"url": "https://svn.example.com/repo/external"
}])
