def test_joinpath_asdir(self):
url = URL("http://heise.de")
new = url.joinpath("hello", asdir=1)
assert new.url == "http://heise.de/hello/"
new = url.joinpath("hello/", asdir=1)
assert new.url == "http://heise.de/hello/"
url = URL("http://heise.de?foo=bar")
new = url.joinpath("hello", asdir=1)
assert new.url == "http://heise.de/hello/?foo=bar"
new = url.joinpath("hello/", asdir=1)
assert new.url == "http://heise.de/hello/?foo=bar"
def test_joinpath(self, url, path, expected):
d_url = URL(url)
url_joined = d_url.joinpath(path).url
assert url_joined == expected
assert URL(url, path).url == expected
assert d_url.joinpath(path, "end").url == expected.rstrip('/') + "/end"
assert URL(url, path, "end").url == expected.rstrip('/') + "/end"
assert d_url.joinpath(path, "end",
asdir=1).url == expected.rstrip('/') + "/end/"
assert URL(url, path, "end",
asdir=1).url == expected.rstrip('/') + "/end/"
class ProjectParser(HTMLParser):
def __init__(self, url):
HTMLParser.__init__(self)
self.projects = set()
self.baseurl = URL(url)
self.basehost = self.baseurl.replace(path='')
self.project = None
def handle_starttag(self, tag, attrs):
if tag == 'a':
self.project = None
attrs = dict(attrs)
if 'href' not in attrs:
return
href = attrs['href']
if '://' not in href:
project = href.rstrip('/').rsplit('/', 1)[-1]
else:
newurl = self.baseurl.joinpath(href)
# remove trailing slashes, so basename works correctly
newurl = newurl.asfile()
if not newurl.is_valid_http_url():
return
if not newurl.path.startswith(self.baseurl.path):
return
if self.basehost != newurl.replace(path=''):
return
project = newurl.basename
self.project = project
def handle_endtag(self, tag):
if tag == 'a' and self.project:
self.projects.add(self.project)
self.project = None
def test_include_mirrordata(self, caplog, makeimpexp, maketestapp,
pypistage):
impexp = makeimpexp(options=('--include-mirrored-files', ))
mapp1 = impexp.mapp1
testapp = maketestapp(mapp1.xom)
api = mapp1.use('root/pypi')
pypistage.mock_simple(
"package", '<a href="/package-1.0.zip" />\n'
'<a href="/package-1.1.zip#sha256=a665a45920422f9d417e4867efdc4fb8a04a1f3fff1fa07e998e86f7f7a27ae3" />\n'
'<a href="/package-1.2.zip#sha256=b3a8e0e1f9ab1bfe3a36f231f676f78bb30a519d2b21e6c530c0eee8ebb4a5d0" data-yanked="" />\n'
'<a href="/package-2.0.zip#sha256=35a9e381b1a27567549b5f8a6f783c167ebf809f1c4d6a9e367240484d8ce281" data-requires-python=">=3.5" />'
)
pypistage.mock_extfile("/package-1.1.zip", b"123")
pypistage.mock_extfile("/package-1.2.zip", b"456")
pypistage.mock_extfile("/package-2.0.zip", b"789")
r = testapp.get(api.index + "/+simple/package/")
assert r.status_code == 200
# fetch some files, so they are included in the dump
(_, link1, link2, link3) = sorted(
(x.attrs['href'] for x in r.html.select('a')),
key=lambda x: x.split('/')[-1])
baseurl = URL(r.request.url)
r = testapp.get(baseurl.joinpath(link1).url)
assert r.body == b"123"
r = testapp.get(baseurl.joinpath(link2).url)
assert r.body == b"456"
r = testapp.get(baseurl.joinpath(link3).url)
assert r.body == b"789"
impexp.export()
mapp2 = impexp.new_import()
with mapp2.xom.keyfs.transaction(write=False):
stage = mapp2.xom.model.getstage(api.stagename)
stage.offline = True
projects = stage.list_projects_perstage()
assert projects == {'package'}
links = sorted(
get_mutable_deepcopy(
stage.get_simplelinks_perstage("package")))
assert links == [
('package-1.1.zip',
'root/pypi/+f/a66/5a45920422f9d/package-1.1.zip', None, None),
('package-1.2.zip',
'root/pypi/+f/b3a/8e0e1f9ab1bfe/package-1.2.zip', None, True),
('package-2.0.zip',
'root/pypi/+f/35a/9e381b1a27567/package-2.0.zip', '>=3.5',
None)
]
def get_release_paths(self, project):
r = self.get_simple(project)
pkg_url = URL(r.request.url)
paths = [
pkg_url.joinpath(link["href"]).path
for link in BeautifulSoup(r.body, "html.parser").findAll("a")
]
return paths
def _value_from_dict_by_url(self, d, url, default=None):
# searches for longest match, so there can be multiple devpi instances
# on the same domain with different paths
url = URL(url)
while url:
if url in d or url.path == '/':
break
url = url.joinpath('..')
return d.get(url.url, default)
def json_info_view(context, request):
baseurl = URL(request.application_url).asdir()
version = context.stage.get_latest_version(context.project, stable=True)
info = get_mutable_deepcopy(context.stage.get_versiondata(context.project, version))
info.pop("+elinks", None)
result = dict(info=info, releases={})
for release in context.stage.get_releaselinks(context.project):
result["releases"].setdefault(release.version, []).append(
dict(url=baseurl.joinpath(release.relpath).url)
)
return result
def _get_remote_projects(self):
headers = {"Accept": "text/html"}
# use a minimum of 30 seconds as timeout for remote server and
# 60s when running as replica, because the list can be quite large
# and the master might take a while to process it
if self.xom.is_replica():
timeout = max(self.timeout, 60)
else:
timeout = max(self.timeout, 30)
response = self.httpget(self.mirror_url,
allow_redirects=True,
extra_headers=headers,
timeout=timeout)
if response.status_code != 200:
raise self.UpstreamError("URL %r returned %s %s", self.mirror_url,
response.status_code, response.reason)
projects = set()
baseurl = URL(response.url)
basehost = baseurl.replace(path='')
for elem in self._iter_remote_project_links(response):
href = elem.attrib['href']
if '://' not in href:
project = href.rstrip('/').rsplit('/', 1)[-1]
else:
newurl = baseurl.joinpath(href)
# remove trailing slashes, so basename works correctly
newurl = newurl.asfile()
if not newurl.is_valid_http_url():
continue
if not newurl.path.startswith(baseurl.path):
continue
if basehost != newurl.replace(path=''):
continue
project = newurl.basename
projects.add(project)
return projects
def get_release_paths(self, projectname):
r = self.get_simple(projectname)
pkg_url = URL(r.request.url)
paths = [pkg_url.joinpath(link["href"]).path
for link in BeautifulSoup(r.body).findAll("a")]
return paths
