def test_multiple_docs_on_same_version(self, impexp): mapp1 = impexp.mapp1 api = mapp1.create_and_use() mapp1.set_versiondata({"name": "hello", "version": "1.0"}) with mapp1.xom.keyfs.transaction(write=True): # create entries with and without log stage = mapp1.xom.model.getstage(mapp1.current_stage) link = stage.store_doczip( "Hello", "1.0", content=zip_dict({ "index.html": "<html><body>Hello"})) link.add_log('upload', stage.user.name, dst=stage.name) time.sleep(1.1) # force different times in log entry link = stage.store_doczip( "hello", "1.0", content=zip_dict({ "index.html": "<html><body>hello"})) link.add_log('upload', stage.user.name, dst=stage.name) impexp.export() mapp2 = impexp.new_import() with mapp2.xom.keyfs.transaction(write=False): stage = mapp2.xom.model.getstage(api.stagename) doczip = stage.get_doczip("hello", "1.0") archive = Archive(py.io.BytesIO(doczip)) assert 'index.html' in archive.namelist() assert py.builtin._totext( archive.read("index.html"), 'utf-8') == "<html><body>hello"
def test_hard_links(self, makeimpexp): impexp = makeimpexp(options=('--hard-links',)) mapp1 = impexp.mapp1 api = mapp1.create_and_use() content = b'content' mapp1.upload_file_pypi("he-llo-1.0.tar.gz", content, "he_llo", "1.0") content = zip_dict({"index.html": "<html/>"}) mapp1.upload_doc("he-llo.zip", content, "he-llo", "") impexp.export() assert impexp.exportdir.join( 'dataindex.json').stat().nlink == 1 assert impexp.exportdir.join( 'user1', 'dev', 'he_llo-1.0.doc.zip').stat().nlink == 2 assert impexp.exportdir.join( 'user1', 'dev', 'he_llo', 'he-llo-1.0.tar.gz').stat().nlink == 2 mapp2 = impexp.new_import() with mapp2.xom.keyfs.transaction(): stage = mapp2.xom.model.getstage(api.stagename) verdata = stage.get_versiondata_perstage("he_llo", "1.0") assert verdata["version"] == "1.0" links = stage.get_releaselinks("he_llo") assert len(links) == 1 assert links[0].entry.file_get_content() == b'content' doczip = stage.get_doczip("he_llo", "1.0") archive = Archive(py.io.BytesIO(doczip)) assert 'index.html' in archive.namelist() assert py.builtin._totext( archive.read("index.html"), 'utf-8') == "<html/>"
def test_export_hard_links(self, makeimpexp): impexp = makeimpexp(options=('--hard-links', )) mapp1 = impexp.mapp1 api = mapp1.create_and_use() content = b'content' mapp1.upload_file_pypi("he-llo-1.0.tar.gz", content, "he_llo", "1.0") content = zip_dict({"index.html": "<html/>"}) mapp1.upload_doc("he-llo.zip", content, "he-llo", "") # export the data impexp.export() # check the number of links of the files in the exported data assert impexp.exportdir.join('dataindex.json').stat().nlink == 1 assert impexp.exportdir.join('user1', 'dev', 'he_llo-1.0.doc.zip').stat().nlink == 2 assert impexp.exportdir.join('user1', 'dev', 'he-llo', '1.0', 'he-llo-1.0.tar.gz').stat().nlink == 2 # now import the data mapp2 = impexp.new_import() # and check that the files have the expected content with mapp2.xom.keyfs.transaction(): stage = mapp2.xom.model.getstage(api.stagename) verdata = stage.get_versiondata_perstage("he_llo", "1.0") assert verdata["version"] == "1.0" links = stage.get_releaselinks("he_llo") assert len(links) == 1 assert links[0].entry.file_get_content() == b'content' doczip = stage.get_doczip("he_llo", "1.0") archive = Archive(py.io.BytesIO(doczip)) assert 'index.html' in archive.namelist() assert py.builtin._totext(archive.read("index.html"), 'utf-8') == "<html/>"
def test_upload_docs_no_version(mapp, testapp, proj): api = mapp.create_and_use() content = zip_dict({"index.html": "<html/>"}) mapp.set_versiondata(dict(name="Pkg1", version="1.0")) mapp.upload_doc("pkg1.zip", content, "Pkg1", "") vv = get_view_version_links(testapp, api.index, "Pkg1", "1.0", proj=proj) link = vv.get_link("doczip") assert link.href.endswith("/Pkg1-1.0.doc.zip") r = testapp.get(link.href) archive = Archive(py.io.BytesIO(r.body)) assert 'index.html' in archive.namelist()
def test_upload_docs(mapp, testapp, proj): api = mapp.create_and_use() content = zip_dict({"index.html": "<html/>"}) mapp.upload_doc("pkg1.zip", content, "pkg1", "2.6", code=400) mapp.set_versiondata({"name": "pkg1", "version": "2.6"}) mapp.upload_doc("pkg1.zip", content, "pkg1", "2.6", code=200) vv = get_view_version_links(testapp, api.index, "pkg1", "2.6", proj=proj) link = vv.get_link(rel="doczip") assert link.href.endswith("/pkg1-2.6.doc.zip") r = testapp.get(link.href) archive = Archive(py.io.BytesIO(r.body)) assert 'index.html' in archive.namelist()
def test_docs_are_preserved(self, impexp): mapp1 = impexp.mapp1 api = mapp1.create_and_use() mapp1.set_versiondata({"name": "hello", "version": "1.0"}) content = zip_dict({"index.html": "<html/>"}) mapp1.upload_doc("hello.zip", content, "hello", "") impexp.export() mapp2 = impexp.new_import() with mapp2.xom.keyfs.transaction(write=False): stage = mapp2.xom.model.getstage(api.stagename) doczip = stage.get_doczip("hello", "1.0") archive = Archive(py.io.BytesIO(doczip)) assert 'index.html' in archive.namelist() assert py.builtin._totext( archive.read("index.html"), 'utf-8') == "<html/>"
def test_docs_are_preserved(self, impexp): mapp1 = impexp.mapp1 api = mapp1.create_and_use() mapp1.set_versiondata({"name": "hello", "version": "1.0"}) content = zip_dict({"index.html": "<html/>"}) mapp1.upload_doc("hello.zip", content, "hello", "") impexp.export() mapp2 = impexp.new_import() with mapp2.xom.keyfs.transaction(write=False): stage = mapp2.xom.model.getstage(api.stagename) doczip = stage.get_doczip("hello", "1.0") archive = Archive(py.io.BytesIO(doczip)) assert 'index.html' in archive.namelist() assert py.builtin._totext(archive.read("index.html"), 'utf-8') == "<html/>"
def store_doczip(self, name, version, docfile): """ store zip file and unzip doc content for the specified "name" project. """ if not version: version = self.get_metadata_latest(name)["version"] log.info("store_doczip: derived version %s", version) key = self.keyfs.PROJCONFIG(user=self.user, index=self.index, name=name) with key.locked_update() as projectconfig: verdata = projectconfig[version] filename = "%s-%s.doc.zip" % (name, version) entry = self.xom.filestore.store_file(self.user, self.index, filename, docfile) verdata["+doczip"] = entry.relpath # unpack key = self._doc_key(name, version) # XXX locking? (unzipping could happen concurrently in theory) tempdir = self.keyfs.mkdtemp(name) with Archive(entry.filepath.open("rb")) as archive: archive.extract(tempdir) keypath = key.filepath if keypath.check(): old = keypath.new(basename="old-" + keypath.basename) keypath.move(old) tempdir.move(keypath) old.remove() else: keypath.dirpath().ensure(dir=1) tempdir.move(keypath) return keypath
def test_storedoczipfile(self, stage, bases): from devpi_common.archive import Archive stage.set_versiondata(udict(name="pkg1", version="1.0")) content = zip_dict({"index.html": "<html/>", "_static": {}, "_templ": {"x.css": ""}}) stage.store_doczip("pkg1", "1.0", content) archive = Archive(BytesIO(stage.get_doczip("pkg1", "1.0"))) assert 'index.html' in archive.namelist() content = zip_dict({"nothing": "hello"}) stage.store_doczip("pkg1", "1.0", content) archive = Archive(BytesIO(stage.get_doczip("pkg1", "1.0"))) namelist = archive.namelist() assert 'nothing' in namelist assert 'index.html' not in namelist assert '_static' not in namelist assert '_templ' not in namelist
def test_zip_dir(tmpdir): source = tmpdir.join("source") newdest = tmpdir.join("newdest") dest = tmpdir.join("dest.zip") source.ensure("file") source.ensure("sub", "subfile") zip_dir(source, dest) with Archive(dest) as archive: archive.extract(newdest) assert newdest.join("file").isfile() assert newdest.join("sub", "subfile").isfile() newdest.remove() with Archive(py.io.BytesIO(zip_dir(source))) as archive: archive.extract(newdest) assert newdest.join("file").isfile() assert newdest.join("sub", "subfile").isfile()
def unpack(self): self.hub.info("unpacking", self.path_archive, "to", str(self.rootdir)) with Archive(self.path_archive) as archive: archive.extract(self.rootdir) basename = URL(self.link.url).basename pkgname, version = splitbasename(basename)[:2] subdir = "%s-%s" % (pkgname, version) inpkgdir = self.rootdir.join(subdir) assert inpkgdir.check(), inpkgdir self.path_unpacked = inpkgdir
def unpack(self): self.hub.info("unpacking", self.path_archive, "to", str(self.rootdir)) with Archive(self.path_archive) as archive: archive.extract(self.rootdir) pkgname = self.versioninfo.versiondata['name'] version = self.versioninfo.versiondata['version'] if self.link.basename.endswith(".whl"): inpkgdir = self.rootdir else: inpkgdir = self.rootdir.join("%s-%s" %(pkgname, version)) assert inpkgdir.check(), inpkgdir self.path_unpacked = inpkgdir
def test_getdoczip(self, stage, bases, tmpdir): stage.set_versiondata(udict(name="pkg1", version="1.0")) assert not stage.get_doczip("pkg1", "1.0") content = zip_dict({"index.html": "<html/>", "_static": {}, "_templ": {"x.css": ""}}) stage.store_doczip("pkg1", "1.0", content) doczip = stage.get_doczip("pkg1", "1.0") assert doczip with Archive(BytesIO(doczip)) as archive: archive.extract(tmpdir) assert tmpdir.join("index.html").read() == "<html/>" assert tmpdir.join("_static").check(dir=1) assert tmpdir.join("_templ", "x.css").check(file=1)
def test_upload_and_push_internal(mapp, testapp, monkeypatch, proj): mapp.create_user("user1", "1") mapp.create_and_login_user("user2") mapp.create_index("prod", indexconfig=dict(acl_upload=["user1", "user2"])) mapp.create_index("dev", indexconfig=dict(acl_upload=["user2"])) mapp.login("user1", "1") mapp.create_index("dev") mapp.use("user1/dev") mapp.upload_file_pypi("pkg1-2.6.tgz", b"123", "pkg1", "2.6") content = zip_dict({"index.html": "<html/>"}) mapp.upload_doc("pkg1.zip", content, "pkg1", "") # check that push is authorized and executed towards user2/prod index req = dict(name="pkg1", version="2.6", targetindex="user2/prod") r = testapp.push("/user1/dev", json.dumps(req)) assert r.status_code == 200 vv = get_view_version_links(testapp, "/user2/prod", "pkg1", "2.6", proj=proj) link = vv.get_link(rel="releasefile") assert link.href.endswith("/pkg1-2.6.tgz") # we check here that the upload of docs without version was # automatically tied to the newest release metadata link = vv.get_link(rel="doczip") assert link.href.endswith("/pkg1-2.6.doc.zip") r = testapp.get(link.href) archive = Archive(py.io.BytesIO(r.body)) assert 'index.html' in archive.namelist() # reconfigure inheritance and see if get shadowing information mapp.modify_index("user1/dev", indexconfig=dict(bases=("/user2/prod",))) vv = get_view_version_links(testapp, "/user1/dev", "pkg1", "2.6", proj=proj) link = vv.get_link(rel="releasefile") assert link.href.endswith("/pkg1-2.6.tgz") shadows = vv.shadowed() assert len(shadows) == 1, vv.versiondata vv = shadows[0] link = vv.get_link(rel="releasefile") assert link.href.endswith("/pkg1-2.6.tgz")
def archive(self, request, archive_path): if request.param == "path": arch = Archive(archive_path) else: f = archive_path.open("rb") arch = Archive(f) yield arch arch.close()
def test_multiple_store_doczip_uses_project(self, stage, bases, tmpdir): # check that two store_doczip calls with slightly # different names will not lead to two doczip entries stage.set_versiondata(udict(name="pkg1", version="1.0")) stage.store_doczip("pkg1", "1.0", zip_dict({})) content2 = zip_dict({"index.html": "<html/>"}) stage.store_doczip("Pkg1", "1.0", content2) # check we have only have one doczip link linkstore = stage.get_linkstore_perstage("pkg1", "1.0") links = linkstore.get_links(rel="doczip") assert len(links) == 1 # get doczip and check it's really the latest one doczip2 = stage.get_doczip("pkg1", "1.0") with Archive(BytesIO(doczip2)) as archive: archive.extract(tmpdir) assert tmpdir.join("index.html").read() == "<html/>"
def unpack(self): self.hub.info("unpacking", self.path_archive, "to", str(self.rootdir)) with Archive(self.path_archive) as archive: archive.extract(self.rootdir) pkgname = self.versioninfo.versiondata['name'] version = self.versioninfo.versiondata['version'] if self.link.basename.endswith(".whl"): inpkgdir = self.rootdir else: inpkgdir = self.rootdir.join("%s-%s" % (pkgname, version)) if not inpkgdir.check(): # sometimes dashes are replaced by underscores, # for example the source releases of argon2_cffi inpkgdir = self.rootdir.join( "%s-%s" % (pkgname.replace('-', '_'), version)) if not inpkgdir.check(): self.hub.fatal("Couldn't find unpacked package in", inpkgdir) self.path_unpacked = inpkgdir
def unpack_docs(stage, name, version, entry): # unpack, maybe a bit uncarefully but in principle # we are not loosing the original zip file anyway unpack_path = get_unpack_path(stage, name, version) hash_path = unpack_path.join('.hash') if hash_path.exists(): with hash_path.open() as f: if f.read().strip() == entry.hash_spec: return unpack_path if unpack_path.exists(): unpack_path.remove() with entry.file_open_read() as f: with Archive(f) as archive: archive.extract(unpack_path) with hash_path.open('w') as f: f.write(entry.hash_spec) threadlog.debug("%s: unpacked %s-%s docs to %s", stage.name, name, version, unpack_path) return unpack_path
def test_unknown_archive(self): with pytest.raises(UnsupportedArchive): Archive(py.io.BytesIO(b"123"))
def test_zip_dict(tmpdir): content = zip_dict({"one": {"nested": "1"}, "two": {}}) with Archive(py.io.BytesIO(content)) as archive: archive.extract(tmpdir) assert tmpdir.join("one", "nested").read() == "1" assert tmpdir.join("two").isdir()
def test_tarfile_outofbound(tmpdir): with Archive(datadir.join("slash.tar.gz")) as archive: with pytest.raises(ValueError): archive.extract(tmpdir)