def register_and_store(stage, basename, content=b"123", name=None):
assert py.builtin._isbytes(content), content
n, version = splitbasename(basename)[:2]
if name is None:
name = n
stage.register_metadata(dict(name=name, version=version))
return stage.store_releasefile(basename, content)
def import_filedesc(self, stage, filedesc):
assert stage.ixconfig["type"] != "mirror"
rel = filedesc["relpath"]
projectname = filedesc["projectname"]
p = self.import_rootdir.join(rel)
assert p.check(), p
if filedesc["type"] == "releasefile":
mapping = filedesc["entrymapping"]
if self.dumpversion == "1":
# previous versions would not add a version attribute
version = BasenameMeta(p.basename).version
else:
version = filedesc["version"]
entry = stage.store_releasefile(projectname, version,
p.basename, p.read("rb"),
last_modified=mapping["last_modified"])
assert entry.md5 == mapping["md5"]
self.import_pre2_toxresults(stage, entry)
elif filedesc["type"] == "doczip":
basename = os.path.basename(rel)
name, version, suffix = splitbasename(basename)
stage.store_doczip(name, version, p.read("rb"))
elif filedesc["type"] == "toxresult":
linkstore = stage.get_linkstore_perstage(filedesc["projectname"],
filedesc["version"])
link, = linkstore.get_links(entrypath=filedesc["for_entrypath"])
stage.store_toxresult(link, json.loads(p.read("rb").decode("utf8")))
else:
fatal("unknown file type: %s" % (type,))
def main(hub, args):
pusher = parse_target(hub, args)
name = None
version = None
if '==' not in args.pkgspec and '-' in args.pkgspec:
name, version = splitbasename(args.pkgspec + ".zip")[:2]
if not name or not version:
req = parse_requirement(args.pkgspec)
if len(req.specs) != 1 or req.specs[0][0] != '==':
hub.fatal(
"The release specification needs to be of this form: name==version"
)
name = req.project_name
version = req.specs[0][1]
else:
hub.warn("Old style package specification is deprecated, "
"use this form: your-pkg-name==your.version.specifier")
r = pusher.execute(hub, name, version)
failed = r.status_code not in (200, 201)
if r.type == "actionlog":
for action in r["result"]:
red = int(action[0]) not in (200, 201, 410)
failed = failed or red
for line in (" ".join(map(str, action))).split("\n"):
hub.line(" " + line, red=red)
if failed:
hub.fatal("Failure during upload")
def project_get(context, request):
context = ContextWrapper(context)
releaselinks = context.stage.get_releaselinks(context.name)
if not releaselinks:
raise HTTPNotFound("The project %s does not exist." % context.name)
versions = []
seen = set()
for release in releaselinks:
user, index = release.entrypath.split("/", 2)[:2]
name, version = splitbasename(release)[:2]
seen_key = (user, index, name, version)
if seen_key in seen:
continue
versions.append(dict(
index_title="%s/%s" % (user, index),
index_url=request.route_url(
"/{user}/{index}", user=user, index=index),
title=version,
url=request.route_url(
"/{user}/{index}/{name}/{version}",
user=user, index=index, name=name, version=version)))
seen.add(seen_key)
return dict(
title="%s/: %s versions" % (context.stage.name, name),
versions=versions)
def register_and_store(stage, basename, content=b"123", name=None):
assert py.builtin._isbytes(content), content
n, version = splitbasename(basename)[:2]
if name is None:
name = n
stage.register_metadata(dict(name=name, version=version))
return stage.store_releasefile(basename, content)
def file(self, filename, content, metadata, code=200):
if "version" not in metadata:
metadata["version"] = splitbasename(filename,
checkarch=False)[1]
return mapp.upload_file_pypi(
filename, content,
metadata.get("name"), metadata.get("version"),
indexname=self.stagename, register=False,
code=code)
def main(hub, args):
pusher = parse_target(hub, args)
name, version = splitbasename(args.nameversion + ".zip")[:2]
r = pusher.execute(hub, name, version)
if r.type == "actionlog":
for action in r["result"]:
red = int(action[0]) >= 400
for line in (" ".join(map(str, action))).split("\n"):
hub.line(" " + line, red=red)
def main(hub, args):
pusher = parse_target(hub, args)
name, version = splitbasename(args.nameversion + ".zip")[:2]
r = pusher.execute(hub, name, version)
if r.type == "actionlog":
for action in r["result"]:
red = int(action[0]) >= 400
for line in (" ".join(map(str, action))).split("\n"):
hub.line(" " + line, red=red)
def getnewestversion(self, pkgname):
best = None
for link in self.links:
basename = URL(link.url).basename
name, version = splitbasename(basename)[:2]
ver = Version(version)
if name == pkgname:
if best is None or ver > best[0]:
best = ver, link
return best and best[1] or None
def unpack(self):
self.hub.info("unpacking", self.path_archive, "to", str(self.rootdir))
with Archive(self.path_archive) as archive:
archive.extract(self.rootdir)
basename = URL(self.link.url).basename
pkgname, version = splitbasename(basename)[:2]
subdir = "%s-%s" % (pkgname, version)
inpkgdir = self.rootdir.join(subdir)
assert inpkgdir.check(), inpkgdir
self.path_unpacked = inpkgdir
def unpack(self):
self.hub.info("unpacking", self.path_archive, "to", str(self.rootdir))
with Archive(self.path_archive) as archive:
archive.extract(self.rootdir)
basename = URL(self.link.url).basename
pkgname, version = splitbasename(basename)[:2]
subdir = "%s-%s" %(pkgname, version)
inpkgdir = self.rootdir.join(subdir)
assert inpkgdir.check(), inpkgdir
self.path_unpacked = inpkgdir
def _make_elinks(self, projectname, data):
from .model import ELink
for relpath, hash_spec, eggfragment in data:
linkdict = {"entrypath": relpath, "hash_spec": hash_spec,
"eggfragment": eggfragment}
version = "XXX"
try:
name, version = splitbasename(relpath)[:2]
except ValueError:
pass
yield ELink(self.filestore, linkdict, projectname, version)
def getnewestversion(self, pkgname):
""" returns newest applicable version of package """
req = next(pkg_resources.parse_requirements(pkgname))
best = None
for link in self.links:
basename = URL(link.url).basename
name, version = splitbasename(basename)[:2]
ver = Version(version)
if name in (req.project_name, pkgname) and version in req:
if best is None or ver > best[0]:
best = ver, link
return best and best[1] or None
def getnewestversion(self, pkgname):
""" returns newest applicable version of package """
req = next(pkg_resources.parse_requirements(pkgname))
best = None
for link in self.links:
basename = URL(link.url).basename
name, version = splitbasename(basename)[:2]
ver = Version(version)
if name in (req.project_name, pkgname) and version in req:
if best is None or ver > best[0]:
best = ver, link
return best and best[1] or None
def get_meta_data(package_path):
"""Get package meta-data from a package distribution file.
Arguments:
package_path (str): Path to the package file to be inspected.
Returns:
PackageFileMetaData: The discovered meta-data
"""
_LOGGER.debug(
"Begin getting meta-data about a package from its file at %s",
package_path)
# verify the file exists etc.
if not os.path.exists(package_path):
raise Exception(
"Package path '{}' does not exist".format(package_path))
base_name = os.path.basename(package_path)
# scrape it for py version metadata
try:
(py_version,
file_type) = dc_metadata.get_pyversion_filetype(base_name)
except Exception:
error_message = (
"Error getting 'py_version' and 'file_type' metadata from "
"possible package file {}").format(package_path)
_LOGGER.exception(error_message)
raise Exception(error_message)
# scrape it for additional metadata
try:
(project_name, version,
suffix) = dc_metadata.splitbasename(base_name)
except Exception:
error_message = (
"Error getting 'project_name', 'version', 'suffix' from "
"package file {}").format(package_path)
_LOGGER.exception(error_message)
raise Exception(error_message)
meta_data = PackageFileMetaData(py_version=py_version,
file_type=file_type,
project_name=project_name,
version=version,
suffix=suffix,
path=package_path)
_LOGGER.debug("Got this metadata from package file %s : %s",
package_path, pprint.pformat(meta_data))
_LOGGER.debug(
"Finished getting meta-data about a package from its file at %s",
package_path)
return meta_data
def maplink(self, link, user, index, project):
parts = link.torelpath().split("/")
assert parts
basename = unquote(parts[-1])
if link.hash_spec:
# we can only create 32K entries per directory
# so let's take the first 3 bytes which gives
# us a maximum of 16^3 = 4096 entries in the root dir
a, b = make_splitdir(link.hash_spec)
key = self.keyfs.STAGEFILE(user=user,
index=index,
hashdir_a=a,
hashdir_b=b,
filename=link.basename)
else:
dirname = "_".join(parts[:-1])
dirname = re.sub('[^a-zA-Z0-9_.-]', '_', dirname)
key = self.keyfs.PYPIFILE_NOMD5(user=user,
index=index,
dirname=unquote(dirname),
basename=basename)
entry = FileEntry(self.xom, key, readonly=False)
entry.url = link.geturl_nofragment().url
entry.eggfragment = link.eggfragment
# verify checksum if the entry is fresh, a file exists
# and the link specifies a checksum. It's a situation
# that shouldn't happen unless some manual file system
# intervention or corruption happened
if link.hash_spec and entry.file_exists() and not entry.hash_spec:
threadlog.debug("verifying checksum of %s", entry.relpath)
err = get_checksum_error(entry.file_get_content(), link.hash_spec)
if err:
threadlog.error(err)
entry.file_delete()
entry.hash_spec = unicode_if_bytes(link.hash_spec)
entry.project = project
version = None
if link.eggfragment:
version = link.eggfragment[len(project) + 1:]
else:
try:
(projectname, version, ext) = splitbasename(basename)
except ValueError:
pass
# only store version on entry if we can determine it
# since version is a meta property of FileEntry, it will return None
# if not set, if we set it explicitly, it would waste space in the
# database
if version is not None:
entry.version = version
return entry
def import_filedesc(self, stage, filedesc):
assert stage.ixconfig["type"] != "mirror"
rel = filedesc["relpath"]
#projectname = filedesc["projectname"]
p = self.import_rootdir.join(rel)
assert p.check(), p
if filedesc["type"] == "releasefile":
mapping = filedesc["entrymapping"]
entry = stage.store_releasefile(p.basename, p.read("rb"),
last_modified=mapping["last_modified"])
assert entry.md5 == mapping["md5"]
assert entry.size == mapping["size"]
self.import_attachments(entry.md5)
elif filedesc["type"] == "doczip":
basename = os.path.basename(rel)
name, version, suffix = splitbasename(basename)
stage.store_doczip(name, version, p.open("rb"))
else:
fatal("unknown file type: %s" % (type,))
def import_filedesc(self, stage, filedesc):
assert stage.ixconfig["type"] != "mirror"
rel = filedesc["relpath"]
#projectname = filedesc["projectname"]
p = self.import_rootdir.join(rel)
assert p.check(), p
if filedesc["type"] == "releasefile":
mapping = filedesc["entrymapping"]
entry = stage.store_releasefile(
p.basename,
p.read("rb"),
last_modified=mapping["last_modified"])
assert entry.md5 == mapping["md5"]
assert entry.size == mapping["size"]
self.import_attachments(entry.md5)
elif filedesc["type"] == "doczip":
basename = os.path.basename(rel)
name, version, suffix = splitbasename(basename)
stage.store_doczip(name, version, p.open("rb"))
else:
fatal("unknown file type: %s" % (type, ))
def __init__(self, path):
self.name, self.version = splitbasename(path + ".zip")[:2]
def test_splitbasename(releasename, expected):
result = splitbasename(releasename)
assert result == expected
def __init__(self, path):
self.name, self.version = splitbasename(path + ".zip")[:2]
def __init__(self, key_href):
self.key, self.href = key_href
self._url = URL(self.href)
self.name, self.version, self.ext = splitbasename(self._url.basename,
checkarch=False)
self.eggfragment = self._url.eggfragment
