Esempio n. 1
0
def package_versions_json(context, request):
    """ Render the package versions in JSON format """
    pkgs = package_versions(context, request)
    if not isinstance(pkgs, dict):
        return pkgs
    response = {
        'info': {
            'name': context.name,
        },
        'releases': {},
    }
    max_version = None
    for filename, url in six.iteritems(pkgs['pkgs']):
        name, version_str = parse_filename(filename)
        version = pkg_resources.parse_version(version_str)
        if max_version is None or version > max_version:
            max_version = version

        response['releases'].setdefault(version_str, []).append({
            'filename': filename,
            'url': url,
        })
    if max_version is not None:
        response['urls'] = response['releases'].get(str(max_version), [])
    return response
Esempio n. 2
0
def upload(request,
           content,
           name=None,
           version=None,
           summary=None,
           requires_python=None):
    """ Handle update commands """
    action = request.param(":action", "file_upload")
    # Direct uploads from the web UI go here, and don't have a name/version
    if name is None or version is None:
        name, version = parse_filename(content.filename)
    else:
        name = normalize_name(name)
    if action == "file_upload":
        if not request.access.has_permission(name, "write"):
            return request.forbid()
        try:
            return request.db.upload(
                content.filename,
                content.file,
                name=name,
                version=version,
                summary=summary,
                requires_python=requires_python or None,
            )
        except ValueError as e:
            return HTTPConflict(*e.args)
    else:
        return HTTPBadRequest("Unknown action '%s'" % action)
Esempio n. 3
0
    def list(self, factory=Package):
        keys = self.bucket.list(self.bucket_prefix)
        for key in keys:
            # Boto doesn't send down metadata from bucket.list()
            # so we are forced to retrieve each key individually.
            key = self.bucket.get_key(key.key)
            filename = posixpath.basename(key.key)
            name = key.get_metadata('name')
            version = key.get_metadata('version')
            summary = key.get_metadata('summary')

            # We used to not store metadata. This is for backwards
            # compatibility
            if name is None or version is None:
                try:
                    name, version = parse_filename(filename)
                except ValueError:
                    LOG.warning("S3 file %s has no package name", key.key)
                    continue

            last_modified = boto.utils.parse_ts(key.last_modified)

            pkg = factory(name,
                          version,
                          filename,
                          last_modified,
                          summary,
                          path=key.key)

            yield pkg
Esempio n. 4
0
def package_versions_json(context, request):
    """ Render the package versions in JSON format """
    pkgs = _package_versions(context, request)
    if not isinstance(pkgs, dict):
        return pkgs
    response = {"info": {"name": context.name}, "releases": {}}
    max_version = None
    for filename, pkg in pkgs["pkgs"].items():
        name, version_str = parse_filename(filename)
        version = pkg_resources.parse_version(version_str)
        if max_version is None or version > max_version:
            max_version = version

        release = {
            "filename": filename,
            "url": pkg.get("non_hashed_url", pkg["url"]),
            "requires_python": pkg["requires_python"],
        }
        if pkg.get("hash_sha256"):
            release["digests"] = {
                "md5": pkg["hash_md5"],
                "sha256": pkg["hash_sha256"]
            }
            release["md5_digest"] = pkg["hash_md5"]

        response["releases"].setdefault(version_str, []).append(release)
    if max_version is not None:
        response["urls"] = response["releases"].get(str(max_version), [])
    return response
Esempio n. 5
0
def upload(request, content, name=None, version=None, summary=None):
    """ Handle update commands """
    action = request.param(":action", "file_upload")
    # Direct uploads from the web UI go here, and don't have a name/version
    if name is None or version is None:
        name, version = parse_filename(content.filename)
    else:
        name = normalize_name(name)
    if action == "file_upload":
        if not request.access.has_permission(name, "write"):
            return request.forbid()
        print("TYPE", type(content.file))
        try:
            file_content = content.file.read()
            return request.db.upload(
                content.filename,
                file_content,
                name=name,
                digest=hashlib.sha256(file_content).hexdigest(),
                version=version,
                summary=summary,
            )
        except ValueError as e:
            return HTTPConflict(*e.args)
    else:
        return HTTPBadRequest("Unknown action '%s'" % action)
Esempio n. 6
0
    def upload(
        self,
        filename: str,
        data: BinaryIO,
        name: Optional[str] = None,
        version: Optional[str] = None,
        summary: Optional[str] = None,
        requires_python: Optional[str] = None,
    ) -> Package:
        """
        Save this package to the storage mechanism and to the cache

        Parameters
        ----------
        filename : str
            Name of the package file
        data : file
            File-like readable object
        name : str, optional
            The name of the package (if not provided, will be parsed from
            filename)
        version : str, optional
            The version number of the package (if not provided, will be parsed
            from filename)
        summary : str, optional
            The summary of the package
        requires_python : str, optional
            The Python version requirement

        Returns
        -------
        package : :class:`~pypicloud.models.Package`
            The Package object that was uploaded

        Raises
        ------
        e : ValueError
            If the package already exists and allow_overwrite = False

        """
        if version is None or name is None:
            name, version = parse_filename(filename, name)
        name = normalize_name(name)
        filename = posixpath.basename(filename)
        old_pkg = self.fetch(filename)
        metadata = {"requires_python": requires_python}
        if old_pkg is not None and not self.allow_overwrite:
            raise ValueError("Package '%s' already exists!" % filename)
        if self.calculate_hashes:
            file_data = data.read()
            metadata["hash_sha256"] = hashlib.sha256(file_data).hexdigest()
            metadata["hash_md5"] = hashlib.md5(file_data).hexdigest()
            data = BytesIO(file_data)

        new_pkg = self.new_package(name, version, filename, summary=summary, **metadata)
        self.storage.upload(new_pkg, data)
        self.save(new_pkg)
        return new_pkg
Esempio n. 7
0
    def upload(
        self,
        filename,
        data,
        name=None,
        version=None,
        summary=None,
        requires_python=None,
    ):
        """
        Save this package to the storage mechanism and to the cache

        Parameters
        ----------
        filename : str
            Name of the package file
        data : file
            File-like readable object
        name : str, optional
            The name of the package (if not provided, will be parsed from
            filename)
        version : str, optional
            The version number of the package (if not provided, will be parsed
            from filename)
        summary : str, optional
            The summary of the package
        requires_python : str, optional
            The Python version requirement

        Returns
        -------
        package : :class:`~pypicloud.models.Package`
            The Package object that was uploaded

        Raises
        ------
        e : ValueError
            If the package already exists and allow_overwrite = False

        """
        if version is None:
            name, version = parse_filename(filename, name)
        name = normalize_name(name)
        filename = posixpath.basename(filename)
        old_pkg = self.fetch(filename)
        if old_pkg is not None and not self.allow_overwrite:
            raise ValueError("Package '%s' already exists!" % filename)
        new_pkg = self.package_class(name,
                                     version,
                                     filename,
                                     summary=summary,
                                     requires_python=requires_python)
        self.storage.upload(new_pkg, data)
        self.save(new_pkg)
        return new_pkg
Esempio n. 8
0
def upload(request, content, name=None, version=None):
    """ Handle update commands """
    action = request.param(':action', 'file_upload')
    # Direct uploads from the web UI go here, and don't have a name/version
    if name is None or version is None:
        name, version = parse_filename(content.filename)
    else:
        name = normalize_name(name)
    if action == 'file_upload':
        if not request.access.has_permission(name, 'write'):
            return request.forbid()
        try:
            return request.db.upload(content.filename, content.file, name=name,
                                     version=version)
        except ValueError as e:
            return HTTPBadRequest(*e.args)
    else:
        return HTTPBadRequest("Unknown action '%s'" % action)
Esempio n. 9
0
    def package_from_object(cls, obj, factory):
        """ Create a package from a S3 object """
        filename = posixpath.basename(obj.key)
        name = obj.metadata.get("name")
        version = obj.metadata.get("version")
        summary = obj.metadata.get("summary")
        # We used to not store metadata. This is for backwards
        # compatibility
        if name is None or version is None:
            try:
                name, version = parse_filename(filename)
            except ValueError:
                LOG.warning("S3 file %s has no package name", obj.key)
                return None

        return factory(
            name, version, filename, obj.last_modified, summary, path=obj.key
        )
Esempio n. 10
0
    def package_from_object(cls, obj, factory):
        """ Create a package from a S3 object """
        filename = posixpath.basename(obj.key)
        name = obj.metadata.get("name")
        version = obj.metadata.get("version")
        summary = obj.metadata.get("summary")
        digest = obj.metadata.get("digest")
        # We used to not store metadata. This is for backwards
        # compatibility
        if name is None or version is None:
            try:
                name, version = parse_filename(filename)
            except ValueError:
                LOG.warning("S3 file %s has no package name", obj.key)
                return None

        return factory(name, version, filename, digest, obj.last_modified,
                       summary)
Esempio n. 11
0
def package_versions_json(context, request):
    """ Render the package versions in JSON format """
    pkgs = _package_versions(context, request)
    if not isinstance(pkgs, dict):
        return pkgs
    response = {"info": {"name": context.name}, "releases": {}}
    max_version = None
    for filename, url in six.iteritems(pkgs["pkgs"]):
        name, version_str = parse_filename(filename)
        version = pkg_resources.parse_version(version_str)
        if max_version is None or version > max_version:
            max_version = version

        response["releases"].setdefault(version_str, []).append(
            {"filename": filename, "url": url}
        )
    if max_version is not None:
        response["urls"] = response["releases"].get(str(max_version), [])
    return response
Esempio n. 12
0
def package_versions_json(context, request):
    """ Render the package versions in JSON format """
    pkgs = _package_versions(context, request)
    if not isinstance(pkgs, dict):
        return pkgs
    response = {"info": {"name": context.name}, "releases": {}}
    max_version = None
    for filename, pkg in six.iteritems(pkgs["pkgs"]):
        name, version_str = parse_filename(filename)
        version = pkg_resources.parse_version(version_str)
        if max_version is None or version > max_version:
            max_version = version

        response["releases"].setdefault(version_str, []).append(
            {"filename": filename, "url": pkg["url"]}
        )
    if max_version is not None:
        response["urls"] = response["releases"].get(str(max_version), [])
    return response
Esempio n. 13
0
def handle_s3_event(event, context):
    """ Handle S3 object notification """
    from pypicloud.cache import get_cache_impl
    from pypicloud.storage.s3 import S3Storage
    from pypicloud.util import parse_filename

    settings = json.loads(os.environ["PYPICLOUD_SETTINGS"])
    # Set 'file' storage as a hack. We're going to load the cache, which will
    # load a storage. We won't actually be using the storage for anything, but
    # the settings have to be present.
    settings.setdefault("pypi.storage", "file")
    settings.setdefault("storage.dir", "/tmp")
    cache_impl = get_cache_impl(settings)
    kwargs = cache_impl.configure(settings)
    cache = cache_impl(**kwargs)

    s3 = boto3.resource("s3")
    for record in event["Records"]:
        bucket = record["s3"]["bucket"]["name"]
        key = record["s3"]["object"]["key"]
        event_name = record["eventName"]
        if event_name.startswith("ObjectCreated"):
            print("S3 object %r created" % key)
            obj = s3.Object(bucket, key)
            package = S3Storage.package_from_object(obj, cache.package_class)
            existing_pkg = cache.fetch(package.filename)
            if existing_pkg is None:
                print("Saving package %s" % package)
                cache.save(package)
            else:
                print("Package already cached")
        else:
            print("S3 object %r deleted" % key)
            filename = posixpath.basename(key)
            try:
                name, version = parse_filename(filename)
            except ValueError:
                name = version = "dummy"
            package = cache.package_class(
                name, version, filename, datetime.utcnow(), ""
            )
            print("Deleting package %s" % package)
            cache.clear(package)
Esempio n. 14
0
def handle_s3_event(event, context):
    """ Handle S3 object notification """
    from pypicloud.cache import get_cache_impl
    from pypicloud.storage.s3 import package_from_object
    from pypicloud.util import parse_filename

    settings = json.loads(os.environ['PYPICLOUD_SETTINGS'])
    # Set 'file' storage as a hack. We're going to load the cache, which will
    # load a storage. We won't actually be using the storage for anything, but
    # the settings have to be present.
    settings.setdefault('pypi.storage', 'file')
    settings.setdefault('storage.dir', '/tmp')
    cache_impl = get_cache_impl(settings)
    kwargs = cache_impl.configure(settings)
    cache = cache_impl(**kwargs)

    s3 = boto3.resource('s3')
    for record in event['Records']:
        bucket = record['s3']['bucket']['name']
        key = record['s3']['object']['key']
        event_name = record['eventName']
        if event_name.startswith('ObjectCreated'):
            print("S3 object %r created" % key)
            obj = s3.Object(bucket, key)
            package = package_from_object(obj, cache.package_class)
            existing_pkg = cache.fetch(package.filename)
            if existing_pkg is None:
                print("Saving package %s" % package)
                cache.save(package)
            else:
                print("Package already cached")
        else:
            print("S3 object %r deleted" % key)
            filename = posixpath.basename(key)
            try:
                name, version = parse_filename(filename)
            except ValueError:
                name = version = 'dummy'
            package = cache.package_class(name, version, filename,
                                          datetime.utcnow(), '')
            print("Deleting package %s" % package)
            cache.clear(package)
Esempio n. 15
0
    def upload(self, filename, data, name=None, version=None, summary=None):
        """
        Save this package to the storage mechanism and to the cache

        Parameters
        ----------
        filename : str
            Name of the package file
        data : file
            File-like readable object
        name : str, optional
            The name of the package (if not provided, will be parsed from
            filename)
        version : str, optional
            The version number of the package (if not provided, will be parsed
            from filename)
        summary : str, optional
            The summary of the package

        Returns
        -------
        package : :class:`~pypicloud.models.Package`
            The Package object that was uploaded

        Raises
        ------
        e : ValueError
            If the package already exists and allow_overwrite = False

        """
        if version is None:
            name, version = parse_filename(filename, name)
        name = normalize_name(name)
        filename = posixpath.basename(filename)
        old_pkg = self.fetch(filename)
        if old_pkg is not None and not self.allow_overwrite:
            raise ValueError("Package '%s' already exists!" % filename)
        new_pkg = self.package_class(name, version, filename, summary=summary)
        self.storage.upload(new_pkg, data)
        self.save(new_pkg)
        return new_pkg
Esempio n. 16
0
    def package_from_object(cls, obj, factory):
        """Create a package from a S3 object"""
        filename = posixpath.basename(obj.key)
        name = obj.metadata.get("name")
        version = obj.metadata.get("version")
        metadata = Package.read_metadata(obj.metadata)
        # We used to not store metadata. This is for backwards
        # compatibility
        if name is None or version is None:
            try:
                name, version = parse_filename(filename)
            except PackageParseError:
                LOG.warning("S3 file %s has no package name", obj.key)
                return None

        return factory(name,
                       version,
                       filename,
                       obj.last_modified,
                       path=obj.key,
                       **metadata)
Esempio n. 17
0
    def list(self, factory=Package):
        keys = self.bucket.list(self.bucket_prefix)
        for key in keys:
            # Moto doesn't send down metadata from bucket.list()
            if self.test:
                key = self.bucket.get_key(key.key)
            filename = posixpath.basename(key.key)
            name = key.get_metadata('name')
            version = key.get_metadata('version')

            # We used to not store metadata. This is for backwards
            # compatibility
            if name is None or version is None:
                try:
                    name, version = parse_filename(filename)
                except ValueError:
                    LOG.warning("S3 file %s has no package name", key.key)
                    continue

            last_modified = boto.utils.parse_ts(key.last_modified)

            pkg = factory(name, version, filename, last_modified, path=key.key)

            yield pkg
Esempio n. 18
0
    def list(self, factory=Package):
        keys = self.bucket
        for key in keys:
            # Moto doesn't send down metadata from bucket.list()
            if self.test:
                key = self.bucket.get_key(key.key)
            filename = posixpath.basename(key.key)
            name = key.get_metadata('name')
            version = key.get_metadata('version')

            # We used to not store metadata. This is for backwards
            # compatibility
            if name is None or version is None:
                try:
                    name, version = parse_filename(filename)
                except ValueError:
                    LOG.warning("GCS file %s has no package name", key.key)
                    continue

            last_modified = boto.utils.parse_ts(key.last_modified)

            pkg = factory(name, version, filename, last_modified, path=key.key)

            yield pkg
Esempio n. 19
0
 def test_valid_source(self):
     """ Parse a valid source package """
     name, version = util.parse_filename("mypkg-1.1.tar.gz")
     self.assertEqual(name, "mypkg")
     self.assertEqual(version, "1.1")
Esempio n. 20
0
 def test_valid_source(self):
     """ Parse a valid source package """
     name, version = util.parse_filename("mypkg-1.1.tar.gz")
     self.assertEqual(name, "mypkg")
     self.assertEqual(version, "1.1")
Esempio n. 21
0
 def test_use_name(self):
     """ Can pass in name to assist parsing """
     name, version = util.parse_filename("mypkg-1.1-py2.py3-none-any.whl", "mypkg")
     self.assertEqual(name, "mypkg")
     self.assertEqual(version, "1.1")
Esempio n. 22
0
 def test_use_name(self):
     """ Can pass in name to assist parsing """
     name, version = util.parse_filename('mypkg-1.1-py2.py3-none-any.whl',
                                         'mypkg')
     self.assertEqual(name, 'mypkg')
     self.assertEqual(version, '1.1')
Esempio n. 23
0
 def test_invalid_file_ext(self):
     """ Parse fails on invalid file extension """
     with self.assertRaises(ValueError):
         util.parse_filename('mypkg-1.1.pdf')
Esempio n. 24
0
 def test_valid_source(self):
     """ Parse a valid source package """
     name, version = util.parse_filename('mypkg-1.1.tar.gz')
     self.assertEqual(name, 'mypkg')
     self.assertEqual(version, '1.1')
Esempio n. 25
0
 def test_invalid_source(self):
     """ Parse fails on invalid package name """
     with self.assertRaises(ValueError):
         util.parse_filename('invalid_package_name.tar.gz')
Esempio n. 26
0
 def test_valid_source(self):
     """ Parse a valid source package """
     name, version = util.parse_filename('mypkg-1.1.tar.gz')
     self.assertEqual(name, 'mypkg')
     self.assertEqual(version, '1.1')
Esempio n. 27
0
 def test_invalid_file_ext(self):
     """ Parse fails on invalid file extension """
     with self.assertRaises(ValueError):
         util.parse_filename("mypkg-1.1.pdf")
Esempio n. 28
0
 def test_invalid_source(self):
     """ Parse fails on invalid package name """
     with self.assertRaises(ValueError):
         util.parse_filename("invalid_package_name.tar.gz")
Esempio n. 29
0
 def test_valid_wheel(self):
     """ Parse a valid wheel package """
     name, version = util.parse_filename("mypkg-1.1-py2.py3-none-any.whl")
     self.assertEqual(name, "mypkg")
     self.assertEqual(version, "1.1")
Esempio n. 30
0
 def test_valid_wheel(self):
     """ Parse a valid wheel package """
     name, version = util.parse_filename('mypkg-1.1-py2.py3-none-any.whl')
     self.assertEqual(name, 'mypkg')
     self.assertEqual(version, '1.1')
Esempio n. 31
0
 def test_use_name(self):
     """ Can pass in name to assist parsing """
     name, version = util.parse_filename("mypkg-1.1-py2.py3-none-any.whl", "mypkg")
     self.assertEqual(name, "mypkg")
     self.assertEqual(version, "1.1")
Esempio n. 32
0
 def test_use_name(self):
     """ Can pass in name to assist parsing """
     name, version = util.parse_filename('mypkg-1.1-py2.py3-none-any.whl',
                                         'mypkg')
     self.assertEqual(name, 'mypkg')
     self.assertEqual(version, '1.1')