def create_from_data(cls, project, package_data):
"""
Create and return a DiscoveredPackage for `project` from the `package_data`.
If one of the required fields value is not available, a ProjectError is created
in place of a new DiscoveredPackage instance.
"""
required_fields = ["type", "name", "version"]
required_values = [
package_data.get(field) for field in required_fields
]
if not all(required_values):
message = (f"One or more of the required fields have no value: "
f"{', '.join(required_fields)}")
project.add_error(error=message,
model=cls.__name__,
details=package_data)
return
qualifiers = package_data.get("qualifiers")
if qualifiers:
package_data["qualifiers"] = normalize_qualifiers(qualifiers,
encode=True)
cleaned_package_data = {
field_name: value
for field_name, value in package_data.items()
if field_name in DiscoveredPackage.model_fields() and value
}
discovered_package = cls.objects.create(project=project,
**cleaned_package_data)
if discovered_package.pk:
return discovered_package
def update_or_create_package(project, package_data):
"""
Get and update or create a DiscoveredPackage then return it.
Use the `project` and `package_data` mapping to lookup and create the
DiscoveredPackage using its Package URL as a unique key.
"""
# make a copy
package_data = dict(package_data or {})
if not package_data:
return
# keep only known fields with values
package_data = {
field_name: value
for field_name, value in package_data.items()
if field_name in DiscoveredPackage.model_fields() and value
}
purl_fields = ("type", "namespace", "name", "version", "qualifiers",
"subpath")
purl_data = {}
for k in purl_fields:
# get and remove
v = package_data.pop(k, "")
if k == "qualifiers":
v = normalize_qualifiers(v, encode=True)
purl_data[k] = v or ""
if not purl_data:
raise Exception(
f"Package without any Package URL fields: {package_data}")
# if 'type' not in purl_data and 'name' not in purl_data:
# raise Exception(
# f'Package missing type and name Package URL fields: {package_data}')
# FIXME: we should also consider the download URL as part of the key
# Ensure a purl is treated like if this is the UNIQUE key to a package.
dp, created = DiscoveredPackage.objects.get_or_create(
project=project, **purl_data, defaults=package_data)
if not created:
# update/merge records since we have an existing record
dp_fields = DiscoveredPackage.model_fields()
has_updates = False
for field_name, value in package_data.items():
if field_name not in dp_fields or not value:
continue
existing_value = getattr(dp, field_name, "")
if not existing_value:
setattr(dp, field_name, value)
has_updates = True
elif existing_value != value:
# TODO: handle this case
pass
if has_updates:
dp.save()
return dp
def test_normalize_qualifiers_as_dict(self):
qualifiers_as_dict = {
'classifier': 'sources',
'repository_url': 'repo.spring.io/release'
}
qualifiers_as_string = 'classifier=sources&repository_url=repo.spring.io/release'
assert qualifiers_as_dict == normalize_qualifiers(qualifiers_as_string,
encode=False)
def to_dict(self, **kwargs):
mapping = super().to_dict(**kwargs)
mapping['purl'] = self.purl
if self.qualifiers:
mapping['qualifiers'] = normalize_qualifiers(
qualifiers=self.qualifiers,
encode=False,
)
return mapping
def to_dict(self, **kwargs):
"""
Return an dict of primitive Python types.
"""
mapping = attr.asdict(self, dict_factory=dict)
if not kwargs.get('exclude_properties'):
mapping['purl'] = self.purl
mapping['repository_homepage_url'] = self.repository_homepage_url()
mapping['repository_download_url'] = self.repository_download_url()
mapping['api_data_url'] = self.api_data_url()
if self.qualifiers:
mapping['qualifiers'] = normalize_qualifiers(self.qualifiers,
encode=False)
return mapping
def to_dict(self, **kwargs):
"""
Return an dict of primitive Python types.
"""
mapping = attr.asdict(self)
mapping['purl'] = self.purl
mapping['repository_homepage_url'] = self.repository_homepage_url()
mapping['repository_download_url'] = self.repository_download_url()
mapping['api_data_url'] = self.api_data_url()
if self.qualifiers:
mapping['qualifiers'] = normalize_qualifiers(
qualifiers=self.qualifiers,
encode=False,
)
return mapping
def create_from_data(cls, project, package_data):
"""
Create and return a DiscoveredPackage for `project` using the
`package_data` mapping.
# TODO: we should ensure these entries are UNIQUE
# tomd: Create a ProjectError if not unique?
"""
qualifiers = package_data.get("qualifiers")
if qualifiers:
package_data["qualifiers"] = normalize_qualifiers(qualifiers,
encode=True)
cleaned_package_data = {
field_name: value
for field_name, value in package_data.items()
if field_name in DiscoveredPackage.model_fields() and value
}
return cls.objects.create(project=project, **cleaned_package_data)
def _get_or_create_package(p: PackageURL) -> Tuple[models.Package, bool]:
version = p.version
query_kwargs = {
"name": packageurl.normalize_name(p.name, p.type, encode=True),
"version": version,
"type": packageurl.normalize_type(p.type, encode=True),
}
if p.namespace:
query_kwargs["namespace"] = packageurl.normalize_namespace(p.namespace,
p.type,
encode=True)
if p.qualifiers:
query_kwargs["qualifiers"] = packageurl.normalize_qualifiers(
p.qualifiers, encode=False)
if p.subpath:
query_kwargs["subpath"] = packageurl.normalize_subpath(p.subpath,
encode=True)
return models.Package.objects.get_or_create(**query_kwargs)
class BasePackage(BaseModel):
"""
A base identifiable package object using discrete identifying attributes as
specified here https://github.com/package-url/purl-spec.
"""
# class-level attributes used to recognize a package
filetypes = tuple()
mimetypes = tuple()
extensions = tuple()
# list of known metafiles for a package type
metafiles = []
# Optional. Public default web base URL for package homepages of this
# package type on the default repository.
default_web_baseurl = None
# Optional. Public default download base URL for direct downloads of this
# package type the default repository.
default_download_baseurl = None
# Optional. Public default API repository base URL for package API calls of
# this package type on the default repository.
default_api_baseurl = None
# Optional. Public default type for a package class.
default_type = None
# TODO: add description of the Package type for info
# type_description = None
type = String(
repr=True,
label='package type',
help='Optional. A short code to identify what is the type of this '
'package. For instance gem for a Rubygem, docker for container, '
'pypi for Python Wheel or Egg, maven for a Maven Jar, '
'deb for a Debian package, etc.')
namespace = String(repr=True,
label='package namespace',
help='Optional namespace for this package.')
name = String(repr=True, label='package name', help='Name of the package.')
version = String(repr=True,
label='package version',
help='Optional version of the package as a string.')
qualifiers = Mapping(
default=None,
value_type=str,
converter=lambda v: normalize_qualifiers(v, encode=False),
label='package qualifiers',
help='Optional mapping of key=value pairs qualifiers for this package')
subpath = String(
label='extra package subpath',
help='Optional extra subpath inside a package and relative to the root '
'of this package')
def __attrs_post_init__(self, *args, **kwargs):
if not self.type and hasattr(self, 'default_type'):
self.type = self.default_type
@property
def purl(self):
"""
Return a compact purl package URL string.
"""
if not self.name:
return
return PackageURL(self.type, self.namespace, self.name, self.version,
self.qualifiers, self.subpath).to_string()
def repository_homepage_url(self, baseurl=default_web_baseurl):
"""
Return the package repository homepage URL for this package, e.g. the
URL to the page for this package in its package repository. This is
typically different from the package homepage URL proper.
Subclasses should override to provide a proper value.
"""
return
def repository_download_url(self, baseurl=default_download_baseurl):
"""
Return the package repository download URL to download the actual
archive of code of this package. This may be different than the actual
download URL and is computed from the default public respoitory baseurl.
Subclasses should override to provide a proper value.
"""
return
def api_data_url(self, baseurl=default_api_baseurl):
"""
Return the package repository API URL to obtain structured data for this
package such as the URL to a JSON or XML api.
Subclasses should override to provide a proper value.
"""
return
def set_purl(self, package_url):
"""
Update this Package object with the `package_url` purl string or
PackageURL attributes.
"""
if not package_url:
return
if not isinstance(package_url, PackageURL):
package_url = PackageURL.from_string(package_url)
attribs = [
'type', 'namespace', 'name', 'version', 'qualifiers', 'subpath'
]
for att in attribs:
self_val = getattr(self, att)
purl_val = getattr(package_url, att)
if not self_val and purl_val:
setattr(self, att, purl_val)
def to_dict(self, **kwargs):
"""
Return an dict of primitive Python types.
"""
mapping = attr.asdict(self, dict_factory=dict)
if not kwargs.get('exclude_properties'):
mapping['purl'] = self.purl
mapping['repository_homepage_url'] = self.repository_homepage_url()
mapping['repository_download_url'] = self.repository_download_url()
mapping['api_data_url'] = self.api_data_url()
if self.qualifiers:
mapping['qualifiers'] = normalize_qualifiers(self.qualifiers,
encode=False)
return mapping
@classmethod
def create(cls, ignore_unknown=True, **kwargs):
"""
Return a Package built from kwargs.
Optionally `ignore_unknown` attributes provided in `kwargs`.
"""
from packagedcode import get_package_class
cls = get_package_class(kwargs, default=cls)
return super(BasePackage, cls).create(ignore_unknown=ignore_unknown,
**kwargs)
class IdentifiablePackageData(ModelMixin):
"""
Identifiable package data object using purl as identifying attribute as
specified here https://github.com/package-url/purl-spec.
This base class is used for all package-like objects be they a manifest
or an actual package instance.
"""
type = String(
repr=True,
label='package type',
help='A short code to identify what is the type of this '
'package. For instance gem for a Rubygem, docker for container, '
'pypi for Python Wheel or Egg, maven for a Maven Jar, '
'deb for a Debian package, etc.')
namespace = String(repr=True,
label='package namespace',
help='Namespace for this package.')
name = String(repr=True, label='package name', help='Name of the package.')
version = String(repr=True,
label='package version',
help='Version of the package as a string.')
qualifiers = Mapping(
default=None,
value_type=str,
converter=lambda v: normalize_qualifiers(v, encode=False),
label='package qualifiers',
help='Mapping of key=value pairs qualifiers for this package')
subpath = String(label='extra package subpath',
help='Subpath inside a package and relative to the root '
'of this package')
@property
def purl(self):
"""
Return a compact Package URL string or None.
"""
if self.name:
return PackageURL(
type=self.type,
namespace=self.namespace,
name=self.name,
version=self.version,
qualifiers=self.qualifiers,
subpath=self.subpath,
).to_string()
def set_purl(self, package_url):
"""
Update this object with the ``package_url`` purl string or PackageURL if
there is no pre-existing value for a given purl attribute.
"""
if not package_url:
return
if not isinstance(package_url, PackageURL):
package_url = PackageURL.from_string(package_url)
for key, value in package_url.to_dict().items():
self_val = getattr(self, key)
if not self_val and value:
setattr(self, attr, value)
def to_dict(self, **kwargs):
mapping = super().to_dict(**kwargs)
mapping['purl'] = self.purl
if self.qualifiers:
mapping['qualifiers'] = normalize_qualifiers(
qualifiers=self.qualifiers,
encode=False,
)
return mapping
