def update_or_create_package(project, package_data):
"""
Get and update or create a DiscoveredPackage then return it.
Use the `project` and `package_data` mapping to lookup and create the
DiscoveredPackage using its Package URL as a unique key.
"""
# make a copy
package_data = dict(package_data or {})
if not package_data:
return
# keep only known fields with values
package_data = {
field_name: value
for field_name, value in package_data.items()
if field_name in DiscoveredPackage.model_fields() and value
}
purl_fields = ("type", "namespace", "name", "version", "qualifiers",
"subpath")
purl_data = {}
for k in purl_fields:
# get and remove
v = package_data.pop(k, "")
if k == "qualifiers":
v = normalize_qualifiers(v, encode=True)
purl_data[k] = v or ""
if not purl_data:
raise Exception(
f"Package without any Package URL fields: {package_data}")
# if 'type' not in purl_data and 'name' not in purl_data:
# raise Exception(
# f'Package missing type and name Package URL fields: {package_data}')
# FIXME: we should also consider the download URL as part of the key
# Ensure a purl is treated like if this is the UNIQUE key to a package.
dp, created = DiscoveredPackage.objects.get_or_create(
project=project, **purl_data, defaults=package_data)
if not created:
# update/merge records since we have an existing record
dp_fields = DiscoveredPackage.model_fields()
has_updates = False
for field_name, value in package_data.items():
if field_name not in dp_fields or not value:
continue
existing_value = getattr(dp, field_name, "")
if not existing_value:
setattr(dp, field_name, value)
has_updates = True
elif existing_value != value:
# TODO: handle this case
pass
if has_updates:
dp.save()
return dp
def test_scanpipe_pipes_outputs_to_xlsx(self):
project1 = Project.objects.create(name="Analysis")
codebase_resource = CodebaseResource.objects.create(
project=project1,
path="filename.ext",
)
DiscoveredPackage.create_for_resource(package_data1, codebase_resource)
output_file = output.to_xlsx(project=project1)
self.assertEqual([output_file.name], project1.output_root)
def scan_for_application_packages(project):
"""
Run a package scan on files without status.
"""
queryset = CodebaseResource.objects.project(project).no_status()
for codebase_resource in queryset:
package_info = scancode_api.get_package_info(codebase_resource.location)
packages = package_info.get("packages", [])
if packages:
for package in packages:
DiscoveredPackage.create_for_resource(package, codebase_resource)
codebase_resource.status = "application-package"
codebase_resource.save()
def test_scanpipe_pipes_outputs_queryset_to_csv_stream(self):
project1 = Project.objects.create(name="Analysis")
codebase_resource = CodebaseResource.objects.create(
project=project1,
path="filename.ext",
)
DiscoveredPackage.create_for_resource(
package_data1,
codebase_resource,
)
queryset = project1.discoveredpackages.all()
fieldnames = ["purl", "name", "version"]
output_file = project1.get_output_file_path("packages", "csv")
with output_file.open("w") as output_stream:
generator = output.queryset_to_csv_stream(
queryset, fieldnames, output_stream
)
collections.deque(generator, maxlen=0) # Exhaust the generator
expected = [
"purl,name,version\n",
"pkg:deb/debian/[email protected]?arch=all,adduser,3.118\n",
]
with output_file.open() as f:
self.assertEqual(expected, f.readlines())
queryset = project1.codebaseresources.all()
fieldnames = ["for_packages", "path"]
output_file = project1.get_output_file_path("resources", "csv")
with output_file.open("w") as output_stream:
generator = output.queryset_to_csv_stream(
queryset, fieldnames, output_stream
)
collections.deque(generator, maxlen=0) # Exhaust the generator
output.queryset_to_csv_stream(queryset, fieldnames, output_file)
expected = [
"for_packages,path\n",
"['pkg:deb/debian/[email protected]?arch=all'],filename.ext\n",
]
with output_file.open() as f:
self.assertEqual(expected, f.readlines())
def test_scanpipe_discovered_package_model_create_for_resource(self):
codebase_resource = CodebaseResource.objects.create(
project=self.project1, path="filename.ext")
package = DiscoveredPackage.create_for_resource(
package_data1, codebase_resource)
self.assertEqual(self.project1, package.project)
self.assertEqual("pkg:deb/debian/[email protected]?arch=all", str(package))
self.assertEqual(1, codebase_resource.discovered_packages.count())
self.assertEqual(package, codebase_resource.discovered_packages.get())
def test_scanpipe_pipes_outputs_to_json(self):
project1 = Project.objects.create(name="Analysis")
codebase_resource = CodebaseResource.objects.create(
project=project1,
path="filename.ext",
)
DiscoveredPackage.create_for_resource(package_data1, codebase_resource)
output_file = output.to_json(project=project1)
self.assertEqual([output_file.name], project1.output_root)
with output_file.open() as f:
results = json.loads(f.read())
expected = ["files", "headers", "packages"]
self.assertEqual(expected, sorted(results.keys()))
self.assertEqual(1, len(results["headers"]))
self.assertEqual(1, len(results["files"]))
self.assertEqual(1, len(results["packages"]))
def test_scanpipe_discovered_package_model_create_from_data(self):
package = DiscoveredPackage.create_from_data(self.project1,
package_data1)
self.assertEqual(self.project1, package.project)
self.assertEqual("pkg:deb/debian/[email protected]?arch=all", str(package))
self.assertEqual("deb", package.type)
self.assertEqual("debian", package.namespace)
self.assertEqual("adduser", package.name)
self.assertEqual("3.118", package.version)
self.assertEqual("arch=all", package.qualifiers)
self.assertEqual("add and remove users and groups",
package.description)
self.assertEqual("849", package.size)
self.assertEqual("gpl-2.0 AND gpl-2.0-plus AND unknown",
package.license_expression)
package_count = DiscoveredPackage.objects.count()
missing_required_field = dict(package_data1)
missing_required_field["name"] = ""
self.assertIsNone(
DiscoveredPackage.create_from_data(self.project1,
missing_required_field))
self.assertEqual(package_count, DiscoveredPackage.objects.count())
def test_scanpipe_discovered_package_model_create_from_data(self):
package = DiscoveredPackage.create_from_data(self.project1,
package_data1)
self.assertEqual(self.project1, package.project)
self.assertEqual("pkg:deb/debian/[email protected]?arch=all", str(package))
self.assertEqual("deb", package.type)
self.assertEqual("debian", package.namespace)
self.assertEqual("adduser", package.name)
self.assertEqual("3.118", package.version)
self.assertEqual("arch=all", package.qualifiers)
self.assertEqual("add and remove users and groups",
package.description)
self.assertEqual("849", package.size)
self.assertEqual("gpl-2.0 AND gpl-2.0-plus AND unknown",
package.license_expression)
def setUp(self):
self.project1 = Project.objects.create(name="Analysis")
self.codebase_resource1 = CodebaseResource.objects.create(
project=self.project1, path="filename.ext"
)
self.discovered_package1 = DiscoveredPackage.create_for_resource(
package_data1, self.codebase_resource1
)
self.project_list_url = reverse("project-list")
self.project1_detail_url = reverse("project-detail", args=[self.project1.uuid])
self.user = User.objects.create_user("username", "*****@*****.**", "secret")
self.header_prefix = "Token "
self.token = Token.objects.create(user=self.user)
self.auth = self.header_prefix + self.token.key
self.csrf_client = APIClient(enforce_csrf_checks=True)
self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth)
def test_scanpipe_codebase_resource_type_methods(self):
CodebaseResource.objects.all().delete()
file = CodebaseResource.objects.create(project=self.project1,
type=CodebaseResource.Type.FILE,
path="file")
directory = CodebaseResource.objects.create(
project=self.project1,
type=CodebaseResource.Type.DIRECTORY,
path="directory",
)
symlink = CodebaseResource.objects.create(
project=self.project1,
type=CodebaseResource.Type.SYMLINK,
path="symlink")
self.assertTrue(file.is_file)
self.assertFalse(file.is_dir)
self.assertFalse(file.is_symlink)
self.assertFalse(directory.is_file)
self.assertTrue(directory.is_dir)
self.assertFalse(directory.is_symlink)
self.assertFalse(symlink.is_file)
self.assertFalse(symlink.is_dir)
self.assertTrue(symlink.is_symlink)
qs = CodebaseResource.objects.files()
self.assertEqual(1, len(qs))
self.assertIn(file, qs)
qs = CodebaseResource.objects.directories()
self.assertEqual(1, len(qs))
self.assertIn(directory, qs)
qs = CodebaseResource.objects.symlinks()
self.assertEqual(1, len(qs))
self.assertIn(symlink, qs)
qs = CodebaseResource.objects.without_symlinks()
self.assertEqual(2, len(qs))
self.assertIn(file, qs)
self.assertIn(directory, qs)
self.assertNotIn(symlink, qs)
file.licenses = [{"key": "bsd-new", "name": "BSD-3-Clause"}]
file.save()
qs = CodebaseResource.objects.has_licenses()
self.assertEqual(1, len(qs))
self.assertIn(file, qs)
self.assertNotIn(directory, qs)
self.assertNotIn(symlink, qs)
qs = CodebaseResource.objects.has_no_licenses()
self.assertEqual(2, len(qs))
self.assertNotIn(file, qs)
self.assertIn(directory, qs)
self.assertIn(symlink, qs)
self.assertEqual(0, CodebaseResource.objects.in_package().count())
self.assertEqual(3, CodebaseResource.objects.not_in_package().count())
DiscoveredPackage.create_for_resource(package_data1, file)
self.assertEqual(1, CodebaseResource.objects.in_package().count())
self.assertEqual(2, CodebaseResource.objects.not_in_package().count())
