def update_or_create_package(project, package_data): """ Get and update or create a DiscoveredPackage then return it. Use the `project` and `package_data` mapping to lookup and create the DiscoveredPackage using its Package URL as a unique key. """ # make a copy package_data = dict(package_data or {}) if not package_data: return # keep only known fields with values package_data = { field_name: value for field_name, value in package_data.items() if field_name in DiscoveredPackage.model_fields() and value } purl_fields = ("type", "namespace", "name", "version", "qualifiers", "subpath") purl_data = {} for k in purl_fields: # get and remove v = package_data.pop(k, "") if k == "qualifiers": v = normalize_qualifiers(v, encode=True) purl_data[k] = v or "" if not purl_data: raise Exception( f"Package without any Package URL fields: {package_data}") # if 'type' not in purl_data and 'name' not in purl_data: # raise Exception( # f'Package missing type and name Package URL fields: {package_data}') # FIXME: we should also consider the download URL as part of the key # Ensure a purl is treated like if this is the UNIQUE key to a package. dp, created = DiscoveredPackage.objects.get_or_create( project=project, **purl_data, defaults=package_data) if not created: # update/merge records since we have an existing record dp_fields = DiscoveredPackage.model_fields() has_updates = False for field_name, value in package_data.items(): if field_name not in dp_fields or not value: continue existing_value = getattr(dp, field_name, "") if not existing_value: setattr(dp, field_name, value) has_updates = True elif existing_value != value: # TODO: handle this case pass if has_updates: dp.save() return dp
def test_scanpipe_pipes_outputs_to_xlsx(self): project1 = Project.objects.create(name="Analysis") codebase_resource = CodebaseResource.objects.create( project=project1, path="filename.ext", ) DiscoveredPackage.create_for_resource(package_data1, codebase_resource) output_file = output.to_xlsx(project=project1) self.assertEqual([output_file.name], project1.output_root)
def scan_for_application_packages(project): """ Run a package scan on files without status. """ queryset = CodebaseResource.objects.project(project).no_status() for codebase_resource in queryset: package_info = scancode_api.get_package_info(codebase_resource.location) packages = package_info.get("packages", []) if packages: for package in packages: DiscoveredPackage.create_for_resource(package, codebase_resource) codebase_resource.status = "application-package" codebase_resource.save()
def test_scanpipe_pipes_outputs_queryset_to_csv_stream(self): project1 = Project.objects.create(name="Analysis") codebase_resource = CodebaseResource.objects.create( project=project1, path="filename.ext", ) DiscoveredPackage.create_for_resource( package_data1, codebase_resource, ) queryset = project1.discoveredpackages.all() fieldnames = ["purl", "name", "version"] output_file = project1.get_output_file_path("packages", "csv") with output_file.open("w") as output_stream: generator = output.queryset_to_csv_stream( queryset, fieldnames, output_stream ) collections.deque(generator, maxlen=0) # Exhaust the generator expected = [ "purl,name,version\n", "pkg:deb/debian/[email protected]?arch=all,adduser,3.118\n", ] with output_file.open() as f: self.assertEqual(expected, f.readlines()) queryset = project1.codebaseresources.all() fieldnames = ["for_packages", "path"] output_file = project1.get_output_file_path("resources", "csv") with output_file.open("w") as output_stream: generator = output.queryset_to_csv_stream( queryset, fieldnames, output_stream ) collections.deque(generator, maxlen=0) # Exhaust the generator output.queryset_to_csv_stream(queryset, fieldnames, output_file) expected = [ "for_packages,path\n", "['pkg:deb/debian/[email protected]?arch=all'],filename.ext\n", ] with output_file.open() as f: self.assertEqual(expected, f.readlines())
def test_scanpipe_discovered_package_model_create_for_resource(self): codebase_resource = CodebaseResource.objects.create( project=self.project1, path="filename.ext") package = DiscoveredPackage.create_for_resource( package_data1, codebase_resource) self.assertEqual(self.project1, package.project) self.assertEqual("pkg:deb/debian/[email protected]?arch=all", str(package)) self.assertEqual(1, codebase_resource.discovered_packages.count()) self.assertEqual(package, codebase_resource.discovered_packages.get())
def test_scanpipe_pipes_outputs_to_json(self): project1 = Project.objects.create(name="Analysis") codebase_resource = CodebaseResource.objects.create( project=project1, path="filename.ext", ) DiscoveredPackage.create_for_resource(package_data1, codebase_resource) output_file = output.to_json(project=project1) self.assertEqual([output_file.name], project1.output_root) with output_file.open() as f: results = json.loads(f.read()) expected = ["files", "headers", "packages"] self.assertEqual(expected, sorted(results.keys())) self.assertEqual(1, len(results["headers"])) self.assertEqual(1, len(results["files"])) self.assertEqual(1, len(results["packages"]))
def test_scanpipe_discovered_package_model_create_from_data(self): package = DiscoveredPackage.create_from_data(self.project1, package_data1) self.assertEqual(self.project1, package.project) self.assertEqual("pkg:deb/debian/[email protected]?arch=all", str(package)) self.assertEqual("deb", package.type) self.assertEqual("debian", package.namespace) self.assertEqual("adduser", package.name) self.assertEqual("3.118", package.version) self.assertEqual("arch=all", package.qualifiers) self.assertEqual("add and remove users and groups", package.description) self.assertEqual("849", package.size) self.assertEqual("gpl-2.0 AND gpl-2.0-plus AND unknown", package.license_expression) package_count = DiscoveredPackage.objects.count() missing_required_field = dict(package_data1) missing_required_field["name"] = "" self.assertIsNone( DiscoveredPackage.create_from_data(self.project1, missing_required_field)) self.assertEqual(package_count, DiscoveredPackage.objects.count())
def test_scanpipe_discovered_package_model_create_from_data(self): package = DiscoveredPackage.create_from_data(self.project1, package_data1) self.assertEqual(self.project1, package.project) self.assertEqual("pkg:deb/debian/[email protected]?arch=all", str(package)) self.assertEqual("deb", package.type) self.assertEqual("debian", package.namespace) self.assertEqual("adduser", package.name) self.assertEqual("3.118", package.version) self.assertEqual("arch=all", package.qualifiers) self.assertEqual("add and remove users and groups", package.description) self.assertEqual("849", package.size) self.assertEqual("gpl-2.0 AND gpl-2.0-plus AND unknown", package.license_expression)
def setUp(self): self.project1 = Project.objects.create(name="Analysis") self.codebase_resource1 = CodebaseResource.objects.create( project=self.project1, path="filename.ext" ) self.discovered_package1 = DiscoveredPackage.create_for_resource( package_data1, self.codebase_resource1 ) self.project_list_url = reverse("project-list") self.project1_detail_url = reverse("project-detail", args=[self.project1.uuid]) self.user = User.objects.create_user("username", "*****@*****.**", "secret") self.header_prefix = "Token " self.token = Token.objects.create(user=self.user) self.auth = self.header_prefix + self.token.key self.csrf_client = APIClient(enforce_csrf_checks=True) self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth)
def test_scanpipe_codebase_resource_type_methods(self): CodebaseResource.objects.all().delete() file = CodebaseResource.objects.create(project=self.project1, type=CodebaseResource.Type.FILE, path="file") directory = CodebaseResource.objects.create( project=self.project1, type=CodebaseResource.Type.DIRECTORY, path="directory", ) symlink = CodebaseResource.objects.create( project=self.project1, type=CodebaseResource.Type.SYMLINK, path="symlink") self.assertTrue(file.is_file) self.assertFalse(file.is_dir) self.assertFalse(file.is_symlink) self.assertFalse(directory.is_file) self.assertTrue(directory.is_dir) self.assertFalse(directory.is_symlink) self.assertFalse(symlink.is_file) self.assertFalse(symlink.is_dir) self.assertTrue(symlink.is_symlink) qs = CodebaseResource.objects.files() self.assertEqual(1, len(qs)) self.assertIn(file, qs) qs = CodebaseResource.objects.directories() self.assertEqual(1, len(qs)) self.assertIn(directory, qs) qs = CodebaseResource.objects.symlinks() self.assertEqual(1, len(qs)) self.assertIn(symlink, qs) qs = CodebaseResource.objects.without_symlinks() self.assertEqual(2, len(qs)) self.assertIn(file, qs) self.assertIn(directory, qs) self.assertNotIn(symlink, qs) file.licenses = [{"key": "bsd-new", "name": "BSD-3-Clause"}] file.save() qs = CodebaseResource.objects.has_licenses() self.assertEqual(1, len(qs)) self.assertIn(file, qs) self.assertNotIn(directory, qs) self.assertNotIn(symlink, qs) qs = CodebaseResource.objects.has_no_licenses() self.assertEqual(2, len(qs)) self.assertNotIn(file, qs) self.assertIn(directory, qs) self.assertIn(symlink, qs) self.assertEqual(0, CodebaseResource.objects.in_package().count()) self.assertEqual(3, CodebaseResource.objects.not_in_package().count()) DiscoveredPackage.create_for_resource(package_data1, file) self.assertEqual(1, CodebaseResource.objects.in_package().count()) self.assertEqual(2, CodebaseResource.objects.not_in_package().count())