def setUp(self):
# create importer object
self.importer = TVImporter()
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleCategoryData()
self.insertSampleLicenseData()
self.insertSampleConversionData()
self.insertSampleSubprojectData()
self.insertSampleScanData()
# build sample file data list
self.fd1 = createFD("/tmp/f1", "DoAnything", md5="abcdef")
self.fd2 = createFD("/tmp/f2", "DoAnythingNoncommercial", sha256="abcdef")
self.fd3 = createFD("/tmp/f3", "HarshEULA", sha1="abcdef")
self.fd4 = createFD("/tmp/f4", "HarshEULA")
self.fdList = [self.fd1, self.fd2, self.fd3, self.fd4]
# not in fdList by default
self.fd5 = createFD("/tmp/badLicense", "UnknownLicense")
self.fd6 = createFD("/tmp/badLic2", "SecondUnknownLic")
self.fdConvert = createFD("/tmp/needsConvert", "293")
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# create reporter
self.reporter = JSONReporter(db=self.db)
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleConfigData()
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleCategoryData()
self.insertSampleLicenseData()
self.insertSampleSubprojectData()
self.insertSampleScanData()
self.insertSampleFileData()
def test_can_create_new_database(self):
# don't use db from setUp(); create new in-memory DB from scratch
dbnew = ProjectDB()
dbnew.createDB(":memory:")
self.assertTrue(dbnew.isOpened())
self.assertFalse(dbnew.isInitialized())
dbnew.closeDB()
def test_that_closed_db_reports_as_uninitialized(self):
# don't use db from setUp(); create new in-memory DB from scratch
dbnew = ProjectDB()
dbnew.createDB(":memory:")
# and then close it
dbnew.closeDB()
self.assertFalse(dbnew.isInitialized())
self.assertIsNone(dbnew.session)
self.assertIsNone(dbnew.engine)
def setUp(self):
# fake SLM config
mainconfig_json = """{
"projects": [
{ "name": "frotz", "desc": "The FROTZ Project" },
{ "name": "rezrov", "desc": "The REZROV Project" },
{ "name": "gnusto", "desc": "The GNUSTO Project" }
]
}"""
self.mainconfig = SLMConfig()
self.mainconfig.loadConfig(mainconfig_json)
# SLM manager for testing
self.manager = SLMManager(config=self.mainconfig,
root="/tmp/fake/whatever")
# fake DBs for projects
self.frotz_db = ProjectDB()
self.frotz_db.createDB(":memory:")
self.frotz_db.initializeDBTables()
self.rezrov_db = ProjectDB()
self.rezrov_db.createDB(":memory:")
self.rezrov_db.initializeDBTables()
self.gnusto_db = ProjectDB()
self.gnusto_db.createDB(":memory:")
self.gnusto_db.initializeDBTables()
# insert sample data
self.insertSampleSubprojectData()
self.insertSampleScans()
def test_cannot_open_some_random_file_as_db(self):
# create in temporary directory on disk, so we can re-open it
# (testfixtures will wipe out the directory at end of test)
with TempDirectory() as td:
fakeDBPath = os.path.join(td.path, "tmp.txt")
with open(fakeDBPath, "w") as f:
f.write("some random text")
dbnew = ProjectDB()
with self.assertRaises(ProjectDBConfigError):
dbnew.openDB(fakeDBPath)
self.assertFalse(dbnew.isOpened())
self.assertFalse(dbnew.isInitialized())
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
class DBScanUnitTestSuite(unittest.TestCase):
"""spdxLicenseManager unit test suite for scan metadata in DB."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleSubprojectData()
self.insertSampleScanData()
def tearDown(self):
self.db.closeDB()
self.db = None
def insertSampleSubprojectData(self):
subprojects = [
Subproject(_id=1, name="sub1", desc="subproject 1"),
Subproject(_id=2, name="subX", desc="subproject XYZ"),
Subproject(_id=3, name="subC", desc="subproject B"),
]
self.db.session.bulk_save_objects(subprojects)
self.db.session.commit()
def insertSampleScanData(self):
scans = [
Scan(_id=1,
subproject_id=2,
scan_dt=datetime.date(2017, 1, 10),
desc="XYZ initial scan"),
Scan(_id=2,
subproject_id=1,
scan_dt=datetime.date(2017, 1, 3),
desc="1 initial scan"),
Scan(_id=3,
subproject_id=2,
scan_dt=datetime.date(2017, 2, 10),
desc="XYZ 2017-02 monthly scan"),
Scan(_id=4,
subproject_id=2,
scan_dt=datetime.date(2017, 2, 17),
desc="XYZ 2017-02 rescan"),
]
self.db.session.bulk_save_objects(scans)
self.db.session.commit()
##### Test cases below
def test_can_retrieve_all_scans(self):
scans = self.db.getScansAll()
self.assertIsInstance(scans, list)
self.assertEqual(len(scans), 4)
# will sort by ID
self.assertEqual(scans[0]._id, 1)
self.assertEqual(scans[0].subproject.name, "subX")
self.assertEqual(scans[1]._id, 2)
self.assertEqual(scans[1].subproject.name, "sub1")
self.assertEqual(scans[2]._id, 3)
self.assertEqual(scans[2].subproject.name, "subX")
self.assertEqual(scans[3]._id, 4)
self.assertEqual(scans[3].subproject.name, "subX")
def test_can_retrieve_scans_in_just_one_subproject(self):
scans = self.db.getScansFiltered(subproject="subX")
self.assertIsInstance(scans, list)
self.assertEqual(len(scans), 3)
# will sort by scan ID
self.assertEqual(scans[0]._id, 1)
self.assertEqual(scans[0].desc, "XYZ initial scan")
self.assertEqual(scans[1]._id, 3)
self.assertEqual(scans[1].desc, "XYZ 2017-02 monthly scan")
self.assertEqual(scans[2]._id, 4)
self.assertEqual(scans[2].desc, "XYZ 2017-02 rescan")
def test_cannot_retrieve_scans_in_subproject_that_does_not_exist(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getScansFiltered(subproject="invalid")
def test_can_retrieve_scans_by_subproject_and_month(self):
scans = self.db.getScansFiltered(subproject="subX",
month_tuple=(2017, 2))
self.assertIsInstance(scans, list)
self.assertEqual(len(scans), 2)
# will sort by scan ID
self.assertEqual(scans[0]._id, 3)
self.assertEqual(scans[0].desc, "XYZ 2017-02 monthly scan")
self.assertEqual(scans[1]._id, 4)
self.assertEqual(scans[1].desc, "XYZ 2017-02 rescan")
def test_cannot_retrieve_scans_in_subproject_with_integer_month(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getScansFiltered(subproject="subX", month_tuple=2)
def test_cannot_retrieve_scans_in_subproject_with_invalid_month_tuple(
self):
with self.assertRaises(ProjectDBQueryError):
self.db.getScansFiltered(subproject="subX",
month_tuple=("hi", "there"))
def test_retrieve_scan_with_unknown_month_returns_empty_list(self):
scans = self.db.getScansFiltered(subproject="subX",
month_tuple=(2017, 3))
self.assertIsInstance(scans, list)
self.assertEqual(scans, [])
def test_can_retrieve_scans_by_month_without_subproject(self):
scans = self.db.getScansFiltered(month_tuple=(2017, 1))
self.assertIsInstance(scans, list)
self.assertEqual(len(scans), 2)
# will sort by scan ID
self.assertEqual(scans[0]._id, 1)
self.assertEqual(scans[0].desc, "XYZ initial scan")
self.assertEqual(scans[1]._id, 2)
self.assertEqual(scans[1].desc, "1 initial scan")
def test_cannot_retrieve_filtered_scans_without_subproject_or_month(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getScansFiltered()
def test_cannot_retrieve_filtered_scans_with_positional_args(self):
with self.assertRaises(TypeError):
self.db.getScansFiltered("subX")
with self.assertRaises(TypeError):
self.db.getScansFiltered("subX", (2017, 1))
with self.assertRaises(TypeError):
self.db.getScansFiltered((2017, 1))
def test_can_retrieve_one_scan_by_id(self):
scan = self.db.getScan(_id=2)
self.assertEqual(scan.desc, "1 initial scan")
def test_returns_none_if_scan_not_found_by_id(self):
scan = self.db.getScan(_id=17)
self.assertIsNone(scan)
def test_can_add_and_retrieve_scans(self):
scan_id = self.db.addScan(subproject="subX",
scan_dt_str="2017-03-05",
desc="XYZ 2017-03 monthly scan")
# confirm that we now have five scans
scans = self.db.getScansAll()
self.assertEqual(len(scans), 5)
self.assertEqual(scan_id, 5)
# and confirm that we can retrieve this one by id
scan = self.db.getScan(_id=5)
self.assertEqual(scan.desc, "XYZ 2017-03 monthly scan")
self.assertEqual(scan.subproject.name, "subX")
def test_can_start_adding_but_rollback_scan(self):
scan_id = self.db.addScan(subproject="subX",
scan_dt_str="2011-01-01",
desc="will rollback",
commit=False)
self.db.rollback()
# confirm that we still only have four scans
scans = self.db.getScansAll()
self.assertEqual(len(scans), 4)
# and confirm that this scan ID doesn't exist in database
scan = self.db.getScan(_id=scan_id)
self.assertIsNone(scan)
def test_can_start_adding_and_then_commit_scans(self):
s1_id = self.db.addScan(subproject="subX",
scan_dt_str="2011-01-01",
desc="s1",
commit=False)
s2_id = self.db.addScan(subproject="subX",
scan_dt_str="2012-02-02",
desc="s2",
commit=False)
self.db.commit()
# confirm that we now have six scans
scans = self.db.getScansAll()
self.assertEqual(len(scans), 6)
def test_cannot_add_scan_without_subproject(self):
with self.assertRaises(TypeError):
self.db.addScan(scan_dt_str="2011-01-01", desc="oops")
# confirm it wasn't added either
scan = self.db.getScan(_id=5)
self.assertIsNone(scan)
def test_cannot_add_scan_without_scan_date_string(self):
with self.assertRaises(TypeError):
self.db.addScan(subproject="subX", desc="oops")
# confirm it wasn't added either
scan = self.db.getScan(_id=5)
self.assertIsNone(scan)
def test_can_add_scan_with_duplicate_desc(self):
scan_id = self.db.addScan(subproject="sub1",
scan_dt_str="2017-02-02",
desc="1 initial scan")
# confirm that we now have five scans and desc matches
scans = self.db.getScansAll()
self.assertEqual(len(scans), 5)
self.assertEqual(scan_id, 5)
scan = self.db.getScan(_id=5)
self.assertEqual(scan.desc, "1 initial scan")
def test_can_add_scan_with_duplicate_scan_date(self):
scan_id = self.db.addScan(subproject="sub1",
scan_dt_str="2017-01-03",
desc="1 rescan")
# confirm that we now have five scans and desc matches
scans = self.db.getScansAll()
self.assertEqual(len(scans), 5)
self.assertEqual(scan_id, 5)
scan = self.db.getScan(_id=5)
self.assertEqual(scan.desc, "1 rescan")
def test_cannot_create_new_database_if_file_already_exists(
self, os_exists):
dbnew = ProjectDB()
with self.assertRaises(ProjectDBConfigError):
dbnew.createDB("/tmp/fake/existing.db")
class DBFileUnitTestSuite(unittest.TestCase):
"""spdxLicenseManager unit test suite for scan metadata in DB."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleCategoryData()
self.insertSampleLicenseData()
self.insertSampleSubprojectData()
self.insertSampleScanData()
self.insertSampleFileData()
def tearDown(self):
self.db.closeDB()
self.db = None
def insertSampleCategoryData(self):
categories = [
Category(_id=1, name="a category", order=3),
Category(_id=2, name="cat", order=2),
Category(_id=3, name="blah category", order=1),
]
self.db.session.bulk_save_objects(categories)
self.db.session.commit()
def insertSampleLicenseData(self):
licenses = [
License(_id=1, name="DoAnything", category_id=1),
License(_id=2, name="HarshEULA", category_id=2),
License(_id=3, name="293PageEULA", category_id=3),
License(_id=4, name="DoAnythingNoncommercial", category_id=1),
]
self.db.session.bulk_save_objects(licenses)
self.db.session.commit()
def insertSampleSubprojectData(self):
subprojects = [
Subproject(_id=1, name="sub1", desc="subproject 1"),
Subproject(_id=2, name="subX", desc="subproject XYZ"),
Subproject(_id=3, name="subC", desc="subproject B"),
]
self.db.session.bulk_save_objects(subprojects)
self.db.session.commit()
def insertSampleScanData(self):
scans = [
Scan(_id=1,
subproject_id=2,
scan_dt=datetime.date(2017, 1, 10),
desc="XYZ initial scan"),
Scan(_id=2,
subproject_id=1,
scan_dt=datetime.date(2017, 1, 3),
desc="1 initial scan"),
Scan(_id=3,
subproject_id=2,
scan_dt=datetime.date(2017, 2, 10),
desc="XYZ 2017-02 monthly scan"),
Scan(_id=4,
subproject_id=2,
scan_dt=datetime.date(2017, 2, 17),
desc="XYZ 2017-02 rescan"),
]
self.db.session.bulk_save_objects(scans)
self.db.session.commit()
def insertSampleFileData(self):
files = [
File(_id=1,
scan_id=1,
path="/fileC.c",
license_id=1,
sha1="aabbcc",
md5="ddeeff",
sha256="aaccee"),
File(_id=2,
scan_id=1,
path="/fileA.c",
license_id=1,
sha1="112233",
md5="445566",
sha256="778899"),
File(_id=3,
scan_id=1,
path="/fileB.c",
license_id=2,
sha1=None,
md5=None,
sha256=None),
File(_id=4,
scan_id=1,
path="/dir/fileA.c",
license_id=4,
sha1="123456",
md5="789012",
sha256="345678"),
]
self.db.session.bulk_save_objects(files)
self.db.session.commit()
##### Test cases below
def test_can_retrieve_files_in_one_scan(self):
files = self.db.getFiles(scan_id=1)
self.assertIsInstance(files, list)
self.assertEqual(len(files), 4)
# will sort by file path
self.assertEqual(files[0]._id, 4)
self.assertEqual(files[0].path, "/dir/fileA.c")
self.assertEqual(files[1]._id, 2)
self.assertEqual(files[1].path, "/fileA.c")
self.assertEqual(files[2]._id, 3)
self.assertEqual(files[2].path, "/fileB.c")
self.assertEqual(files[3]._id, 1)
self.assertEqual(files[3].path, "/fileC.c")
def test_cannot_retrieve_files_in_scan_that_does_not_exist(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getFiles(scan_id=17)
def test_returns_empty_list_if_no_files_in_known_scan(self):
files = self.db.getFiles(scan_id=4)
self.assertEqual(files, [])
def test_can_get_file_by_id(self):
file = self.db.getFile(_id=3)
self.assertEqual(file.path, "/fileB.c")
self.assertEqual(file.license.name, "HarshEULA")
def test_can_get_file_by_scan_and_path(self):
file = self.db.getFile(scan_id=1, path="/fileB.c")
self.assertEqual(file._id, 3)
self.assertEqual(file.license.name, "HarshEULA")
def test_cannot_get_file_by_id_with_scan_or_path(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getFile(_id=3, scan_id=1)
with self.assertRaises(ProjectDBQueryError):
self.db.getFile(_id=3, path="/fileB.c")
def test_cannot_get_file_with_no_id_or_scan_or_path(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getFile()
def test_cannot_get_file_with_only_one_of_scan_or_path(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getFile(scan_id=1)
with self.assertRaises(ProjectDBQueryError):
self.db.getFile(path="/fileB.c")
def test_returns_none_if_file_not_found_by_id(self):
file = self.db.getFile(_id=17)
self.assertIsNone(file)
def test_returns_none_if_file_not_found_by_scan_plus_path(self):
file = self.db.getFile(scan_id=1, path="/nope")
self.assertIsNone(file)
file = self.db.getFile(scan_id=6, path="/fileB.c")
self.assertIsNone(file)
def test_can_add_and_retrieve_files(self):
self.db.addFile(scan_id=1,
path="/file17.py",
license_id=3,
sha1=None,
md5=None,
sha256=None)
self.db.addFile(scan_id=1,
path="/file13.py",
license_id=2,
sha1=None,
md5=None,
sha256=None)
file_id = self.db.addFile(scan_id=1,
path="/dir5/file128.py",
license_id=4,
sha1="123456",
md5="789012",
sha256="345678")
# confirm that we now have seven files in this scan
files = self.db.getFiles(scan_id=1)
self.assertEqual(len(files), 7)
self.assertEqual(file_id, 7)
# and confirm that we can retrieve this one by id
file = self.db.getFile(_id=7)
self.assertEqual(file.path, "/dir5/file128.py")
self.assertEqual(file.license.name, "DoAnythingNoncommercial")
def test_can_start_adding_but_rollback_file(self):
file_id = self.db.addFile(scan_id=1,
path="/will_rollback",
license_id=3,
sha1=None,
md5=None,
sha256=None,
commit=False)
self.db.rollback()
# confirm that we still only have four files
files = self.db.getFiles(scan_id=1)
self.assertEqual(len(files), 4)
# and confirm that this file ID doesn't exist in database
file = self.db.getFile(_id=file_id)
self.assertIsNone(file)
def test_can_start_adding_and_then_commit_files(self):
f1_id = self.db.addFile(scan_id=1,
path="/f1",
license_id=1,
sha1=None,
md5=None,
sha256=None,
commit=False)
f2_id = self.db.addFile(scan_id=1,
path="/f2",
license_id=1,
sha1=None,
md5=None,
sha256=None,
commit=False)
self.db.commit()
# confirm that we now have six files
files = self.db.getFiles(scan_id=1)
self.assertEqual(len(files), 6)
def test_can_bulk_add_and_retrieve_files(self):
bulkfiles = [
("/file17.py", 3, None, None, None),
("/file13.py", 2, None, None, None),
("/dir5/file128.py", 4, "123456", "789012", "345678"),
]
self.db.addBulkFiles(scan_id=1, file_tuples=bulkfiles)
# confirm that we now have seven files in this scan
files = self.db.getFiles(scan_id=1)
self.assertEqual(len(files), 7)
# and confirm that we can retrieve last one by id
file = self.db.getFile(_id=7)
self.assertEqual(file.path, "/dir5/file128.py")
self.assertEqual(file.license.name, "DoAnythingNoncommercial")
def test_can_start_bulk_adding_files_but_rollback(self):
bulkfiles = [
("/file17.py", 3, None, None, None),
("/file13.py", 2, None, None, None),
("/dir5/file128.py", 4, "123456", "789012", "345678"),
]
self.db.addBulkFiles(scan_id=1, file_tuples=bulkfiles, commit=False)
self.db.rollback()
# confirm that we still only have four files
files = self.db.getFiles(scan_id=1)
self.assertEqual(len(files), 4)
# and confirm that this file ID doesn't exist in database
file = self.db.getFile(_id=7)
self.assertIsNone(file)
def test_can_start_bulk_adding_and_then_commit_files(self):
bulkfiles = [
("/file17.py", 3, None, None, None),
("/file13.py", 2, None, None, None),
("/dir5/file128.py", 4, "123456", "789012", "345678"),
]
self.db.addBulkFiles(scan_id=1, file_tuples=bulkfiles, commit=False)
self.db.commit()
# confirm that we now have seven files
files = self.db.getFiles(scan_id=1)
self.assertEqual(len(files), 7)
class DBLicenseUnitTestSuite(unittest.TestCase):
"""spdxLicenseManager unit test suite for license data in DB."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleCategoryData()
self.insertSampleLicenseData()
def tearDown(self):
self.db.closeDB()
self.db = None
def insertSampleCategoryData(self):
categories = [
Category(_id=1, name="a category", order=3),
Category(_id=2, name="cat", order=2),
Category(_id=3, name="blah category", order=1),
]
self.db.session.bulk_save_objects(categories)
self.db.session.commit()
def insertSampleLicenseData(self):
licenses = [
License(_id=1, name="DoAnything", category_id=1),
License(_id=2, name="HarshEULA", category_id=2),
License(_id=3, name="293PageEULA", category_id=3),
License(_id=4, name="DoAnythingNoncommercial", category_id=1),
]
self.db.session.bulk_save_objects(licenses)
self.db.session.commit()
##### Test cases below
def test_can_retrieve_all_license_names(self):
licenses = self.db.getLicensesAll()
self.assertIsInstance(licenses, list)
self.assertEqual(len(licenses), 4)
# will sort alphabetically
self.assertEqual(licenses[0]._id, 3)
self.assertEqual(licenses[0].name, "293PageEULA")
def test_can_retrieve_all_license_names_by_category(self):
cat_lics = self.db.getLicensesAllByCategory()
self.assertIsInstance(cat_lics, list)
self.assertEqual(len(cat_lics), 4)
# returns a flat list of tuples, of form [(cat name, lic name)]
# categories should be ordered by cat order
# and within each category, licenses should be ordered alphabetically
self.assertEqual(cat_lics[0][0], "blah category")
self.assertEqual(cat_lics[0][1], "293PageEULA")
self.assertEqual(cat_lics[1][0], "cat")
self.assertEqual(cat_lics[1][1], "HarshEULA")
self.assertEqual(cat_lics[2][0], "a category")
self.assertEqual(cat_lics[2][1], "DoAnything")
self.assertEqual(cat_lics[3][0], "a category")
self.assertEqual(cat_lics[3][1], "DoAnythingNoncommercial")
def test_can_retrieve_licenses_in_just_one_category(self):
licenses = self.db.getLicensesInCategory(category="a category")
self.assertIsInstance(licenses, list)
self.assertEqual(len(licenses), 2)
# will sort alphabetically
self.assertEqual(licenses[0]._id, 1)
self.assertEqual(licenses[0].name, "DoAnything")
self.assertEqual(licenses[1]._id, 4)
self.assertEqual(licenses[1].name, "DoAnythingNoncommercial")
def test_cannot_retrieve_license_in_category_that_does_not_exist(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getLicensesInCategory(category="invalid")
def test_can_retrieve_one_license_by_id(self):
license = self.db.getLicense(_id=2)
self.assertEqual(license.name, "HarshEULA")
def test_can_retrieve_one_license_by_name(self):
license = self.db.getLicense(name="DoAnything")
self.assertEqual(license._id, 1)
def test_cannot_retrieve_license_by_both_name_and_id(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getLicense(_id=3, name="293PageEULA")
def test_cannot_retrieve_license_without_either_name_or_id(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getLicense()
def test_cannot_retrieve_license_with_positional_args(self):
with self.assertRaises(TypeError):
self.db.getLicense("DoAnything")
def test_returns_none_if_license_not_found_by_id(self):
license = self.db.getLicense(_id=17)
self.assertIsNone(license)
def test_returns_none_if_license_not_found_by_name(self):
license = self.db.getLicense(name="noSuchLicense")
self.assertIsNone(license)
def test_can_retrieve_multiple_licenses_by_name(self):
licenses = [
"293PageEULA",
"DoAnything",
"UnknownLicense",
]
ldict = self.db.getMultipleLicenses(licenses)
self.assertEqual(ldict["293PageEULA"], 3)
self.assertEqual(ldict["DoAnything"], 1)
self.assertIsNone(ldict["UnknownLicense"])
def test_can_add_and_retrieve_licenses(self):
license_id = self.db.addLicense(name="SomeOtherEULA",
category="blah category")
# confirm that we now have five licenses
licenses = self.db.getLicensesAll()
self.assertEqual(len(licenses), 5)
# and confirm that we can retrieve this one by name
license = self.db.getLicense(name="SomeOtherEULA")
self.assertEqual(license._id, 5)
self.assertEqual(license.category_id, 3)
self.assertIsNotNone(license.category)
self.assertEqual(license.category.name, "blah category")
# and confirm that we can retrieve this one by id
license = self.db.getLicense(_id=5)
self.assertEqual(license.name, "SomeOtherEULA")
self.assertEqual(license.category_id, 3)
def test_can_start_adding_but_rollback_license(self):
license_id = self.db.addLicense(name="will rollback",
category="blah category", commit=False)
self.db.rollback()
# confirm that we still only have four licenses
licenses = self.db.getLicensesAll()
self.assertEqual(len(licenses), 4)
# and confirm that this license ID doesn't exist in database
license = self.db.getLicense(_id=license_id)
self.assertIsNone(license)
def test_can_start_adding_and_then_commit_licenses(self):
l1_id = self.db.addLicense(name="newl1", category="cat", commit=False)
l2_id = self.db.addLicense(name="newl2", category="cat", commit=False)
self.db.commit()
# confirm that we now have six licenses
licenses = self.db.getLicensesAll()
self.assertEqual(len(licenses), 6)
def test_cannot_add_license_without_category(self):
with self.assertRaises(TypeError):
self.db.addLicense(name="oops")
# confirm it wasn't added either
license = self.db.getLicense(name="oops")
self.assertIsNone(license)
def test_cannot_add_license_without_existing_category(self):
with self.assertRaises(ProjectDBInsertError):
self.db.addLicense(name="oops", category="noSuchCategory")
# confirm it wasn't added either
license = self.db.getLicense(name="oops")
self.assertIsNone(license)
def test_cannot_add_license_with_duplicate_name(self):
with self.assertRaises(ProjectDBInsertError):
self.db.addLicense(name="DoAnything", category="cat")
def test_can_edit_license_name(self):
self.db.changeLicenseName(name="293PageEULA", newName="432PageEULA")
license = self.db.getLicense(name="432PageEULA")
self.assertEqual(license._id, 3)
def test_cannot_edit_license_name_that_does_not_exist(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeLicenseName(name="invalid", newName="this will fail")
def test_cannot_change_license_name_to_existing_name(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeLicenseName(name="293PageEULA", newName="DoAnything")
def test_can_edit_license_category(self):
self.db.changeLicenseCategory(name="293PageEULA", newCat="a category")
license = self.db.getLicense(name="293PageEULA")
self.assertEqual(license.category.name, "a category")
def test_cannot_change_license_to_category_that_does_not_exist(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeLicenseCategory(name="293PageEULA", newCat="invalid")
def test_can_get_max_license_id(self):
lic_id = self.db.getLicenseMaxID()
self.assertEqual(4, lic_id)
class SLMManagerTestSuite(unittest.TestCase):
"""spdxLicenseManager SLM manager unit test suite."""
def setUp(self):
# fake SLM config
mainconfig_json = """{
"projects": [
{ "name": "frotz", "desc": "The FROTZ Project" },
{ "name": "rezrov", "desc": "The REZROV Project" },
{ "name": "gnusto", "desc": "The GNUSTO Project" }
]
}"""
self.mainconfig = SLMConfig()
self.mainconfig.loadConfig(mainconfig_json)
# SLM manager for testing
self.manager = SLMManager(config=self.mainconfig,
root="/tmp/fake/whatever")
# fake DBs for projects
self.frotz_db = ProjectDB()
self.frotz_db.createDB(":memory:")
self.frotz_db.initializeDBTables()
self.rezrov_db = ProjectDB()
self.rezrov_db.createDB(":memory:")
self.rezrov_db.initializeDBTables()
self.gnusto_db = ProjectDB()
self.gnusto_db.createDB(":memory:")
self.gnusto_db.initializeDBTables()
# insert sample data
self.insertSampleSubprojectData()
self.insertSampleScans()
def tearDown(self):
pass
def insertSampleSubprojectData(self):
# for frotz
frotz_subprojects = [
Subproject(_id=1,
name="f1",
spdx_search="f1",
desc="subproject f1"),
Subproject(_id=2,
name="f2",
spdx_search="fX2",
desc="subproject fX2"),
Subproject(_id=3, name="f3", spdx_search="f3", desc="f3"),
]
self.frotz_db.session.bulk_save_objects(frotz_subprojects)
self.frotz_db.session.commit()
# for gnusto
# note that IDs restart b/c projects keep separate databases
gnusto_subprojects = [
Subproject(_id=1,
name="g1",
spdx_search="g1",
desc="subproject g1"),
Subproject(_id=2, name="g2", spdx_search="gAX2", desc="gAX2"),
]
self.gnusto_db.session.bulk_save_objects(gnusto_subprojects)
self.gnusto_db.session.commit()
# none for rezrov
def insertSampleScans(self):
# for frotz
frotz_scans = [
Scan(_id=1,
subproject_id=2,
scan_dt=datetime.date(2017, 1, 10),
desc="fX2 initial scan"),
Scan(_id=2,
subproject_id=1,
scan_dt=datetime.date(2017, 2, 3),
desc="f1 initial scan"),
Scan(_id=3,
subproject_id=2,
scan_dt=datetime.date(2017, 2, 10),
desc="fX2 2017-02 monthly scan"),
Scan(_id=4,
subproject_id=2,
scan_dt=datetime.date(2017, 2, 17),
desc="fX2 2017-02 rescan"),
# none for subproject 3
]
self.frotz_db.session.bulk_save_objects(frotz_scans)
self.frotz_db.session.commit()
# for gnusto
# note that IDs restart b/c projects keep separate databases
gnusto_scans = [
# none for subproject 1
Scan(_id=1,
subproject_id=2,
scan_dt=datetime.date(2017, 1, 10),
desc="gAX2 initial scan"),
Scan(_id=2,
subproject_id=2,
scan_dt=datetime.date(2017, 2, 3),
desc="gAX2 2017-02 monthly scan"),
]
self.gnusto_db.session.bulk_save_objects(gnusto_scans)
self.gnusto_db.session.commit()
# none for rezrov
##### Test cases below
def test_can_get_list_of_all_projects(self):
projects = self.manager.getProjects()
self.assertIsInstance(projects, list)
self.assertEqual(len(projects), 3)
self.assertIn("frotz", projects)
self.assertIn("gnusto", projects)
self.assertIn("rezrov", projects)
def test_can_get_project_directory_path(self):
frotz_dir = self.manager.getProjectDir("frotz")
self.assertEqual(frotz_dir, "/tmp/fake/whatever/projects/frotz")
def test_can_get_project_database_path(self):
frotz_db_path = self.manager.getProjectDBPath("frotz")
self.assertEqual(frotz_db_path,
"/tmp/fake/whatever/projects/frotz/frotz.db")
def test_can_get_project_reports_directory_path(self):
frotz_reports_dir = self.manager.getProjectReportsDir("frotz")
self.assertEqual(frotz_reports_dir,
"/tmp/fake/whatever/projects/frotz/reports")
def test_can_get_subprojects_directory_path(self):
f2_dir = self.manager.getSubprojectDir("frotz", "f2")
self.assertEqual(f2_dir,
"/tmp/fake/whatever/projects/frotz/subprojects/f2")
def test_can_get_subprojects_reports_directory_path(self):
f2_reports_dir = self.manager.getSubprojectReportsDir("frotz", "f2")
self.assertEqual(
f2_reports_dir,
"/tmp/fake/whatever/projects/frotz/subprojects/f2/reports")
def test_can_get_subprojects_spdx_directory_path(self):
f2_spdx_dir = self.manager.getSubprojectSPDXDir("frotz", "f2")
self.assertEqual(
f2_spdx_dir,
"/tmp/fake/whatever/projects/frotz/subprojects/f2/spdx")
def test_can_get_subproject_scan_dates_for_month_with_one_scan(self):
dates = self.manager.getScanDates(self.frotz_db, "f1", "2017-02")
self.assertIsInstance(dates, list)
self.assertEqual(len(dates), 1)
self.assertEqual(dates[0], "2017-02-03")
def test_can_get_subproject_scan_dates_for_month_with_two_scans(self):
dates = self.manager.getScanDates(self.frotz_db, "f2", "2017-02")
self.assertIsInstance(dates, list)
self.assertEqual(len(dates), 2)
self.assertEqual(dates[0], "2017-02-10")
self.assertEqual(dates[1], "2017-02-17")
def test_getting_scan_dates_without_scans_returns_empty_list(self):
dates = self.manager.getScanDates(self.frotz_db, "f3", "2017-02")
self.assertIsInstance(dates, list)
self.assertEqual(len(dates), 0)
def test_getting_scan_dates_without_choosing_subproject_raises_exception(
self):
with self.assertRaises(SLMManagerError):
self.manager.getScanDates(self.frotz_db, None, "2017-02")
def test_getting_scan_dates_without_any_subprojects_raises_exception(self):
with self.assertRaises(SLMManagerError):
self.manager.getScanDates(self.rezrov_db, None, "2017-02")
def test_getting_scan_dates_with_nonexistent_subproject_raises_exception(
self):
with self.assertRaises(SLMManagerError):
self.manager.getScanDates(self.frotz_db, "oops", "2017-02")
def test_can_check_if_any_scan_is_present_for_given_month(self):
retval = self.manager.isScanForMonth(self.frotz_db, "f1", "2017-02")
self.assertTrue(retval)
retval = self.manager.isScanForMonth(self.frotz_db, "f2", "2017-02")
self.assertTrue(retval)
retval = self.manager.isScanForMonth(self.frotz_db, "f3", "2017-02")
self.assertFalse(retval)
def test_can_get_spdx_possible_filenames_for_scan_dt(self):
spdx_paths = self.manager.getSPDXExpectedPaths(self.frotz_db, "frotz",
"f1", "2017-02")
self.assertIsInstance(spdx_paths, list)
self.assertEqual(len(spdx_paths), 1)
self.assertEqual(
spdx_paths[0],
"/tmp/fake/whatever/projects/frotz/subprojects/f1/spdx/f1-2017-02-03.spdx"
)
spdx_paths2 = self.manager.getSPDXExpectedPaths(
self.frotz_db, "frotz", "f2", "2017-02")
self.assertIsInstance(spdx_paths2, list)
self.assertEqual(len(spdx_paths2), 2)
self.assertEqual(
spdx_paths2[0],
"/tmp/fake/whatever/projects/frotz/subprojects/f2/spdx/f2-2017-02-10.spdx"
)
self.assertEqual(
spdx_paths2[1],
"/tmp/fake/whatever/projects/frotz/subprojects/f2/spdx/f2-2017-02-17.spdx"
)
@mock.patch('slm.projectdb.os.path.isfile', return_value=True)
def test_can_check_if_spdx_file_is_present(self, os_isfile):
retval = self.manager.isSPDXForMonth(self.frotz_db, "frotz", "f1",
"2017-02")
self.assertTrue(retval)
@mock.patch('slm.projectdb.os.path.isfile', return_value=False)
def test_can_check_if_spdx_file_is_absent(self, os_isfile):
retval = self.manager.isSPDXForMonth(self.frotz_db, "frotz", "f1",
"2017-02")
self.assertFalse(retval)
@mock.patch('slm.projectdb.os.path.isfile', return_value=True)
def test_can_check_if_xlsx_report_is_present(self, os_isfile):
retval = self.manager.isXLSXForMonth("frotz", "f1", "2017-02")
self.assertTrue(retval)
@mock.patch('slm.projectdb.os.path.isfile', return_value=False)
def test_can_check_if_xlsx_report_is_absent(self, os_isfile):
retval = self.manager.isXLSXForMonth("frotz", "f1", "2017-02")
self.assertFalse(retval)
@mock.patch('slm.projectdb.os.path.isfile', return_value=True)
def test_can_check_if_json_report_is_present(self, os_isfile):
retval = self.manager.isJSONForMonth("frotz", "f1", "2017-02")
self.assertTrue(retval)
@mock.patch('slm.projectdb.os.path.isfile', return_value=False)
def test_can_check_if_json_report_is_absent(self, os_isfile):
retval = self.manager.isJSONForMonth("frotz", "f1", "2017-02")
self.assertFalse(retval)
def test_cannot_open_in_memory_db(self):
dbnew = ProjectDB()
with self.assertRaises(ProjectDBConfigError):
dbnew.openDB(":memory:")
class DBConfigUnitTestSuite(unittest.TestCase):
"""spdxLicenseManager unit test suite for configuration data in DB."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleConfigData()
def tearDown(self):
self.db.closeDB()
self.db = None
def insertSampleConfigData(self):
configs = [
Config(key="report-strip-licenseref", value="yes"),
]
self.db.session.bulk_save_objects(configs)
self.db.session.commit()
##### Test cases below
def test_can_retrieve_config_value_from_key(self):
value = self.db.getConfigValue(key="report-strip-licenseref")
self.assertEqual(value, "yes")
def test_cannot_retrieve_config_value_for_unknown_key(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getConfigValue(key="unknown_key")
def test_can_set_and_get_config_value_for_new_key_if_known(self):
self.db.setConfigValue(key="analyze-extensions-list", value="json")
value = self.db.getConfigValue(key="analyze-extensions-list")
self.assertEqual(value, "json")
def test_can_update_and_get_config_value_for_existing_key(self):
self.db.setConfigValue(key="report-strip-licenseref", value="no")
value = self.db.getConfigValue(key="report-strip-licenseref")
self.assertEqual(value, "no")
def test_cannot_update_config_value_for_reserved_keys(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.setConfigValue(key="magic", value="blah")
with self.assertRaises(ProjectDBUpdateError):
self.db.setConfigValue(key="initialized", value="blah")
def test_cannot_set_unknown_key(self):
with self.assertRaises(ProjectDBInsertError):
self.db.setConfigValue(key="new_key", value="123abc")
def test_can_get_all_configs(self):
configs = self.db.getConfigsAll()
self.assertIsInstance(configs, list)
self.assertEqual(len(configs), 3)
self.assertEqual(configs[0].key, "initialized")
self.assertEqual(configs[0].value, "yes")
self.assertEqual(configs[1].key, "magic")
self.assertEqual(configs[1].value, "spdxLicenseManager")
self.assertEqual(configs[2].key, "report-strip-licenseref")
self.assertEqual(configs[2].value, "yes")
def test_can_unset_config(self):
self.db.unsetConfigValue(key="report-strip-licenseref")
with self.assertRaises(ProjectDBQueryError):
self.db.getConfigValue(key="report-strip-licenseref")
def test_cannot_unset_config_for_internal_config(self):
with self.assertRaises(ProjectDBDeleteError):
self.db.unsetConfigValue(key="magic")
def test_cannot_unset_config_for_unknown_config(self):
with self.assertRaises(ProjectDBDeleteError):
self.db.unsetConfigValue(key="blah")
def test_cannot_unset_config_for_valid_config_that_is_not_set(self):
with self.assertRaises(ProjectDBDeleteError):
self.db.unsetConfigValue(key="analyze-extensions-list")
class ProjectDBUnitTestSuite(unittest.TestCase):
"""spdxLicenseManager unit test suite for DB initialization and lifecycle."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
def tearDown(self):
self.db.closeDB()
self.db = None
def test_can_create_new_database(self):
# don't use db from setUp(); create new in-memory DB from scratch
dbnew = ProjectDB()
dbnew.createDB(":memory:")
self.assertTrue(dbnew.isOpened())
self.assertFalse(dbnew.isInitialized())
dbnew.closeDB()
@mock.patch('slm.projectdb.os.path.exists', return_value=True)
def test_cannot_create_new_database_if_file_already_exists(
self, os_exists):
dbnew = ProjectDB()
with self.assertRaises(ProjectDBConfigError):
dbnew.createDB("/tmp/fake/existing.db")
def test_that_initialized_db_reports_as_initialized(self):
self.assertTrue(self.db.isInitialized())
def test_that_closed_db_reports_as_uninitialized(self):
# don't use db from setUp(); create new in-memory DB from scratch
dbnew = ProjectDB()
dbnew.createDB(":memory:")
# and then close it
dbnew.closeDB()
self.assertFalse(dbnew.isInitialized())
self.assertIsNone(dbnew.session)
self.assertIsNone(dbnew.engine)
def test_can_open_existing_db(self):
# create in temporary directory on disk, so we can re-open DB
# (testfixtures will wipe out the directory at end of test)
with TempDirectory() as td:
dbPath = os.path.join(td.path, "tmp.db")
dbnew = ProjectDB()
dbnew.createDB(dbPath)
dbnew.initializeDBTables()
dbnew.closeDB()
# and reopen it
dbnew.openDB(dbPath)
self.assertTrue(dbnew.isOpened())
self.assertTrue(dbnew.isInitialized())
dbnew.closeDB()
def test_cannot_open_in_memory_db(self):
dbnew = ProjectDB()
with self.assertRaises(ProjectDBConfigError):
dbnew.openDB(":memory:")
def test_open_db_fails_if_invalid_magic_number(self):
# create in temporary directory on disk, so we can re-open it
# (testfixtures will wipe out the directory at end of test)
with TempDirectory() as td:
dbPath = os.path.join(td.path, "tmp.db")
dbnew = ProjectDB()
dbnew.createDB(dbPath)
dbnew.initializeDBTables()
# set invalid magic number
query = dbnew.session.query(Config).filter(Config.key == "magic")
query.update({Config.value: "invalidMagic"})
dbnew.session.commit()
dbnew.closeDB()
# and reopen it
with self.assertRaises(ProjectDBConfigError):
dbnew.openDB(dbPath)
self.assertFalse(dbnew.isOpened())
self.assertFalse(dbnew.isInitialized())
def test_cannot_open_some_random_file_as_db(self):
# create in temporary directory on disk, so we can re-open it
# (testfixtures will wipe out the directory at end of test)
with TempDirectory() as td:
fakeDBPath = os.path.join(td.path, "tmp.txt")
with open(fakeDBPath, "w") as f:
f.write("some random text")
dbnew = ProjectDB()
with self.assertRaises(ProjectDBConfigError):
dbnew.openDB(fakeDBPath)
self.assertFalse(dbnew.isOpened())
self.assertFalse(dbnew.isInitialized())
class ReportJSONTestSuite(unittest.TestCase):
"""spdxLicenseManager JSON reporting unit test suite."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# create reporter
self.reporter = JSONReporter(db=self.db)
def tearDown(self):
pass
##### Test helpers to mimic results from analysis
def _buildCategories(self, results):
self.cat2 = Category()
self.cat2._id = 2
self.cat2.name = "catID2"
self.cat2.licenses = []
results.append(self.cat2)
self.cat1 = Category()
self.cat1._id = 1
self.cat1.name = "catID1"
self.cat1.licenses = []
results.append(self.cat1)
self.cat4 = Category()
self.cat4._id = 4
self.cat4.name = "No license found"
self.cat4.licenses = []
results.append(self.cat4)
def _buildLicenses(self, results):
# should sort alphabetically before lic1cat2 in reports
self.lic2cat2 = License()
self.lic2cat2._id = 2
self.lic2cat2.name = "another lic2cat2"
self.lic2cat2.files = []
self.cat2.licenses.append(self.lic2cat2)
self.lic1cat2 = License()
self.lic1cat2._id = 1
self.lic1cat2.name = "lic1cat2"
self.lic1cat2.files = []
self.cat2.licenses.append(self.lic1cat2)
self.lic5cat1 = License()
self.lic5cat1._id = 5
self.lic5cat1.name = "a license"
self.lic5cat1.files = []
self.cat1.licenses.append(self.lic5cat1)
# no license found option, for analysis outputs
self.noLicFound = License()
self.noLicFound._id = 6
self.noLicFound.name = "No license found"
self.noLicFound.files = []
self.cat4.licenses.append(self.noLicFound)
def _buildFiles(self, results):
# should sort alphabetically by path before f1
self.f2 = File()
self.f2._id = 2
self.f2.path = "/first/tmp/f2"
self.f2.scan_id = 1
self.f2.findings = {}
self.lic2cat2.files.append(self.f2)
self.f1 = File()
self.f1._id = 1
self.f1.path = "/tmp/f1"
self.f1.scan_id = 1
self.f1.findings = {}
self.lic2cat2.files.append(self.f1)
# should sort after f2 and f1 b/c license ID sorts after theirs
self.f3 = File()
self.f3._id = 3
self.f3.path = "/earliest/tmp/f3"
self.f3.scan_id = 1
self.f3.findings = {}
self.lic1cat2.files.append(self.f3)
self.f4 = File()
self.f4._id = 4
self.f4.path = "/tmp/f4"
self.f4.scan_id = 1
self.f4.findings = {}
self.lic1cat2.files.append(self.f4)
self.f5 = File()
self.f5._id = 5
self.f5.path = "/tmp/f5"
self.f5.scan_id = 1
self.f5.findings = {}
self.lic5cat1.files.append(self.f5)
self.f6 = File()
self.f6._id = 6
self.f6.path = "/tmp/f6.png"
self.f6.scan_id = 1
self.f6.findings = {
"extension": "yes",
}
self.noLicFound.files.append(self.f6)
# add some more files for testing findings
self.f7 = File()
self.f7._id = 7
self.f7.path = "/tmp/__init__.py"
self.f7.scan_id = 1
self.f7.findings = {
"emptyfile": "yes",
}
self.noLicFound.files.append(self.f7)
self.f8 = File()
self.f8._id = 8
self.f8.path = "/tmp/vendor/dep.py"
self.f8.scan_id = 1
self.f8.findings = {
"thirdparty": "yes",
}
self.noLicFound.files.append(self.f8)
self.f9 = File()
self.f9._id = 9
self.f9.path = "/tmp/code.py"
self.f9.scan_id = 1
self.f9.findings = {}
self.noLicFound.files.append(self.f9)
def _getAnalysisResults(self):
results = []
self._buildCategories(results)
self._buildLicenses(results)
self._buildFiles(results)
return results
##### Test cases below
def test_new_reporter_is_in_reset_state(self):
self.assertIsNone(self.reporter.results)
self.assertIsNone(self.reporter.rjs)
self.assertFalse(self.reporter.reportSaved)
self.assertEqual({}, self.reporter.kwConfig)
##### Reporter config function tests
def test_reporter_can_take_optional_config_params(self):
configDict = {
"report-pretty-print": "yes",
}
newReporter = JSONReporter(db=self.db, config=configDict)
self.assertEqual("yes", newReporter.kwConfig["report-pretty-print"])
def test_can_get_reporter_final_config(self):
self.db.setConfigValue(key="report-pretty-print", value="yes")
summary = self.reporter._getFinalConfigValue(key="report-pretty-print")
self.assertEqual(summary, "yes")
def test_can_override_db_config_in_reporter_final_config(self):
self.db.setConfigValue(key="report-pretty-print", value="yes")
newReporter = JSONReporter(db=self.db,
config={"report-pretty-print": "no"})
summary = newReporter._getFinalConfigValue(key="report-pretty-print")
self.assertEqual(summary, "no")
##### Reporter setup function tests
def test_can_set_results_and_create_empty_string_for_json(self):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.assertIsInstance(self.reporter.rjs, str)
self.assertEqual(self.reporter.rjs, "")
self.assertEqual(self.reporter.results, results)
##### Reporter custom JSON encoder tests
def test_encoder_creates_correct_values(self):
results = self._getAnalysisResults()
enc = self.reporter.SLMJSONEncoder()
# here's a file with findings
enc_f6 = enc.default(self.f6)
self.assertEqual(enc_f6["path"], "/tmp/f6.png")
self.assertEqual(enc_f6["_id"], 6)
self.assertEqual(enc_f6["findings"], {"extension": "yes"})
# here's a file with no findings
enc_f5 = enc.default(self.f5)
self.assertEqual(enc_f5["path"], "/tmp/f5")
self.assertEqual(enc_f5["_id"], 5)
with self.assertRaises(KeyError):
na = enc_f5["findings"]
# here's a license
enc_lic1 = enc.default(self.lic1cat2)
self.assertEqual(enc_lic1["name"], "lic1cat2")
self.assertEqual(enc_lic1["_id"], 1)
self.assertEqual(enc_lic1["numFiles"], 2)
self.assertEqual(enc_lic1["files"], self.lic1cat2.files)
# here's a category
enc_cat2 = enc.default(self.cat2)
self.assertEqual(enc_cat2["name"], "catID2")
self.assertEqual(enc_cat2["_id"], 2)
self.assertEqual(enc_cat2["numFiles"], 4)
self.assertEqual(enc_cat2["licenses"], self.cat2.licenses)
##### Reporter helpers
def test_helper_can_get_number_of_files_for_license(self):
results = self._getAnalysisResults()
enc = self.reporter.SLMJSONEncoder()
numFiles = enc._getNumFilesForLicense(self.lic1cat2)
self.assertEqual(numFiles, 2)
def test_helper_can_get_number_of_files_for_category(self):
results = self._getAnalysisResults()
enc = self.reporter.SLMJSONEncoder()
numFiles = enc._getNumFilesForCategory(self.cat2)
self.assertEqual(numFiles, 4)
##### Reporter generate function tests
def test_save_report_fails_if_results_not_ready(self):
with self.assertRaises(ReportNotReadyError):
self.reporter.save(path="/tmp/fake/whatever.json")
self.assertFalse(self.reporter.reportSaved)
@mock.patch('slm.reports.json.open')
@mock.patch('slm.reports.json.json.dump')
def test_save_calls_functions_to_save(self, dump_mock, open_mock):
outfile = "/tmp/newfile.json"
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.save(path=outfile)
dump_mock.assert_called()
args = dump_mock.call_args
self.assertEqual(args[0][0], results)
self.assertEqual(args[1], {"cls": JSONReporter.SLMJSONEncoder})
open_mock.assert_called_with(outfile, "w")
##### Reporter save function tests
@mock.patch('slm.reports.json.open')
@mock.patch('slm.reports.json.os.path.exists', return_value=True)
def test_save_check_raises_exception_if_file_already_exists(
self, os_exists, open_mock):
results = self._getAnalysisResults()
self.reporter.setResults(results)
with self.assertRaises(ReportFileError):
self.reporter._saveCheck(path="/tmp/fake/existing.json")
@mock.patch('slm.reports.json.open')
@mock.patch('slm.reports.json.os.path.exists', return_value=True)
@mock.patch('slm.reports.json.os.access', return_value=True)
def test_save_check_doesnt_raise_exception_if_replace_flag_set(
self, os_access, os_exists, open_mock):
results = self._getAnalysisResults()
self.reporter.setResults(results)
try:
self.reporter._saveCheck(path="/tmp/fake/existing.json",
replace=True)
except ReportFileError:
self.fail(
"ReportFileError raised even though replace flag is True")
@mock.patch('slm.reports.json.open')
@mock.patch('slm.reports.json.os.access', return_value=False)
def test_save_check_fails_if_no_write_permission_for_path(
self, os_access, open_mock):
results = self._getAnalysisResults()
self.reporter.setResults(results)
with self.assertRaises(ReportFileError):
self.reporter._saveCheck(path="/tmp/readonly/report.json")
def test_save_check_fails_if_report_not_ready_yet(self):
with self.assertRaises(ReportNotReadyError):
self.reporter._saveCheck(path="whatever")
@mock.patch('slm.reports.json.open')
@mock.patch('slm.reports.json.JSONReporter._saveCheck')
def test_save_calls_save_checker(self, check_mock, open_mock):
results = self._getAnalysisResults()
self.reporter.setResults(results)
path = "something"
replace = True
self.reporter.save(path=path, replace=replace)
check_mock.assert_called_with(path=path, replace=replace)
open_mock.assert_called_with(path, "w")
class ReportXlsxTestSuite(unittest.TestCase):
"""spdxLicenseManager Xlsx reporting unit test suite."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# create reporter
self.reporter = XlsxReporter(db=self.db)
def tearDown(self):
pass
##### Test helpers to mimic results from analysis
def _buildCategories(self, results):
self.cat2 = Category()
self.cat2._id = 2
self.cat2.name = "catID2"
self.cat2.order = 1
self.cat2.licensesSorted = OrderedDict()
self.cat2.hasFiles = True
results[2] = self.cat2
# no files, cat3 should not appear in reports
self.cat3 = Category()
self.cat3._id = 3
self.cat3.name = "catID3"
self.cat3.order = 2
self.cat3.licensesSorted = OrderedDict()
self.cat3.hasFiles = False
results[3] = self.cat3
self.cat1 = Category()
self.cat1._id = 1
self.cat1.name = "catID1"
self.cat1.order = 3
self.cat1.licensesSorted = OrderedDict()
self.cat1.hasFiles = True
results[1] = self.cat1
self.cat4 = Category()
self.cat4._id = 4
self.cat4.name = "No license found"
self.cat4.order = 4
self.cat4.licensesSorted = OrderedDict()
self.cat4.hasFiles = True
results[4] = self.cat4
def _buildLicenses(self, results):
# should sort alphabetically before lic1cat2 in reports
self.lic2cat2 = License()
self.lic2cat2._id = 2
self.lic2cat2.name = "another lic2cat2"
self.lic2cat2.category_id = 2
self.lic2cat2.filesSorted = OrderedDict()
self.lic2cat2.hasFiles = True
self.cat2.licensesSorted[2] = self.lic2cat2
self.lic1cat2 = License()
self.lic1cat2._id = 1
self.lic1cat2.name = "lic1cat2"
self.lic1cat2.category_id = 2
self.lic1cat2.filesSorted = OrderedDict()
self.lic1cat2.hasFiles = True
self.cat2.licensesSorted[1] = self.lic1cat2
# no files, lic3cat2 should not appear in reports
self.lic3cat2 = License()
self.lic3cat2._id = 3
self.lic3cat2.name = "nope lic3cat2"
self.lic3cat2.category_id = 2
self.lic3cat2.filesSorted = OrderedDict()
self.lic3cat2.hasFiles = False
self.cat2.licensesSorted[3] = self.lic3cat2
# no files, lic4cat3 should not appear in reports
self.lic4cat3 = License()
self.lic4cat3._id = 4
self.lic4cat3.name = "nope lic4cat3"
self.lic4cat3.category_id = 3
self.lic4cat3.filesSorted = OrderedDict()
self.lic4cat3.hasFiles = False
self.cat3.licensesSorted[4] = self.lic4cat3
self.lic5cat1 = License()
self.lic5cat1._id = 5
self.lic5cat1.name = "a license"
self.lic5cat1.category_id = 1
self.lic5cat1.filesSorted = OrderedDict()
self.lic5cat1.hasFiles = True
self.cat1.licensesSorted[5] = self.lic5cat1
# no license found option, for analysis outputs
self.noLicFound = License()
self.noLicFound._id = 6
self.noLicFound.name = "No license found"
self.noLicFound.category_id = 4
self.noLicFound.filesSorted = OrderedDict()
self.noLicFound.hasFiles = True
self.cat4.licensesSorted[6] = self.noLicFound
def _buildFiles(self, results):
# should sort alphabetically by path before f1
self.f2 = File()
self.f2._id = 2
self.f2.path = "/first/tmp/f2"
self.f2.scan_id = 1
self.f2.license_id = 2
self.f2.findings = {}
self.lic2cat2.filesSorted[2] = self.f2
self.f1 = File()
self.f1._id = 1
self.f1.path = "/tmp/f1"
self.f1.scan_id = 1
self.f1.license_id = 2
self.f1.findings = {}
self.lic2cat2.filesSorted[1] = self.f1
# should sort after f2 and f1 b/c license ID sorts after theirs
self.f3 = File()
self.f3._id = 3
self.f3.path = "/earliest/tmp/f3"
self.f3.scan_id = 1
self.f3.license_id = 1
self.f3.findings = {}
self.lic1cat2.filesSorted[3] = self.f3
self.f4 = File()
self.f4._id = 4
self.f4.path = "/tmp/f4"
self.f4.scan_id = 1
self.f4.license_id = 1
self.f4.findings = {}
self.lic1cat2.filesSorted[4] = self.f4
self.f5 = File()
self.f5._id = 5
self.f5.path = "/tmp/f5"
self.f5.scan_id = 1
self.f5.license_id = 5
self.f5.findings = {}
self.lic5cat1.filesSorted[5] = self.f5
self.f6 = File()
self.f6._id = 6
self.f6.path = "/tmp/f6.png"
self.f6.scan_id = 1
self.f6.license_id = 6
self.f6.findings = {
"extension": "yes",
}
self.noLicFound.filesSorted[6] = self.f6
# add some more files for testing findings
self.f7 = File()
self.f7._id = 7
self.f7.path = "/tmp/__init__.py"
self.f7.scan_id = 1
self.f7.license_id = 6
self.f7.findings = {
"emptyfile": "yes",
}
self.noLicFound.filesSorted[7] = self.f7
self.f8 = File()
self.f8._id = 8
self.f8.path = "/tmp/vendor/dep.py"
self.f8.scan_id = 1
self.f8.license_id = 6
self.f8.findings = {
"thirdparty": "yes",
}
self.noLicFound.filesSorted[8] = self.f8
self.f9 = File()
self.f9._id = 9
self.f9.path = "/tmp/code.py"
self.f9.scan_id = 1
self.f9.license_id = 6
self.f9.findings = {}
self.noLicFound.filesSorted[9] = self.f9
def _getAnalysisResults(self):
results = OrderedDict()
self._buildCategories(results)
self._buildLicenses(results)
self._buildFiles(results)
return results
##### Test cases below
def test_new_reporter_is_in_reset_state(self):
self.assertIsNone(self.reporter.results)
self.assertIsNone(self.reporter.wb)
self.assertFalse(self.reporter.reportGenerated)
self.assertEqual({}, self.reporter.kwConfig)
##### Reporter config function tests
def test_reporter_can_take_optional_config_params(self):
configDict = {
"report-include-summary": "yes",
}
newReporter = XlsxReporter(db=self.db, config=configDict)
self.assertEqual("yes", newReporter.kwConfig["report-include-summary"])
def test_can_get_reporter_final_config(self):
self.db.setConfigValue(key="report-include-summary", value="yes")
summary = self.reporter._getFinalConfigValue(
key="report-include-summary")
self.assertEqual(summary, "yes")
def test_can_override_db_config_in_reporter_final_config(self):
self.db.setConfigValue(key="report-include-summary", value="yes")
newReporter = XlsxReporter(db=self.db,
config={"report-include-summary": "no"})
summary = newReporter._getFinalConfigValue(
key="report-include-summary")
self.assertEqual(summary, "no")
##### Reporter setup function tests
def test_can_set_results_and_create_workbook(self):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.assertEqual(Workbook, type(self.reporter.wb))
self.assertEqual(results, self.reporter.results)
##### Reporter generate function tests
def test_generate_report_fails_if_results_not_ready(self):
with self.assertRaises(ReportNotReadyError):
self.reporter.generate()
self.assertFalse(self.reporter.reportGenerated)
# even with workbook created, should fail if results aren't ready yet
self.reporter.wb = Workbook()
with self.assertRaises(ReportNotReadyError):
self.reporter.generate()
self.assertFalse(self.reporter.reportGenerated)
@mock.patch('slm.reports.xlsx.XlsxReporter._generateFileListings')
@mock.patch('slm.reports.xlsx.XlsxReporter._generateCategorySheets')
def test_generate_calls_required_helpers(self, cs_mock, file_mock):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
cs_mock.assert_called_with(self.reporter.wb, self.reporter.results)
file_mock.assert_called_with(self.reporter.wb,
self.reporter.results,
strip_licenseref=False)
def test_generate_sets_report_generated_flag(self):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
self.assertTrue(self.reporter.reportGenerated)
def test_can_generate_report_category_pages(self):
wb = Workbook()
results = self._getAnalysisResults()
self.reporter._generateCategorySheets(wb, results)
# workbook now has the expected category sheets
# catID3 does not appear because it has no files
self.assertEqual(["catID2", "catID1", "No license found"],
wb.sheetnames)
# proper headers are set for each category page
# and header fonts are as expected
for sheet in wb:
headerFile = sheet['A1']
self.assertEqual("File", headerFile.value)
self.assertEqual(16, headerFile.font.size)
self.assertTrue(headerFile.font.bold)
self.assertFalse(headerFile.alignment.wrap_text)
headerLicense = sheet['B1']
self.assertEqual("License", sheet['B1'].value)
self.assertEqual(16, headerLicense.font.size)
self.assertTrue(headerLicense.font.bold)
self.assertFalse(headerLicense.alignment.wrap_text)
# column widths are as expected
self.assertEqual(100, sheet.column_dimensions["A"].width)
self.assertEqual(60, sheet.column_dimensions["B"].width)
def test_cannot_generate_files_before_category_sheets_are_generated(self):
wb = Workbook()
results = self._getAnalysisResults()
with self.assertRaises(ReportNotReadyError):
self.reporter._generateFileListings(wb, results)
def test_can_generate_file_listings(self):
wb = Workbook()
results = self._getAnalysisResults()
self.reporter._generateCategorySheets(wb, results)
self.reporter._generateFileListings(wb, results)
# correct files and licenses are in the correct cells
ws1 = wb['catID2']
self.assertEqual("/first/tmp/f2", ws1['A2'].value)
self.assertEqual("another lic2cat2", ws1['B2'].value)
self.assertEqual("/tmp/f1", ws1['A3'].value)
self.assertEqual("another lic2cat2", ws1['B3'].value)
self.assertEqual("/earliest/tmp/f3", ws1['A4'].value)
self.assertEqual("lic1cat2", ws1['B4'].value)
self.assertEqual("/tmp/f4", ws1['A5'].value)
self.assertEqual("lic1cat2", ws1['B5'].value)
ws2 = wb['catID1']
self.assertEqual("/tmp/f5", ws2['A2'].value)
self.assertEqual("a license", ws2['B2'].value)
cellFile = ws2['A2']
self.assertEqual(14, cellFile.font.size)
self.assertFalse(cellFile.font.bold)
self.assertTrue(cellFile.alignment.wrap_text)
cellLicense = ws2['B2']
self.assertEqual(14, cellLicense.font.size)
self.assertFalse(cellLicense.font.bold)
self.assertTrue(cellLicense.alignment.wrap_text)
@mock.patch('slm.reports.xlsx.XlsxReporter._generateSummarySheet')
def test_summary_not_generated_if_config_not_set(self, summary_mock):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
summary_mock.assert_not_called()
@mock.patch('slm.reports.xlsx.XlsxReporter._generateSummarySheet')
def test_summary_is_generated_if_config_set(self, summary_mock):
self.db.setConfigValue(key="report-include-summary", value="yes")
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
summary_mock.assert_called_with(self.reporter.wb,
self.reporter.results,
strip_licenseref=False)
def test_can_generate_summary_sheet(self):
wb = Workbook()
results = self._getAnalysisResults()
self.reporter._generateSummarySheet(wb, results)
# sheet title is as expected
self.assertEqual('License summary', wb.sheetnames[0])
ws = wb.active
# column widths are as expected
self.assertEqual(3, ws.column_dimensions["A"].width)
self.assertEqual(60, ws.column_dimensions["B"].width)
self.assertEqual(10, ws.column_dimensions["C"].width)
# headers are as expected
self.assertEqual("License", ws['A1'].value)
self.assertEqual(16, ws['A1'].font.size)
self.assertTrue(ws['A1'].font.bold)
self.assertFalse(ws['A1'].alignment.wrap_text)
self.assertEqual("# of files", ws['C1'].value)
self.assertEqual(16, ws['C1'].font.size)
self.assertTrue(ws['C1'].font.bold)
self.assertFalse(ws['C1'].alignment.wrap_text)
# and rows/values and formatting are all as expected
# cats and licenses with no files should not appear
self.assertEqual("catID2:", ws['A3'].value)
self.assertEqual(16, ws['A3'].font.size)
self.assertTrue(ws['A3'].font.bold)
self.assertFalse(ws['A3'].alignment.wrap_text)
self.assertEqual("another lic2cat2", ws['B4'].value)
self.assertEqual(14, ws['B4'].font.size)
self.assertFalse(ws['B4'].font.bold)
self.assertTrue(ws['B4'].alignment.wrap_text)
self.assertEqual(2, ws['C4'].value)
self.assertEqual("lic1cat2", ws['B5'].value)
self.assertEqual(2, ws['C5'].value)
self.assertEqual("catID1:", ws['A6'].value)
self.assertEqual("a license", ws['B7'].value)
self.assertEqual(1, ws['C7'].value)
self.assertEqual("No license found:", ws['A8'].value)
self.assertEqual("No license found", ws['B9'].value)
self.assertEqual(4, ws['C9'].value)
self.assertEqual("TOTAL", ws['A11'].value)
self.assertEqual(16, ws['A11'].font.size)
self.assertTrue(ws['A11'].font.bold)
self.assertFalse(ws['A11'].alignment.wrap_text)
self.assertEqual(9, ws['C11'].value)
self.assertEqual(16, ws['C11'].font.size)
self.assertTrue(ws['C11'].font.bold)
self.assertFalse(ws['C11'].alignment.wrap_text)
def test_summary_sheet_not_overwritten_by_other_sheets(self):
self.db.setConfigValue(key="report-include-summary", value="yes")
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
# workbook should have the summary sheet AND the expected category sheets
self.assertEqual(
["License summary", "catID2", "catID1", "No license found"],
self.reporter.wb.sheetnames)
# the summary sheet has the expected summary headers
self.assertEqual("License",
self.reporter.wb['License summary']['A1'].value)
self.assertEqual("# of files",
self.reporter.wb['License summary']['C1'].value)
# and the file listing sheets have the expected file listing headers
self.assertEqual("File", self.reporter.wb['catID2']['A1'].value)
self.assertEqual("License", self.reporter.wb['catID2']['B1'].value)
##### Reporter findings license renaming tests
def test_can_modify_licenses_in_no_lic_found_cat_for_findings(self):
self.db.setConfigValue(key="analyze-extensions", value="yes")
self.db.setConfigValue(key="analyze-emptyfile", value="yes")
self.db.setConfigValue(key="analyze-thirdparty", value="yes")
results = self._getAnalysisResults()
# hand the "No license found" category to the annotate function
self.reporter._annotateNoLicenseFound(
catNoLicense=self.cat4,
nextLicID=7,
)
# check that it has been modified as expected, with new licenses,
# and with the expected files within those licenses
# this will temporarily get assigned the next available license ID
# (only in memory, not committed to db)
noLicExt = self.cat4.licensesSorted[7]
self.assertEqual("No license found - excluded file extension",
noLicExt.name)
self.assertEqual(4, noLicExt.category_id)
self.assertTrue(noLicExt.hasFiles)
self.assertEqual(noLicExt.filesSorted[6], self.f6)
noLicEmpty = self.cat4.licensesSorted[8]
self.assertEqual("No license found - empty file", noLicEmpty.name)
self.assertEqual(4, noLicEmpty.category_id)
self.assertTrue(noLicEmpty.hasFiles)
self.assertEqual(noLicEmpty.filesSorted[7], self.f7)
noLicThird = self.cat4.licensesSorted[9]
self.assertEqual("No license found - third party directory",
noLicThird.name)
self.assertEqual(4, noLicThird.category_id)
self.assertTrue(noLicThird.hasFiles)
self.assertEqual(noLicThird.filesSorted[8], self.f8)
##### Reporter save function tests
@mock.patch('slm.reports.xlsx.os.path.exists', return_value=True)
def test_save_check_raises_exception_if_file_already_exists(
self, os_exists):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
with self.assertRaises(ReportFileError):
self.reporter._saveCheck(path="/tmp/fake/existing.xlsx")
@mock.patch('slm.reports.xlsx.os.path.exists', return_value=True)
@mock.patch('slm.reports.xlsx.os.access', return_value=True)
def test_save_check_doesnt_raise_exception_if_replace_flag_set(
self, os_access, os_exists):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
try:
self.reporter._saveCheck(path="/tmp/fake/existing.xlsx",
replace=True)
except ReportFileError:
self.fail(
"ReportFileError raised even though replace flag is True")
@mock.patch('slm.reports.xlsx.os.access', return_value=False)
def test_save_check_fails_if_no_write_permission_for_path(self, os_access):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
with self.assertRaises(ReportFileError):
self.reporter._saveCheck(path="/tmp/readonly/report.xlsx")
def test_save_check_fails_if_report_not_ready_yet(self):
with self.assertRaises(ReportNotReadyError):
self.reporter._saveCheck(path="whatever")
# and even with results added, should fail if report not generated yet
results = self._getAnalysisResults()
self.reporter.setResults(results)
with self.assertRaises(ReportNotReadyError):
self.reporter._saveCheck(path="whatever")
@mock.patch('slm.reports.xlsx.openpyxl.workbook.Workbook.save')
@mock.patch('slm.reports.xlsx.XlsxReporter._saveCheck')
def test_save_calls_save_checker(self, check_mock, save_mock):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
path = "something"
replace = True
self.reporter.save(path=path, replace=replace)
check_mock.assert_called_with(path=path, replace=replace)
save_mock.assert_called_with(path)
##### Reporter misc helper function tests
def test_can_get_max_lic_id_from_results(self):
results = self._getAnalysisResults()
maxLicID = self.reporter._getResultsMaxLicenseID(results)
self.assertEqual(6, maxLicID)
def test_can_create_temp_license(self):
lic = self.reporter._createTempLicense(catID=4,
nextLicID=7,
name="No license found - blah")
self.assertEqual(7, lic._id)
self.assertEqual(4, lic.category_id)
self.assertEqual("No license found - blah", lic.name)
self.assertTrue(lic.hasFiles)
self.assertEqual(OrderedDict(), lic.filesSorted)
def test_will_strip_licenseref_prefix_if_requested(self):
licName = "LicenseRef-blah-1 AND Apache-2.0 AND LicenseRef-hi"
finalLicName = self.reporter._getFinalLicenseName(licName, True)
self.assertEqual(finalLicName, "blah-1 AND Apache-2.0 AND hi")
def test_will_not_strip_licenseref_prefix_if_not_requested(self):
licName = "LicenseRef-blah-1 AND Apache-2.0 AND LicenseRef-hi"
finalLicName = self.reporter._getFinalLicenseName(licName, False)
self.assertEqual(finalLicName,
"LicenseRef-blah-1 AND Apache-2.0 AND LicenseRef-hi")
@mock.patch('slm.reports.xlsx.XlsxReporter._getFinalLicenseName',
return_value="blah")
def test_generate_file_listings_strips_licenseref_prefix_if_arg_set(
self, final_name):
wb = Workbook()
results = self._getAnalysisResults()
self.reporter._generateCategorySheets(wb, results)
self.reporter._generateFileListings(wb, results, strip_licenseref=True)
final_name.assert_any_call("another lic2cat2", True)
@mock.patch('slm.reports.xlsx.XlsxReporter._getFinalLicenseName',
return_value="blah")
def test_generate_file_listings_does_not_strip_licenseref_prefix_if_arg_not_set(
self, final_name):
wb = Workbook()
results = self._getAnalysisResults()
self.reporter._generateCategorySheets(wb, results)
self.reporter._generateFileListings(wb, results)
final_name.assert_any_call("another lic2cat2", False)
@mock.patch('slm.reports.xlsx.XlsxReporter._generateFileListings')
def test_generate_strips_licenseref_prefix_if_config_set(self, gfl_mock):
self.db.setConfigValue(key="report-strip-licenseref", value="yes")
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
gfl_mock.assert_any_call(self.reporter.wb,
self.reporter.results,
strip_licenseref=True)
@mock.patch('slm.reports.xlsx.XlsxReporter._generateFileListings')
def test_generate_does_not_strip_licenseref_prefix_if_config_not_set(
self, gfl_mock):
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
gfl_mock.assert_any_call(self.reporter.wb,
self.reporter.results,
strip_licenseref=False)
@mock.patch('slm.reports.xlsx.XlsxReporter._getFinalLicenseName',
return_value="blah")
def test_generate_summary_strips_licenseref_prefix_if_arg_set(
self, final_name):
wb = Workbook()
results = self._getAnalysisResults()
self.reporter._generateCategorySheets(wb, results)
self.reporter._generateSummarySheet(wb, results, strip_licenseref=True)
final_name.assert_any_call("another lic2cat2", True)
@mock.patch('slm.reports.xlsx.XlsxReporter._getFinalLicenseName',
return_value="blah")
def test_generate_summary_does_not_strip_licenseref_prefix_if_arg_not_set(
self, final_name):
wb = Workbook()
results = self._getAnalysisResults()
self.reporter._generateCategorySheets(wb, results)
self.reporter._generateSummarySheet(wb, results)
final_name.assert_any_call("another lic2cat2", False)
@mock.patch('slm.reports.xlsx.XlsxReporter._generateSummarySheet')
def test_generate_strips_licenseref_prefix_from_summary_if_config_set(
self, gss_mock):
self.db.setConfigValue(key="report-include-summary", value="yes")
self.db.setConfigValue(key="report-strip-licenseref", value="yes")
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
gss_mock.assert_any_call(self.reporter.wb,
self.reporter.results,
strip_licenseref=True)
@mock.patch('slm.reports.xlsx.XlsxReporter._generateSummarySheet')
def test_generate_does_not_strip_licenseref_prefix_from_summary_if_config_not_set(
self, gss_mock):
self.db.setConfigValue(key="report-include-summary", value="yes")
results = self._getAnalysisResults()
self.reporter.setResults(results)
self.reporter.generate()
gss_mock.assert_any_call(self.reporter.wb,
self.reporter.results,
strip_licenseref=False)
class DBCategoryUnitTestSuite(unittest.TestCase):
"""spdxLicenseManager unit test suite for category data in DB."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleCategoryData()
def tearDown(self):
self.db.closeDB()
self.db = None
def insertSampleCategoryData(self):
categories = [
Category(_id=1, name="a category", order=3),
Category(_id=2, name="cat of crazy licenses", order=2),
Category(_id=3, name="blah category", order=1),
]
self.db.session.bulk_save_objects(categories)
self.db.session.commit()
##### Test cases below
def test_can_retrieve_all_category_names_and_descs(self):
categories = self.db.getCategoriesAll()
self.assertIsInstance(categories, list)
self.assertEqual(len(categories), 3)
self.assertEqual(categories[0]._id, 3)
self.assertEqual(categories[0].name, "blah category")
self.assertEqual(categories[0].order, 1)
def test_all_categories_are_sorted_by_order(self):
categories = self.db.getCategoriesAll()
self.assertEqual(categories[0].name, "blah category")
self.assertEqual(categories[1].name, "cat of crazy licenses")
self.assertEqual(categories[2].name, "a category")
def test_can_retrieve_one_category_by_id(self):
category = self.db.getCategory(_id=2)
self.assertEqual(category.name, "cat of crazy licenses")
def test_can_retrieve_one_category_by_name(self):
category = self.db.getCategory(name="a category")
self.assertEqual(category._id, 1)
def test_cannot_retrieve_category_by_both_name_and_id(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getCategory(_id=3, name="blah category")
def test_cannot_retrieve_category_without_either_name_or_id(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getCategory()
def test_cannot_retrieve_category_with_positional_args(self):
with self.assertRaises(TypeError):
self.db.getCategory("blah category")
def test_returns_none_if_category_not_found_by_id(self):
category = self.db.getCategory(_id=17)
self.assertIsNone(category)
def test_returns_none_if_category_not_found_by_name(self):
category = self.db.getCategory(name="noSuchCategory")
self.assertIsNone(category)
def test_can_add_and_retrieve_categories(self):
category_id = self.db.addCategory(name="newcat", order=4)
# confirm that we now have four categories
categories = self.db.getCategoriesAll()
self.assertEqual(len(categories), 4)
# and confirm that we can retrieve this one by name
category = self.db.getCategory(name="newcat")
self.assertEqual(category._id, 4)
self.assertEqual(category.order, 4)
# and confirm that we can retrieve this one by id
category = self.db.getCategory(_id=4)
self.assertEqual(category.name, "newcat")
self.assertEqual(category.order, 4)
def test_can_start_adding_but_rollback_category(self):
category_id = self.db.addCategory(name="will rollback",
order=99,
commit=False)
self.db.rollback()
# confirm that we still only have three categories
categories = self.db.getCategoriesAll()
self.assertEqual(len(categories), 3)
# and confirm that this category ID doesn't exist in database
category = self.db.getCategory(_id=category_id)
self.assertIsNone(category)
def test_can_start_adding_and_then_commit_categories(self):
c1_id = self.db.addCategory(name="newc1", order=98, commit=False)
c2_id = self.db.addCategory(name="newc2", order=99, commit=False)
self.db.commit()
# confirm that we now have five categories
categories = self.db.getCategoriesAll()
self.assertEqual(len(categories), 5)
def test_omitting_order_from_new_category_places_it_at_end(self):
category_id = self.db.addCategory(name="newcat")
category = self.db.getCategory(name="newcat")
self.assertEqual(category.order, 4)
def test_cannot_create_category_with_existing_order(self):
with self.assertRaises(ProjectDBInsertError):
self.db.addCategory(name="duplicate order", order=3)
def test_can_get_highest_category_order(self):
highestOrder = self.db.getCategoryHighestOrder()
self.assertEqual(highestOrder, 3)
def test_highest_order_works_even_with_no_categories(self):
newdb = ProjectDB()
newdb.createDB(":memory:")
newdb.initializeDBTables()
highestOrder = newdb.getCategoryHighestOrder()
self.assertEqual(highestOrder, 0)
def test_can_edit_category_name(self):
self.db.changeCategoryName(name="a category",
newName="another category")
category = self.db.getCategory(name="another category")
self.assertEqual(category._id, 1)
def test_cannot_edit_category_name_that_does_not_exist(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeCategoryName(name="invalid",
newName="this will fail")
def test_cannot_change_category_name_to_existing_name(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeCategoryName(name="a category",
newName="blah category")
def test_can_reorder_categories_from_higher_to_lower(self):
self.db.changeCategoryOrder(name="a category",
sortBefore="blah category")
categories = self.db.getCategoriesAll()
self.assertEqual(categories[0].name, "a category")
self.assertEqual(categories[1].name, "blah category")
self.assertEqual(categories[2].name, "cat of crazy licenses")
def test_can_reorder_categories_from_lower_to_higher(self):
self.db.changeCategoryOrder(name="blah category",
sortBefore="a category")
categories = self.db.getCategoriesAll()
self.assertEqual(categories[0].name, "cat of crazy licenses")
self.assertEqual(categories[1].name, "blah category")
self.assertEqual(categories[2].name, "a category")
def test_cannot_reorder_category_name_before_one_that_does_not_exist(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeCategoryOrder(name="a category", sortBefore="oops")
def test_cannot_reorder_category_name_that_does_not_exist(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeCategoryOrder(name="oops", sortBefore="a category")
def test_cannot_create_category_with_order_less_than_one(self):
with self.assertRaises(ProjectDBInsertError):
self.db.addCategory(name="need positive order", order=0)
def test_highest_order_works_even_with_no_categories(self):
newdb = ProjectDB()
newdb.createDB(":memory:")
newdb.initializeDBTables()
highestOrder = newdb.getCategoryHighestOrder()
self.assertEqual(highestOrder, 0)
class DBConversionUnitTestSuite(unittest.TestCase):
"""spdxLicenseManager unit test suite for converting license names in DB."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleCategoryData()
self.insertSampleLicenseData()
self.insertSampleConversionData()
def tearDown(self):
self.db.closeDB()
self.db = None
def insertSampleCategoryData(self):
categories = [
Category(_id=1, name="a category", order=3),
Category(_id=2, name="cat", order=2),
Category(_id=3, name="blah category", order=1),
]
self.db.session.bulk_save_objects(categories)
self.db.session.commit()
def insertSampleLicenseData(self):
licenses = [
License(_id=1, name="DoAnything", category_id=1),
License(_id=2, name="HarshEULA", category_id=2),
License(_id=3, name="293PageEULA", category_id=3),
License(_id=4, name="DoAnythingNoncommercial", category_id=1),
]
self.db.session.bulk_save_objects(licenses)
self.db.session.commit()
def insertSampleConversionData(self):
conversions = [
Conversion(_id=1, old_text="293", new_license_id=3),
Conversion(_id=2, old_text="NC", new_license_id=4),
Conversion(_id=3, old_text="anything", new_license_id=1),
Conversion(_id=4, old_text="Anything", new_license_id=1),
]
self.db.session.bulk_save_objects(conversions)
self.db.session.commit()
##### Test cases below
def test_can_retrieve_one_conversion_by_id(self):
conversion = self.db.getConversion(_id=2)
self.assertEqual(conversion.old_text, "NC")
self.assertEqual(conversion.new_license_id, 4)
self.assertEqual(conversion.new_license._id, 4)
self.assertEqual(conversion.new_license.name, "DoAnythingNoncommercial")
def test_can_retrieve_one_conversion_by_name(self):
conversion = self.db.getConversion(old_text="293")
self.assertEqual(conversion._id, 1)
def test_cannot_retrieve_conversion_by_both_name_and_id(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getConversion(_id=3, old_text="anything")
def test_cannot_retrieve_conversion_without_either_name_or_id(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getConversion()
def test_cannot_retrieve_conversion_with_positional_args(self):
with self.assertRaises(TypeError):
self.db.getConversion("NC")
def test_returns_none_if_conversion_not_found_by_id(self):
conversion = self.db.getConversion(_id=17)
self.assertIsNone(conversion)
def test_returns_none_if_conversion_not_found_by_name(self):
conversion = self.db.getConversion(old_text="noSuchConversion")
self.assertIsNone(conversion)
def test_can_add_and_retrieve_conversions(self):
conv_id = self.db.addConversion(old_text="harsh", new_license="HarshEULA")
# confirm that we now have five conversions
convs = self.db.getConversionsAll()
self.assertEqual(len(convs), 5)
# and confirm that we can retrieve this one by its text
conv = self.db.getConversion(old_text="harsh")
self.assertEqual(conv._id, 5)
self.assertEqual(conv.new_license_id, 2)
self.assertIsNotNone(conv.new_license)
self.assertEqual(conv.new_license.name, "HarshEULA")
# and confirm that we can retrieve this one by id
conv = self.db.getConversion(_id=5)
self.assertEqual(conv.old_text, "harsh")
self.assertEqual(conv.new_license_id, 2)
def test_can_start_adding_but_rollback_conversion(self):
conv_id = self.db.addConversion(old_text="will rollback",
new_license="293PageEULA", commit=False)
self.db.rollback()
# confirm that we still only have four conversions
convs = self.db.getConversionsAll()
self.assertEqual(len(convs), 4)
# and confirm that this conversion ID doesn't exist in database
conv = self.db.getConversion(_id=5)
self.assertIsNone(conv)
def test_can_start_adding_and_then_commit_conversions(self):
c1_id = self.db.addConversion(old_text="c1", new_license="293PageEULA",
commit=False)
c2_id = self.db.addConversion(old_text="c2", new_license="293PageEULA",
commit=False)
self.db.commit()
# confirm that we now have six conversions
convs = self.db.getConversionsAll()
self.assertEqual(len(convs), 6)
def test_cannot_add_conversion_without_license(self):
with self.assertRaises(TypeError):
self.db.addConversion(old_text="oops")
# confirm it wasn't added either
conv = self.db.getConversion(old_text="oops")
self.assertIsNone(conv)
def test_cannot_add_conversion_without_existing_license(self):
with self.assertRaises(ProjectDBInsertError):
self.db.addConversion(old_text="oops", new_license="blah")
# confirm it wasn't added either
conv = self.db.getConversion(old_text="oops")
self.assertIsNone(conv)
def test_cannot_add_conversion_with_duplicate_name(self):
with self.assertRaises(ProjectDBInsertError):
self.db.addConversion(old_text="NC", new_license="293PageEULA")
def test_can_edit_conversion_license(self):
self.db.changeConversion(old_text="NC", new_license="DoAnything")
conv = self.db.getConversion(old_text="NC")
self.assertEqual(conv.new_license_id, 1)
def test_cannot_edit_conversion_name_that_does_not_exist(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeConversion(old_text="invalid", new_license="will fail")
def test_can_retrieve_all_conversions(self):
convs = self.db.getConversionsAll()
self.assertIsInstance(convs, list)
self.assertEqual(len(convs), 4)
self.assertIsInstance(convs[0], Conversion)
# will sort alphabetically
self.assertEqual(convs[2]._id, 2)
self.assertEqual(convs[2].old_text, "NC")
class TVImporterTestSuite(unittest.TestCase):
"""spdxLicenseManager SPDX tag-value importer unit test suite."""
def setUp(self):
# create importer object
self.importer = TVImporter()
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleCategoryData()
self.insertSampleLicenseData()
self.insertSampleConversionData()
self.insertSampleSubprojectData()
self.insertSampleScanData()
# build sample file data list
self.fd1 = createFD("/tmp/f1", "DoAnything", md5="abcdef")
self.fd2 = createFD("/tmp/f2", "DoAnythingNoncommercial", sha256="abcdef")
self.fd3 = createFD("/tmp/f3", "HarshEULA", sha1="abcdef")
self.fd4 = createFD("/tmp/f4", "HarshEULA")
self.fdList = [self.fd1, self.fd2, self.fd3, self.fd4]
# not in fdList by default
self.fd5 = createFD("/tmp/badLicense", "UnknownLicense")
self.fd6 = createFD("/tmp/badLic2", "SecondUnknownLic")
self.fdConvert = createFD("/tmp/needsConvert", "293")
def tearDown(self):
pass
def insertSampleCategoryData(self):
categories = [
Category(_id=1, name="a category", order=3),
Category(_id=2, name="cat", order=2),
Category(_id=3, name="blah category", order=1),
]
self.db.session.bulk_save_objects(categories)
self.db.session.commit()
def insertSampleLicenseData(self):
licenses = [
License(_id=1, name="DoAnything", category_id=1),
License(_id=2, name="HarshEULA", category_id=2),
License(_id=3, name="293PageEULA", category_id=3),
License(_id=4, name="DoAnythingNoncommercial", category_id=1),
]
self.db.session.bulk_save_objects(licenses)
self.db.session.commit()
def insertSampleConversionData(self):
conversions = [
Conversion(_id=1, old_text="293", new_license_id=3),
Conversion(_id=2, old_text="NC", new_license_id=4),
Conversion(_id=3, old_text="anything", new_license_id=1),
Conversion(_id=4, old_text="Anything", new_license_id=1),
]
self.db.session.bulk_save_objects(conversions)
self.db.session.commit()
def insertSampleSubprojectData(self):
subprojects = [
Subproject(_id=1, name="sub1", desc="subproject 1"),
]
self.db.session.bulk_save_objects(subprojects)
self.db.session.commit()
def insertSampleScanData(self):
scans = [
Scan(_id=1, subproject_id=1, scan_dt=datetime.date(2017, 1, 10),
desc="new scan"),
]
self.db.session.bulk_save_objects(scans)
self.db.session.commit()
self.scan_id = 1
##### Test cases below
def test_new_importer_is_in_expected_reset_state(self):
self.assertEqual(self.importer.scanChecked, False)
self.assertEqual(self.importer.licensesAll, [])
self.assertEqual(self.importer.licensesUnknown, [])
self.assertEqual(self.importer.licensesMapping, {})
self.assertEqual(self.importer.pathDuplicates, [])
self.assertEqual(self.importer.importedCount, 0)
def test_import_fails_if_scan_not_checked_first(self):
with self.assertRaises(ProjectDBInsertError):
self.importer.importFileDataList(fdList=self.fdList, db=self.db,
scan_id=self.scan_id)
def test_cannot_check_without_providing_valid_fdList(self):
with self.assertRaises(ProjectDBInsertError):
self.importer.checkFileDataList(db=self.db)
def test_cannot_check_without_providing_database(self):
with self.assertRaises(ProjectDBInsertError):
self.importer.checkFileDataList(fdList=self.fdList)
def test_checking_valid_fdList_returns_true(self):
retval = self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
self.assertEqual(True, retval)
def test_checker_returns_false_if_any_licenses_are_unknown(self):
self.fdList.append(self.fd5)
retval = self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
self.assertEqual(False, retval)
def test_can_get_license_list_if_any_are_unknown(self):
self.fdList.append(self.fd5)
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
unknowns = self.importer.getUnknowns()
self.assertIn("UnknownLicense", unknowns)
self.assertNotIn("DoAnything", unknowns)
self.assertNotIn("HarshEULA", unknowns)
def test_license_list_is_sorted_if_multiple_are_unknown(self):
self.fdList.append(self.fd5)
self.fdList.append(self.fd6)
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
unknowns = self.importer.getUnknowns()
self.assertEqual("SecondUnknownLic", unknowns[0])
self.assertEqual("UnknownLicense", unknowns[1])
def test_checker_returns_true_if_all_paths_are_unique(self):
retval = self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
self.assertEqual(True, retval)
def test_checker_returns_false_if_any_paths_are_duplicates(self):
fdup = createFD("/tmp/f2", "DoAnythingNoncommercial", sha256="abcdef")
self.fdList.append(fdup)
retval = self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
self.assertEqual(False, retval)
def test_duplicates_list_is_empty_if_all_paths_are_unique(self):
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
dups = self.importer.getDuplicatePaths()
self.assertEqual([], dups)
def test_duplicates_list_has_paths_if_any_paths_are_duplicates(self):
fdup = createFD("/tmp/f2", "DoAnythingNoncommercial", sha256="abcdef")
self.fdList.append(fdup)
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
dups = self.importer.getDuplicatePaths()
self.assertEqual(["/tmp/f2"], dups)
def test_can_get_duplicate_paths_after_checker_if_any(self):
fdup = createFD("/tmp/f2", "DoAnythingNoncommercial", sha256="abcdef")
self.fdList.append(fdup)
retval = self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
dups = self.importer.getDuplicatePaths()
self.assertIn("/tmp/f2", dups)
self.assertNotIn("/tmp/f1", dups)
def test_checker_returns_true_if_all_is_good(self):
retval = self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
self.assertEqual(True, retval)
def test_reads_licenses_into_licensesAll(self):
# fill in finalLicense, since we are skipping _applyConversions
for fd in self.fdList:
fd.finalLicense = fd.license
self.importer._checkFileDataListForLicenses(fdList=self.fdList, db=self.db)
self.assertIn("DoAnything", self.importer.licensesAll)
def test_reads_only_unknown_licenses_into_licensesUnknown(self):
self.fdList.append(self.fd5)
# fill in finalLicense, since we are skipping _applyConversions
for fd in self.fdList:
fd.finalLicense = fd.license
self.importer._checkFileDataListForLicenses(fdList=self.fdList, db=self.db)
self.assertIn("UnknownLicense", self.importer.licensesUnknown)
self.assertNotIn("DoAnything", self.importer.licensesUnknown)
def test_reads_only_known_licenses_into_licensesMapping(self):
self.fdList.append(self.fd5)
# fill in finalLicense, since we are skipping _applyConversions
for fd in self.fdList:
fd.finalLicense = fd.license
self.importer._checkFileDataListForLicenses(fdList=self.fdList, db=self.db)
self.assertEqual(1, self.importer.licensesMapping.get("DoAnything", None))
self.assertEqual(2, self.importer.licensesMapping.get("HarshEULA", None))
self.assertEqual(None, self.importer.licensesMapping.get("UnknownLicense", None))
def test_checker_returns_true_if_all_licenses_are_known(self):
# fill in finalLicense, since we are skipping _applyConversions
for fd in self.fdList:
fd.finalLicense = fd.license
retval = self.importer._checkFileDataListForLicenses(fdList=self.fdList,
db=self.db)
self.assertEqual(True, retval)
def test_checker_returns_false_if_any_licenses_are_unknown(self):
# fill in finalLicense, since we are skipping _applyConversions
for fd in self.fdList:
fd.finalLicense = fd.license
self.fdList.append(self.fd5)
retval = self.importer._checkFileDataListForLicenses(fdList=self.fdList,
db=self.db)
self.assertEqual(False, retval)
def test_cannot_import_without_providing_valid_fdList(self):
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
with self.assertRaises(ProjectDBInsertError):
self.importer.importFileDataList(db=self.db, scan_id=self.scan_id)
def test_cannot_import_without_providing_database(self):
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
with self.assertRaises(ProjectDBInsertError):
self.importer.importFileDataList(fdList=self.fdList,
scan_id=self.scan_id)
def test_cannot_import_without_providing_scan_id(self):
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
with self.assertRaises(ProjectDBInsertError):
self.importer.importFileDataList(fdList=self.fdList, db=self.db)
def test_cannot_import_with_positional_args(self):
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
with self.assertRaises(TypeError):
self.importer.importFileDataList(self.fdList)
with self.assertRaises(TypeError):
self.importer.importFileDataList(self.fdList, self.db)
with self.assertRaises(TypeError):
self.importer.importFileDataList(self.fdList, self.db, self.scan_id)
def test_checker_returns_true_if_all_paths_are_unique(self):
retval = self.importer._checkFileDataListForDuplicatePaths(fdList=self.fdList)
self.assertEqual(True, retval)
def test_checker_returns_false_if_any_paths_are_duplicates(self):
fdup = createFD("/tmp/f2", "DoAnythingNoncommercial", sha256="abcdef")
self.fdList.append(fdup)
retval = self.importer._checkFileDataListForDuplicatePaths(fdList=self.fdList)
self.assertEqual(False, retval)
def test_files_are_imported_if_all_is_good(self):
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
retval = self.importer.importFileDataList(fdList=self.fdList, db=self.db,
scan_id=self.scan_id)
self.assertEqual(True, retval)
f1 = self.db.getFile(scan_id=self.scan_id, path="/tmp/f1")
self.assertEqual("/tmp/f1", f1.path)
self.assertEqual("abcdef", f1.md5)
self.assertEqual("DoAnything", f1.license.name)
f4 = self.db.getFile(scan_id=self.scan_id, path="/tmp/f4")
self.assertEqual("/tmp/f4", f4.path)
self.assertEqual(None, f4.md5)
self.assertEqual("HarshEULA", f4.license.name)
def test_can_get_count_of_imported_files_if_all_licenses_are_known(self):
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
self.importer.importFileDataList(fdList=self.fdList, db=self.db,
scan_id=self.scan_id)
count = self.importer.getImportedCount()
self.assertEqual(4, count)
def test_files_are_not_imported_if_any_licenses_are_unknown(self):
self.fdList.append(self.fd5)
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
with self.assertRaises(ProjectDBInsertError):
self.importer.importFileDataList(fdList=self.fdList, db=self.db,
scan_id=self.scan_id)
f1 = self.db.getFile(scan_id=self.scan_id, path="/tmp/f1")
self.assertIsNone(f1)
def test_checker_applies_conversions(self):
self.fdList.append(self.fdConvert)
retval = self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
self.assertTrue(retval)
self.assertEqual("293PageEULA", self.fdConvert.finalLicense)
def test_path_prefixes_are_stripped_if_config_is_yes(self):
self.db.setConfigValue("import-strip-path-prefixes", "Yes")
prefix = self.importer._applyPathPrefixStrip(fdList=self.fdList,
db=self.db)
self.assertEqual("/tmp", prefix)
self.assertEqual("/tmp/f1", self.fd1.path)
self.assertEqual("/f1", self.fd1.finalPath)
def test_path_prefixes_are_not_stripped_if_config_is_no(self):
self.db.setConfigValue("import-strip-path-prefixes", "no")
prefix = self.importer._applyPathPrefixStrip(fdList=self.fdList,
db=self.db)
self.assertEqual("", prefix)
self.assertEqual("/tmp/f1", self.fd1.path)
self.assertEqual("/tmp/f1", self.fd1.finalPath)
def test_path_prefixes_are_not_stripped_if_config_is_not_set(self):
prefix = self.importer._applyPathPrefixStrip(fdList=self.fdList,
db=self.db)
self.assertEqual("", prefix)
self.assertEqual("/tmp/f1", self.fd1.path)
self.assertEqual("/tmp/f1", self.fd1.finalPath)
def test_path_prefixes_are_not_stripped_if_mixing_abs_and_rel_paths(self):
self.db.setConfigValue("import-strip-path-prefixes", "yes")
self.fdDifferentPath = createFD("absolutePath", "293PageEULA")
self.fdList.append(self.fdDifferentPath)
prefix = self.importer._applyPathPrefixStrip(fdList=self.fdList,
db=self.db)
self.assertEqual("", prefix)
self.assertEqual("/tmp/f1", self.fd1.path)
self.assertEqual("/tmp/f1", self.fd1.finalPath)
def test_path_prefixes_are_not_stripped_if_no_common_prefix(self):
self.db.setConfigValue("import-strip-path-prefixes", "yes")
fdDifferentPath1 = createFD("somewhereElse", "293PageEULA")
fdDifferentPath2 = createFD("something", "293PageEULA")
tmpFDList = [fdDifferentPath1, fdDifferentPath2]
prefix = self.importer._applyPathPrefixStrip(fdList=tmpFDList,
db=self.db)
self.assertEqual("", prefix)
self.assertEqual("somewhereElse", fdDifferentPath1.finalPath)
self.assertEqual("something", fdDifferentPath2.finalPath)
def test_file_path_prefixes_are_stripped_on_import_if_configured(self):
self.db.setConfigValue("import-strip-path-prefixes", "yes")
self.importer.checkFileDataList(fdList=self.fdList, db=self.db)
retval = self.importer.importFileDataList(fdList=self.fdList, db=self.db,
scan_id=self.scan_id)
self.assertEqual(True, retval)
f1 = self.db.getFile(scan_id=self.scan_id, path="/f1")
self.assertEqual("/f1", f1.path)
self.assertEqual("DoAnything", f1.license.name)
f4 = self.db.getFile(scan_id=self.scan_id, path="/f4")
self.assertEqual("/f4", f4.path)
self.assertEqual("HarshEULA", f4.license.name)
wrongPathFile = self.db.getFile(scan_id=self.scan_id, path="/tmp/f1")
self.assertIsNone(wrongPathFile)
class ReportAnalysisTestSuite(unittest.TestCase):
"""spdxLicenseManager analysis for reporting unit test suite."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleCategoryData()
self.insertSampleLicenseData()
self.insertSampleSubprojectData()
self.insertSampleScanData()
self.insertSampleFileData()
# create analyzer
self.analyzer = Analyzer(db=self.db)
def tearDown(self):
pass
def insertSampleCategoryData(self):
categories = [
Category(_id=1, name="a category", order=3),
Category(_id=2, name="cat", order=2),
Category(_id=3, name="blah category", order=1),
Category(_id=4, name="no lic category", order=4),
]
self.db.session.bulk_save_objects(categories)
self.db.session.commit()
def insertSampleLicenseData(self):
licenses = [
License(_id=1, name="DoAnything", category_id=1),
License(_id=2, name="HarshEULA", category_id=2),
License(_id=3, name="293PageEULA", category_id=3),
License(_id=4, name="DoAnythingNoncommercial", category_id=1),
License(_id=5, name="No license found", category_id=4),
License(_id=6, name="Also no license found", category_id=4),
License(_id=7, name="LicWithNoFiles", category_id=4),
]
self.db.session.bulk_save_objects(licenses)
self.db.session.commit()
def insertSampleSubprojectData(self):
subprojects = [
Subproject(_id=1, name="sub1", desc="subproject 1"),
]
self.db.session.bulk_save_objects(subprojects)
self.db.session.commit()
def insertSampleScanData(self):
scans = [
Scan(_id=1,
subproject_id=1,
scan_dt=datetime.date(2017, 1, 10),
desc="new scan"),
Scan(_id=2,
subproject_id=1,
scan_dt=datetime.date(2017, 2, 10),
desc="monthly scan 2"),
Scan(_id=3,
subproject_id=1,
scan_dt=datetime.date(2017, 3, 10),
desc="monthly scan 3"),
]
self.db.session.bulk_save_objects(scans)
self.db.session.commit()
def insertSampleFileData(self):
self.f1 = File(_id=1,
scan_id=1,
license_id=1,
path="/tmp/f1",
sha1=None,
md5="abcdef",
sha256=None)
self.f2 = File(_id=2,
scan_id=1,
license_id=4,
path="/tmp/f2",
sha1=None,
md5=None,
sha256="abcdef")
self.f3 = File(_id=3,
scan_id=1,
license_id=2,
path="/tmp/f3",
sha1="abcdef",
md5=None,
sha256=None)
self.f4 = File(_id=4,
scan_id=1,
license_id=2,
path="/tmp/f4",
sha1=None,
md5=None,
sha256=None)
self.f5 = File(_id=5,
scan_id=1,
license_id=5,
path="/tmp/nolic/image.png",
sha1=None,
md5=None,
sha256=None)
self.f6 = File(_id=6,
scan_id=1,
license_id=2,
path="/tmp/nolic/vendor/whatever",
sha1=None,
md5=None,
sha256=None)
self.f7 = File(_id=7,
scan_id=1,
license_id=6,
path="/tmp/nolic/vendor/image-both.png",
sha1=None,
md5=None,
sha256=None)
self.f8 = File(_id=8,
scan_id=1,
license_id=6,
path="/tmp/nolic/emptyfile",
sha1=None,
md5="d41d8cd98f00b204e9800998ecf8427e",
sha256=None)
self.f9 = File(_id=9,
scan_id=1,
license_id=6,
path="/tmp/nolic/vendor/emptyfile",
sha1=None,
md5="d41d8cd98f00b204e9800998ecf8427e",
sha256=None)
self.f10 = File(_id=10,
scan_id=1,
license_id=6,
path="/tmp/nolic/emptyfile.png",
sha1=None,
md5="d41d8cd98f00b204e9800998ecf8427e",
sha256=None)
self.f11 = File(_id=11,
scan_id=1,
license_id=6,
path="/tmp/nolic/vendor/emptyfile.json",
sha1=None,
md5="d41d8cd98f00b204e9800998ecf8427e",
sha256=None)
self.f21 = File(_id=21,
scan_id=2,
license_id=1,
path="/scan2/tmp/f1",
sha1=None,
md5="abcdef",
sha256=None)
self.f22 = File(_id=22,
scan_id=2,
license_id=4,
path="/scan2/tmp/f2",
sha1=None,
md5=None,
sha256="abcdef")
self.f23 = File(_id=23,
scan_id=2,
license_id=3,
path="/scan2/tmp/f3-lic3",
sha1="abcdef",
md5=None,
sha256=None)
self.f24 = File(_id=24,
scan_id=2,
license_id=2,
path="/scan2/tmp/f4",
sha1=None,
md5=None,
sha256=None)
self.f31 = File(_id=31,
scan_id=3,
license_id=1,
path="/scan3/tmp/f1",
sha1=None,
md5="abcdef",
sha256=None)
self.f32 = File(_id=32,
scan_id=3,
license_id=4,
path="/scan3/tmp/f2",
sha1=None,
md5=None,
sha256="abcdef")
self.f33 = File(_id=33,
scan_id=3,
license_id=2,
path="/scan3/tmp/f3",
sha1="abcdef",
md5=None,
sha256=None)
self.f34 = File(_id=34,
scan_id=3,
license_id=2,
path="/scan3/tmp/f4",
sha1=None,
md5=None,
sha256=None)
self.files = [
self.f1,
self.f2,
self.f3,
self.f4,
self.f5,
self.f6,
self.f7,
self.f8,
self.f9,
self.f10,
self.f11,
self.f21,
self.f22,
self.f23,
self.f24,
self.f31,
self.f32,
self.f33,
self.f34,
]
self.db.session.bulk_save_objects(self.files)
self.db.session.commit()
##### Helpers for tests
def _checkFileExtFindingIsNone(self, file_id):
ext = self.analyzer._getFile(file_id).findings.get("extension", None)
self.assertIsNone(ext)
def _checkFileExtFindingIsYes(self, file_id):
ext = self.analyzer._getFile(file_id).findings.get("extension", None)
self.assertEqual("yes", ext)
def _checkFileEmptyFindingIsNone(self, file_id):
empty = self.analyzer._getFile(file_id).findings.get("emptyfile", None)
self.assertIsNone(empty)
def _checkFileEmptyFindingIsYes(self, file_id):
empty = self.analyzer._getFile(file_id).findings.get("emptyfile", None)
self.assertEqual("yes", empty)
def _checkFileDirFindingIsNone(self, file_id):
tp = self.analyzer._getFile(file_id).findings.get("thirdparty", None)
self.assertIsNone(tp)
def _checkFileDirFindingIsYes(self, file_id):
tp = self.analyzer._getFile(file_id).findings.get("thirdparty", None)
self.assertEqual("yes", tp)
##### Test cases below
def test_analyzer_is_in_reset_state(self):
self.assertEqual(self.db, self.analyzer.db)
self.assertFalse(self.analyzer.isReady)
self.assertIsNone(self.analyzer.primaryScan)
self.assertEqual(OrderedDict(), self.analyzer.primaryScanCategories)
self.assertEqual({}, self.analyzer.kwConfig)
self.assertFalse(self.analyzer.analysisDone)
def test_analyzer_retrieves_all_categories_for_report(self):
self.analyzer._buildScanCategories()
self.assertEqual(len(self.analyzer.primaryScanCategories), 4)
def test_analyzer_still_not_ready_after_just_buildScanCategories(self):
self.analyzer._buildScanCategories()
self.assertFalse(self.analyzer.isReady)
def test_analyzer_cannot_call_buildScanCategories_twice(self):
self.analyzer._buildScanCategories()
with self.assertRaises(ReportAnalysisError):
self.analyzer._buildScanCategories()
def test_analyzer_builds_categories_with_OD_for_licenses(self):
self.analyzer._buildScanCategories()
for cat in self.analyzer.primaryScanCategories.values():
self.assertIs(type(cat.licensesSorted), OrderedDict)
c1 = self.analyzer.primaryScanCategories[1]
l1 = c1.licensesSorted[1]
self.assertEqual(l1._id, 1)
self.assertEqual(l1.name, "DoAnything")
l4 = c1.licensesSorted[4]
self.assertEqual(l4._id, 4)
self.assertEqual(l4.name, "DoAnythingNoncommercial")
# and it doesn't have licenses from a different category
with self.assertRaises(KeyError):
c1.licensesSorted[2]
def test_analyzer_builds_categories_OD_in_sort_order(self):
self.analyzer._buildScanCategories()
(c_id1, c1) = self.analyzer.primaryScanCategories.popitem(last=False)
self.assertEqual(3, c_id1)
self.assertEqual("blah category", c1.name)
(c_id2, c2) = self.analyzer.primaryScanCategories.popitem(last=False)
self.assertEqual(2, c_id2)
self.assertEqual("cat", c2.name)
(c_id3, c3) = self.analyzer.primaryScanCategories.popitem(last=False)
self.assertEqual(1, c_id3)
self.assertEqual("a category", c3.name)
##### Analyzer adding files tests
def test_analyzer_cannot_add_files_before_categories_are_built(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer._addFiles(scan_id=1)
def test_analyzer_cannot_add_files_from_invalid_scan_id(self):
self.analyzer._buildScanCategories()
with self.assertRaises(ReportAnalysisError):
self.analyzer._addFiles(scan_id=187)
def test_analyzer_can_add_files_to_categories_for_a_scan(self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
# check category 1, with one license and NO FILES
(c_id1, c1) = self.analyzer.primaryScanCategories.popitem(last=False)
self.assertEqual("blah category", c1.name)
(l_id11, l11) = c1.licensesSorted.popitem(last=False)
self.assertEqual("293PageEULA", l11.name)
self.assertEqual([], list(l11.filesSorted.items()))
# check category 2, with one license and three files
(c_id2, c2) = self.analyzer.primaryScanCategories.popitem(last=False)
self.assertEqual("cat", c2.name)
(l_id21, l21) = c2.licensesSorted.popitem(last=False)
self.assertEqual("HarshEULA", l21.name)
self.assertEqual(len(l21.filesSorted), 3)
(f_id211, f211) = l21.filesSorted.popitem(last=False)
self.assertEqual(f_id211, 3)
self.assertEqual(f211.path, "/tmp/f3")
self.assertEqual(f211.findings, {})
(f_id212, f212) = l21.filesSorted.popitem(last=False)
self.assertEqual(f_id212, 4)
self.assertEqual(f212.path, "/tmp/f4")
self.assertEqual(f212.findings, {})
(f_id216, f216) = l21.filesSorted.popitem(last=False)
self.assertEqual(f_id216, 6)
self.assertEqual(f216.path, "/tmp/nolic/vendor/whatever")
self.assertEqual(f216.findings, {}) # haven't run analysis yet
# check category 3, with two licenses and two files
(c_id3, c3) = self.analyzer.primaryScanCategories.popitem(last=False)
self.assertEqual("a category", c3.name)
(l_id31, l31) = c3.licensesSorted.popitem(last=False)
self.assertEqual("DoAnything", l31.name)
self.assertEqual(len(l31.filesSorted), 1)
(f_id311, f311) = l31.filesSorted.popitem(last=False)
self.assertEqual(f_id311, 1)
self.assertEqual(f311.path, "/tmp/f1")
self.assertEqual(f311.findings, {})
(l_id32, l32) = c3.licensesSorted.popitem(last=False)
self.assertEqual("DoAnythingNoncommercial", l32.name)
self.assertEqual(len(l32.filesSorted), 1)
(f_id321, f321) = l32.filesSorted.popitem(last=False)
self.assertEqual(f_id321, 2)
self.assertEqual(f321.path, "/tmp/f2")
self.assertEqual(f321.findings, {})
def test_can_add_files_from_multiple_scans(self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._addFiles(scan_id=3)
# check category 1, with one license and still NO FILES
(c_id1, c1) = self.analyzer.primaryScanCategories.popitem(last=False)
self.assertEqual("blah category", c1.name)
(l_id11, l11) = c1.licensesSorted.popitem(last=False)
self.assertEqual("293PageEULA", l11.name)
self.assertEqual([], list(l11.filesSorted.items()))
# check category 2, with one license and now five files
(c_id2, c2) = self.analyzer.primaryScanCategories.popitem(last=False)
self.assertEqual("cat", c2.name)
(l_id21, l21) = c2.licensesSorted.popitem(last=False)
self.assertEqual("HarshEULA", l21.name)
self.assertEqual(len(l21.filesSorted), 5)
(f_id211, f211) = l21.filesSorted.popitem(last=False)
self.assertEqual(f_id211, 3)
self.assertEqual(f211.path, "/tmp/f3")
self.assertEqual(f211.findings, {})
(f_id212, f212) = l21.filesSorted.popitem(last=False)
self.assertEqual(f_id212, 4)
self.assertEqual(f212.path, "/tmp/f4")
self.assertEqual(f212.findings, {})
(f_id216, f216) = l21.filesSorted.popitem(last=False)
self.assertEqual(f_id216, 6)
self.assertEqual(f216.path, "/tmp/nolic/vendor/whatever")
self.assertEqual(f216.findings, {}) # haven't run analysis yet
(f3_id211, f3211) = l21.filesSorted.popitem(last=False)
self.assertEqual(f3_id211, 33)
self.assertEqual(f3211.path, "/scan3/tmp/f3")
self.assertEqual(f3211.findings, {})
(f3_id212, f3212) = l21.filesSorted.popitem(last=False)
self.assertEqual(f3_id212, 34)
self.assertEqual(f3212.path, "/scan3/tmp/f4")
self.assertEqual(f3212.findings, {})
def test_analyzer_marks_cats_and_lics_that_have_files(self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
# check categories
c1 = self.analyzer._getCategory(category_id=1)
self.assertTrue(c1.hasFiles)
c2 = self.analyzer._getCategory(category_id=2)
self.assertTrue(c2.hasFiles)
c3 = self.analyzer._getCategory(category_id=3)
self.assertFalse(c3.hasFiles)
# check licenses
l2 = self.analyzer._getLicense(license_id=2)
self.assertTrue(l2.hasFiles)
l4 = self.analyzer._getLicense(license_id=4)
self.assertTrue(l4.hasFiles)
l7 = self.analyzer._getLicense(license_id=7)
self.assertFalse(l7.hasFiles)
##### Analyzer config params tests
def test_analyzer_can_take_optional_config_params(self):
configDict = {
"analyze-extensions": "yes",
"analyze-extensions-list": "json;jpeg;png"
}
newAnalyzer = Analyzer(db=self.db, config=configDict)
self.assertEqual("yes", newAnalyzer.kwConfig["analyze-extensions"])
self.assertEqual("json;jpeg;png",
newAnalyzer.kwConfig["analyze-extensions-list"])
def test_can_get_analyze_final_config_from_analyzer(self):
self.db.setConfigValue(key="analyze-extensions", value="yes")
exts = self.analyzer._getFinalConfigValue(key="analyze-extensions")
self.assertEqual(exts, "yes")
def test_can_override_db_config_in_analyzer_final_config(self):
self.db.setConfigValue(key="analyze-extensions", value="yes")
newAnalyzer = Analyzer(db=self.db, config={"analyze-extensions": "no"})
exts = newAnalyzer._getFinalConfigValue(key="analyze-extensions")
self.assertEqual(exts, "no")
##### Analyzer main analysis function tests
def test_analyzer_cannot_analyze_before_categories_are_built(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer._runAnalysis()
@mock.patch('slm.reports.analysis.Analyzer._analyzeExtensions')
def test_analyzer_runs_extensions_check_if_set(self, ext_mock):
self.db.setConfigValue(key="analyze-extensions", value="yes")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
ext_mock.assert_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeExtensions')
def test_analyzer_does_not_run_extensions_check_if_not_set(self, ext_mock):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
ext_mock.assert_not_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeExtensions')
def test_analyzer_does_not_run_extensions_check_if_set_to_no(
self, ext_mock):
self.db.setConfigValue(key="analyze-extensions", value="no")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
ext_mock.assert_not_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeThirdparty')
def test_analyzer_runs_thirparty_check_if_set(self, tp_mock):
self.db.setConfigValue(key="analyze-thirdparty", value="yes")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
tp_mock.assert_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeThirdparty')
def test_analyzer_does_not_run_thirdparty_check_if_not_set(self, tp_mock):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
tp_mock.assert_not_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeEmptyFile')
def test_analyzer_runs_emptyfile_check_if_set(self, empty_mock):
self.db.setConfigValue(key="analyze-emptyfile", value="yes")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
empty_mock.assert_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeEmptyFile')
def test_analyzer_does_not_run_emptyfile_check_if_not_set(
self, empty_mock):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
empty_mock.assert_not_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeExcludePathPrefix')
def test_analyzer_runs_excludePathPrefix_if_set(self, path_mock):
self.db.setConfigValue(key="analyze-exclude-path-prefix", value="yes")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
path_mock.assert_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeExcludePathPrefix')
def test_analyzer_does_not_run_excludePathPrefix_if_not_set(
self, path_mock):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
path_mock.assert_not_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeExcludeEmptyCatsAndLics'
)
def test_analyzer_runs_excludeEmptyCatsAndLics_if_set(self, ecl_mock):
self.db.setConfigValue(key="analyze-exclude-empty-cats-and-lics",
value="yes")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
ecl_mock.assert_called()
@mock.patch('slm.reports.analysis.Analyzer._analyzeExcludeEmptyCatsAndLics'
)
def test_analyzer_does_not_run_excludeEmptyCatsAndLics_if_not_set(
self, ecl_mock):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
ecl_mock.assert_not_called()
##### getter helper tests
def test_analyzer_cannot_get_specific_cat_before_building_categories(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer._getCategory(category_id=3)
def test_analyzer_returns_none_if_getting_specific_cat_by_unknown_id(self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
cat = self.analyzer._getCategory(category_id=77)
self.assertIsNone(cat)
def test_analyzer_can_get_specific_category_by_id(self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
cat = self.analyzer._getCategory(category_id=3)
self.assertEqual("blah category", cat.name)
def test_analyzer_cannot_get_specific_lic_before_building_categories(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer._getLicense(license_id=6)
def test_analyzer_returns_none_if_getting_specific_lic_by_unknown_id(self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
lic = self.analyzer._getLicense(license_id=77)
self.assertIsNone(lic)
def test_analyzer_can_get_specific_license_by_id(self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
lic = self.analyzer._getLicense(license_id=6)
self.assertEqual("Also no license found", lic.name)
def test_analyzer_cannot_get_specific_file_before_building_categories(
self):
with self.assertRaises(ReportAnalysisError):
self.analyzer._getFile(7)
def test_analyzer_returns_none_if_getting_specific_file_by_unknown_id(
self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
f = self.analyzer._getFile(77)
self.assertIsNone(f)
def test_analyzer_can_get_specific_file_by_id(self):
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
f = self.analyzer._getFile(file_id=7)
self.assertEqual("/tmp/nolic/vendor/image-both.png", f.path)
##### file extension analysis tests
def test_can_parse_file_extension_string(self):
extString = "json;jpeg;png;gif"
self.db.setConfigValue(key="analyze-extensions-list", value=extString)
exts = self.analyzer._parseExtConfig()
# extensions are sorted in alphabetical order
self.assertEqual(["gif", "jpeg", "json", "png"], exts)
def test_extension_string_parser_returns_empty_list_if_not_set(self):
exts = self.analyzer._parseExtConfig()
self.assertEqual([], exts)
def test_ignore_whitespace_in_file_extension_string(self):
extString = " json ; jpeg; png ;gif "
self.db.setConfigValue(key="analyze-extensions-list", value=extString)
exts = self.analyzer._parseExtConfig()
# extensions are sorted in alphabetical order
self.assertEqual(["gif", "jpeg", "json", "png"], exts)
def test_files_with_extension_match_get_extra_flag_and_others_dont(self):
self.db.setConfigValue(key="analyze-extensions", value="yes")
extString = "json;jpeg;png;gif"
self.db.setConfigValue(key="analyze-extensions-list", value=extString)
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
# check specific files
self._checkFileExtFindingIsNone(1)
self._checkFileExtFindingIsNone(2)
self._checkFileExtFindingIsNone(3)
self._checkFileExtFindingIsNone(4)
self._checkFileExtFindingIsYes(5)
self._checkFileExtFindingIsNone(6)
self._checkFileExtFindingIsYes(7)
self._checkFileExtFindingIsNone(8)
self._checkFileExtFindingIsNone(9)
self._checkFileExtFindingIsYes(10)
self._checkFileExtFindingIsYes(11)
##### empty file analysis tests
def test_empty_files_get_extra_flag_and_others_dont(self):
self.db.setConfigValue(key="analyze-emptyfile", value="yes")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
# check specific files
self._checkFileEmptyFindingIsNone(1)
self._checkFileEmptyFindingIsNone(2)
self._checkFileEmptyFindingIsNone(3)
self._checkFileEmptyFindingIsNone(4)
self._checkFileEmptyFindingIsNone(5)
self._checkFileEmptyFindingIsNone(6)
self._checkFileEmptyFindingIsNone(7)
self._checkFileEmptyFindingIsYes(8)
self._checkFileEmptyFindingIsYes(9)
self._checkFileEmptyFindingIsYes(10)
self._checkFileEmptyFindingIsYes(11)
##### third party directory analysis tests
def test_can_parse_thirdparty_dirs_string(self):
dirString = "vendor;thirdparty;third-party"
self.db.setConfigValue(key="analyze-thirdparty-dirs", value=dirString)
dirs = self.analyzer._parseDirConfig()
# directories are sorted in alphabetical order
self.assertEqual(["third-party", "thirdparty", "vendor"], dirs)
def test_directories_string_parser_returns_empty_list_if_not_set(self):
dirs = self.analyzer._parseDirConfig()
self.assertEqual([], dirs)
def test_ignore_whitespace_in_thirdparty_dirs_string(self):
dirString = " thirdparty ; vendor ;third-party "
self.db.setConfigValue(key="analyze-thirdparty-dirs", value=dirString)
dirs = self.analyzer._parseDirConfig()
# directories are sorted in alphabetical order
self.assertEqual(["third-party", "thirdparty", "vendor"], dirs)
def test_files_with_directory_match_get_extra_flag_and_others_dont(self):
self.db.setConfigValue(key="analyze-thirdparty", value="yes")
dirString = "vendor;thirdparty;third-party"
self.db.setConfigValue(key="analyze-thirdparty-dirs", value=dirString)
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
# check specific files
self._checkFileDirFindingIsNone(1)
self._checkFileDirFindingIsNone(2)
self._checkFileDirFindingIsNone(3)
self._checkFileDirFindingIsNone(4)
self._checkFileDirFindingIsNone(5)
self._checkFileDirFindingIsYes(6)
self._checkFileDirFindingIsYes(7)
self._checkFileDirFindingIsNone(8)
self._checkFileDirFindingIsYes(9)
self._checkFileDirFindingIsNone(10)
self._checkFileDirFindingIsYes(11)
##### exclude path prefix analysis tests
def test_analyzer_can_exclude_common_path_prefix(self):
self.db.setConfigValue(key="analyze-exclude-path-prefix", value="yes")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
# check that file paths have changed
for cat in self.analyzer.primaryScanCategories.values():
for lic in cat.licensesSorted.values():
for file in lic.filesSorted.values():
self.assertNotIn("tmp", file.path)
##### exclude empty categories and licenses analysis tests
def test_analyzer_can_exclude_empty_cats_and_lics(self):
self.db.setConfigValue(key="analyze-exclude-empty-cats-and-lics",
value="yes")
self.analyzer._buildScanCategories()
self.analyzer._addFiles(scan_id=1)
self.analyzer._runAnalysis()
# check that empty cats are gone
with self.assertRaises(KeyError):
na = self.analyzer.primaryScanCategories[3]
# and check that empty licenses are gone
cat4 = self.analyzer.primaryScanCategories[4]
with self.assertRaises(KeyError):
na = cat4.licensesSorted[7]
# and check that cats and licenses with files are still present
self.assertEqual(cat4.name, "no lic category")
self.assertEqual(cat4.licensesSorted[6].name, "Also no license found")
##### main analysis function tests
def test_can_analyze_and_get_results(self):
self.db.setConfigValue(key="analyze-extensions", value="yes")
results = self.analyzer.runAnalysis(scan_id=1)
self.assertEqual(OrderedDict, type(results))
self.assertEqual(4, len(results.items()))
cat1 = results[1]
self.assertEqual("a category", cat1.name)
self.assertTrue(self.analyzer.analysisDone)
def test_can_analyze_and_get_results_for_multiple_scans(self):
self.db.setConfigValue(key="analyze-extensions", value="yes")
results = self.analyzer.runAnalysis(scan_ids=[1, 3])
self.assertEqual(OrderedDict, type(results))
self.assertEqual(4, len(results.items()))
cat1 = results[1]
self.assertEqual("a category", cat1.name)
self.assertTrue(self.analyzer.analysisDone)
def test_cannot_analyze_invalid_scan_id(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer.runAnalysis(scan_id=187)
def test_cannot_analyze_invalid_scan_id_in_multiple_ids(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer.runAnalysis(scan_ids=[1, 3, 187])
def test_cannot_analyze_with_singular_and_multiple_scan_ids(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer.runAnalysis(scan_id=1, scan_ids=[1, 3])
##### create list results from OrderedDict results
def test_cannot_get_list_results_before_results_are_generated(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer.getResultsAsList()
self.analyzer._buildScanCategories()
with self.assertRaises(ReportAnalysisError):
self.analyzer.getResultsAsList()
self.analyzer._addFiles(scan_id=1)
with self.assertRaises(ReportAnalysisError):
self.analyzer.getResultsAsList()
def test_can_get_list_results_from_OrderedDict_results(self):
self.db.setConfigValue(key="analyze-exclude-empty-cats-and-lics",
value="yes")
self.analyzer.runAnalysis(scan_id=1)
list_results = self.analyzer.getResultsAsList()
# check top-level list of categories
# should be sorted by order and should exclude empty categories
self.assertIsInstance(list_results, list)
self.assertEqual(len(list_results), 3)
cat2 = list_results[0]
self.assertIsInstance(cat2, Category)
self.assertEqual(cat2.name, "cat")
self.assertEqual(cat2._id, 2)
# check licenses within a category
# should be sorted alphabetically and should exclude empty licenses
cat4 = list_results[2]
self.assertIsInstance(cat4.licenses, list)
self.assertEqual(len(cat4.licenses), 2)
lic6 = cat4.licenses[0]
self.assertIsInstance(lic6, License)
self.assertEqual(lic6.name, "Also no license found")
self.assertEqual(lic6._id, 6)
# check files within a license
self.assertIsInstance(lic6.files, list)
self.assertEqual(len(lic6.files), 5)
self.assertIsInstance(lic6.files[0], File)
##### Split string into separate scan IDs
def test_can_split_string_with_individual_scan_ids(self):
sstring = "1,3,10,6,2"
scan_ids = self.analyzer.splitScanIDString(sstring)
# scan IDs should be included regardless of whether they're actual
# scans, and should be sorted
self.assertEqual(scan_ids, [1, 2, 3, 6, 10])
def test_can_split_string_with_range_of_scan_ids(self):
sstring = "1,8,4-6,9"
scan_ids = self.analyzer.splitScanIDString(sstring)
self.assertEqual(scan_ids, [1, 4, 5, 6, 8, 9])
def test_raises_error_if_invalid_text_found(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("oops")
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("1,oops")
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("oops,3")
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("1,oops,3")
def test_ids_not_duplicated_if_included_repeatedly(self):
sstring = "1,1-4,1,4,7-8,7-8,7"
scan_ids = self.analyzer.splitScanIDString(sstring)
self.assertEqual(scan_ids, [1, 2, 3, 4, 7, 8])
def test_raises_error_if_range_is_backwards(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("9-2")
def test_raises_error_if_range_format_is_invalid(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("9-")
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("9-10-15")
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("-9")
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("-")
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("9--12")
with self.assertRaises(ReportAnalysisError):
self.analyzer.splitScanIDString("-9-10")
def test_helper_returns_int_from_string(self):
i = self.analyzer._getIntFromStringScanID("3")
self.assertEqual(i, 3)
def test_helper_raises_report_analysis_error_for_non_integers(self):
with self.assertRaises(ReportAnalysisError):
self.analyzer._getIntFromStringScanID("")
with self.assertRaises(ReportAnalysisError):
self.analyzer._getIntFromStringScanID("a")
with self.assertRaises(ReportAnalysisError):
self.analyzer._getIntFromStringScanID("3-")
with self.assertRaises(ReportAnalysisError):
self.analyzer._getIntFromStringScanID("3a")
with self.assertRaises(ReportAnalysisError):
self.analyzer._getIntFromStringScanID(",")
with self.assertRaises(ReportAnalysisError):
self.analyzer._getIntFromStringScanID("3:")
def test_can_open_existing_db(self):
# create in temporary directory on disk, so we can re-open DB
# (testfixtures will wipe out the directory at end of test)
with TempDirectory() as td:
dbPath = os.path.join(td.path, "tmp.db")
dbnew = ProjectDB()
dbnew.createDB(dbPath)
dbnew.initializeDBTables()
dbnew.closeDB()
# and reopen it
dbnew.openDB(dbPath)
self.assertTrue(dbnew.isOpened())
self.assertTrue(dbnew.isInitialized())
dbnew.closeDB()
class DBSubprojectUnitTestSuite(unittest.TestCase):
"""spdxLicenseManager unit test suite for subproject data in DB."""
def setUp(self):
# create and initialize an in-memory database
self.db = ProjectDB()
self.db.createDB(":memory:")
self.db.initializeDBTables()
# insert sample data
self.insertSampleSubprojectData()
def tearDown(self):
self.db.closeDB()
self.db = None
def insertSampleSubprojectData(self):
subprojects = [
Subproject(_id=1,
name="sub1",
spdx_search="sub1",
desc="subproject 1"),
Subproject(_id=2,
name="subX",
spdx_search="subX",
desc="subproject XYZ"),
Subproject(_id=3,
name="subC",
spdx_search="subC",
desc="subproject B"),
]
self.db.session.bulk_save_objects(subprojects)
self.db.session.commit()
##### Test cases below
def test_can_retrieve_all_subproject_names_and_descs(self):
subprojects = self.db.getSubprojectsAll()
self.assertIsInstance(subprojects, list)
self.assertEqual(len(subprojects), 3)
self.assertEqual(subprojects[0]._id, 1)
self.assertEqual(subprojects[0].name, "sub1")
self.assertEqual(subprojects[0].desc, "subproject 1")
self.assertEqual(subprojects[0].spdx_search, "sub1")
def test_all_subprojects_are_sorted_by_name(self):
subprojects = self.db.getSubprojectsAll()
self.assertEqual(subprojects[0].name, "sub1")
self.assertEqual(subprojects[1].name, "subC")
self.assertEqual(subprojects[2].name, "subX")
def test_can_retrieve_one_subproject_by_id(self):
subproject = self.db.getSubproject(_id=2)
self.assertEqual(subproject.name, "subX")
self.assertEqual(subproject.desc, "subproject XYZ")
self.assertEqual(subproject.spdx_search, "subX")
def test_can_retrieve_one_subproject_by_name(self):
subproject = self.db.getSubproject(name="subC")
self.assertEqual(subproject._id, 3)
self.assertEqual(subproject.desc, "subproject B")
self.assertEqual(subproject.spdx_search, "subC")
def test_cannot_retrieve_subproject_by_both_name_and_id(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getSubproject(_id=3, name="subC")
def test_cannot_retrieve_subproject_without_either_name_or_id(self):
with self.assertRaises(ProjectDBQueryError):
self.db.getSubproject()
def test_cannot_retrieve_subproject_with_positional_args(self):
with self.assertRaises(TypeError):
self.db.getSubproject("subC")
def test_returns_none_if_subproject_not_found_by_id(self):
subproject = self.db.getSubproject(_id=17)
self.assertIsNone(subproject)
def test_returns_none_if_subproject_not_found_by_name(self):
subproject = self.db.getSubproject(name="noSuchSubproject")
self.assertIsNone(subproject)
def test_can_add_and_retrieve_subproject(self):
subproject_id = self.db.addSubproject("newsub",
"subproject new",
spdx_search="newsub")
# confirm that we now have four subprojects
subprojects = self.db.getSubprojectsAll()
self.assertEqual(len(subprojects), 4)
# and confirm that we can retrieve this one by name
subproject = self.db.getSubproject(name="newsub")
self.assertEqual(subproject.name, "newsub")
self.assertEqual(subproject.desc, "subproject new")
self.assertEqual(subproject.spdx_search, "newsub")
# and confirm that we can retrieve this one by id
subproject = self.db.getSubproject(_id=4)
self.assertEqual(subproject.name, "newsub")
self.assertEqual(subproject.desc, "subproject new")
self.assertEqual(subproject.spdx_search, "newsub")
def test_can_add_subproject_without_providing_spdx_search_name(self):
subproject_id = self.db.addSubproject("newsub", "subproject new")
# confirm that we now have four subprojects
subprojects = self.db.getSubprojectsAll()
self.assertEqual(len(subprojects), 4)
# and confirm that we can retrieve this one by name
subproject = self.db.getSubproject(name="newsub")
self.assertEqual(subproject.name, "newsub")
self.assertEqual(subproject.desc, "subproject new")
self.assertEqual(subproject.spdx_search, "newsub")
# and confirm that we can retrieve this one by id
subproject = self.db.getSubproject(_id=4)
self.assertEqual(subproject.name, "newsub")
self.assertEqual(subproject.desc, "subproject new")
self.assertEqual(subproject.spdx_search, "newsub")
def test_can_start_adding_but_rollback_subproject(self):
subproject_id = self.db.addSubproject(name="newsub",
spdx_search="newsub",
desc="will rollback",
commit=False)
self.db.rollback()
# confirm that we still only have three subprojects
subprojects = self.db.getSubprojectsAll()
self.assertEqual(len(subprojects), 3)
# and confirm that this subproject ID doesn't exist in database
subproject = self.db.getSubproject(_id=subproject_id)
self.assertIsNone(subproject)
def test_can_start_adding_and_then_commit_subprojects(self):
s2_id = self.db.addSubproject(name="news2",
spdx_search="news2",
desc="new sp 2",
commit=False)
s1_id = self.db.addSubproject(name="news1",
spdx_search="news1",
desc="new sp 1",
commit=False)
self.db.commit()
# confirm that we now have five subprojects
subprojects = self.db.getSubprojectsAll()
self.assertEqual(len(subprojects), 5)
def test_can_edit_subproject_spdx_string(self):
self.db.changeSubprojectSPDXSearch(name="subX",
spdx_search="subXSpecial")
subproject = self.db.getSubproject(name="subX")
self.assertEqual(subproject.name, "subX")
self.assertEqual(subproject.spdx_search, "subXSpecial")
def test_cannot_edit_subproject_spdx_search_for_nonexistent_name(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeSubprojectSPDXSearch(name="invalid",
spdx_search="this will fail")
def test_cannot_change_subproject_spdx_search_to_existing_search(self):
with self.assertRaises(ProjectDBUpdateError):
self.db.changeSubprojectSPDXSearch(name="sub1", spdx_search="subX")
def test_open_db_fails_if_invalid_magic_number(self):
# create in temporary directory on disk, so we can re-open it
# (testfixtures will wipe out the directory at end of test)
with TempDirectory() as td:
dbPath = os.path.join(td.path, "tmp.db")
dbnew = ProjectDB()
dbnew.createDB(dbPath)
dbnew.initializeDBTables()
# set invalid magic number
query = dbnew.session.query(Config).filter(Config.key == "magic")
query.update({Config.value: "invalidMagic"})
dbnew.session.commit()
dbnew.closeDB()
# and reopen it
with self.assertRaises(ProjectDBConfigError):
dbnew.openDB(dbPath)
self.assertFalse(dbnew.isOpened())
self.assertFalse(dbnew.isInitialized())
