def get_installed_packages(root_dir,
distro='debian',
detect_licenses=False,
**kwargs):
"""
Yield installed Package objects given a ``root_dir`` rootfs directory.
"""
base_status_file_loc = os.path.join(root_dir, 'var/lib/dpkg/status')
base_statusd_loc = os.path.join(root_dir, 'var/lib/dpkg/status.d/')
if os.path.exists(base_status_file_loc):
var_lib_dpkg_info_dir = os.path.join(root_dir, 'var/lib/dpkg/info/')
# guard from recursive import
from packagedcode import debian_copyright
for package in parse_status_file(base_status_file_loc, distro=distro):
package.populate_installed_files(var_lib_dpkg_info_dir)
if detect_licenses:
copyright_location = package.get_copyright_file_path(root_dir)
dc = debian_copyright.parse_copyright_file(copyright_location)
if dc:
package.declared_license = dc.get_declared_license(
filter_duplicates=True,
skip_debian_packaging=True,
)
package.license_expression = dc.get_license_expression(
skip_debian_packaging=True,
simplify_licenses=True,
)
package.copyright = dc.get_copyright(
skip_debian_packaging=True,
unique_copyrights=True,
)
yield package
elif os.path.exists(base_statusd_loc):
for root, dirs, files in os.walk(base_statusd_loc):
for f in files:
status_file_loc = os.path.join(root, f)
for package in parse_status_file(status_file_loc,
distro=distro):
yield package
def check_expected_parse_copyright_file(
test_loc,
expected_loc,
regen=False,
with_details=False,
):
"""
Check copyright parsing of `test_loc` location against an expected JSON file
at `expected_loc` location. Regen the expected file if `regen` is True.
"""
if with_details:
skip_debian_packaging = True
simplify_licenses = True
unique = True
else:
skip_debian_packaging = False
simplify_licenses = False
unique = False
parsed = debian_copyright.parse_copyright_file(
copyright_file=test_loc,
skip_debian_packaging=skip_debian_packaging,
simplify_licenses=simplify_licenses,
unique=unique,
)
result = saneyaml.dump(list(parsed))
if regen:
with io.open(expected_loc, 'w', encoding='utf-8') as reg:
reg.write(result)
with io.open(expected_loc, encoding='utf-8') as ex:
expected = ex.read()
if result != expected:
expected = '\n'.join([
'file://' + test_loc,
'file://' + expected_loc,
expected
])
assert result == expected
def check_expected(test_loc, expected_loc, regen=False):
"""
Check copyright parsing of `test_loc` location against an expected JSON file
at `expected_loc` location. Regen the expected file if `regen` is True.
"""
result = saneyaml.dump(
list(debian_copyright.parse_copyright_file(test_loc)))
if regen:
with io.open(expected_loc, 'w', encoding='utf-8') as reg:
reg.write(result)
with io.open(expected_loc, encoding='utf-8') as ex:
expected = ex.read()
if expected != result:
expected = '\n'.join(
['file://' + test_loc, 'file://' + expected_loc, expected])
assert expected == result
def check_expected(test_loc, expected_loc, regen=False):
"""
Check copyright parsing of `test_loc` location against an expected JSON file
at `expected_loc` location. Regen the expected file if `regen` is True.
"""
result = list(debian_copyright.parse_copyright_file(test_loc))
if regen:
with open(expected_loc, 'w') as ex:
ex.write(json.dumps(result, indent=2))
with open(expected_loc) as ex:
expected = json.loads(ex.read())
if expected != result:
expected = [
('copyright file', 'file://' + test_loc),
('expected file', 'file://' + expected_loc),
] + expected
assert expected == result
def check_expected_parse_copyright_file(
test_loc,
expected_loc,
regen=False,
simplified=False,
_licensing=Licensing(),
):
'''
Check copyright parsing of `test_loc` location against an expected JSON file
at `expected_loc` location. Regen the expected file if `regen` is True.
'''
if simplified:
filter_duplicates = True
skip_debian_packaging = True
simplify_licenses = True
unique_copyrights = True
else:
filter_duplicates = False
skip_debian_packaging = False
simplify_licenses = False
unique_copyrights = False
try:
dc = debian_copyright.parse_copyright_file(
location=test_loc,
check_consistency=False,
)
declared_license = dc.get_declared_license(
filter_duplicates=filter_duplicates,
skip_debian_packaging=skip_debian_packaging,
)
license_expression = dc.get_license_expression(
skip_debian_packaging=skip_debian_packaging,
simplify_licenses=simplify_licenses,
)
license_expression_keys = set(_licensing.license_keys(license_expression))
copyrght = dc.get_copyright(
skip_debian_packaging=skip_debian_packaging,
unique_copyrights=unique_copyrights,
).strip()
primary_license = dc.primary_license
match_details = list(map(get_match_details, dc.license_matches))
results = {
'primary_license': primary_license,
'declared_license': declared_license,
'license_expression': license_expression,
'copyright': copyrght,
'matches': match_details,
}
if regen:
expected = results
with open(expected_loc, 'w') as res:
res.write(saneyaml.dump(results))
else:
with open(expected_loc) as ex:
expected = saneyaml.load(ex.read())
except Exception as e:
import traceback
files = [
'file://' + test_loc,
'file://' + expected_loc,
]
raise Exception(repr(e), traceback.format_exc(), files) from e
if (
not regen
and (saneyaml.dump(results) != saneyaml.dump(expected)
or 'unknown-license-reference' in license_expression_keys)
) :
res = {
'test_loc': f'file://{test_loc}',
'expected_loc': f'file://{expected_loc}',
}
res.update(results)
results = saneyaml.dump(res)
results = results.replace(
'unknown-license-reference',
'unknown-license-reference should not be detected',
)
assert results == saneyaml.dump(expected)