def build_index(licenses_db=None, licenses_data_dir=None, rules_data_dir=None):
"""
Return an index built from rules and licenses directories
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
from licensedcode.models import get_all_spdx_key_tokens
from licensedcode.models import get_license_tokens
from licensedcode.models import licenses_data_dir as ldd
from licensedcode.models import rules_data_dir as rdd
from licensedcode.models import load_licenses
from licensedcode.legalese import common_license_words
licenses_data_dir = licenses_data_dir or ldd
rules_data_dir = rules_data_dir or rdd
licenses_db = licenses_db or load_licenses(
licenses_data_dir=licenses_data_dir)
rules = get_rules(licenses_db=licenses_db, rules_data_dir=rules_data_dir)
legalese = common_license_words
spdx_tokens = set(get_all_spdx_key_tokens(licenses_db))
license_tokens = set(get_license_tokens())
return LicenseIndex(
rules,
_legalese=legalese,
_spdx_tokens=spdx_tokens,
_license_tokens=license_tokens,
)
def _load_index(_index_cache_file=index_cache_file):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(_index_cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
idx = LicenseIndex.loads(ifc.read())
return idx
def _load_index(_index_cache_file=index_cache_file):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(_index_cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
idx = LicenseIndex.loads(ifc.read())
return idx
def get_or_build_index_from_cache(force_clear=False):
"""
Return a LicenseIndex loaded from cache. If the index is stale or does not exist,
build a new index and caches it. Clear or purge the LicenseMatch cache as needed.
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
try:
# acquire lock and wait until timeout to get a lock or die
with yg.lockfile.FileLock(index_lock_file, timeout=LICENSE_INDEX_LOCK_TIMEOUT):
current_checksum = None
# if we have a saved cached index
if exists(tree_checksum_file) and exists(index_cache_file):
# load saved tree_checksum and compare with current tree_checksum
with open(tree_checksum_file, 'rb') as etcs:
existing_checksum = etcs.read()
current_checksum = tree_checksum()
if current_checksum == existing_checksum:
# The cache is consistent with the latest code and data:
# we load index from cache
with open(index_cache_file, 'rb') as ifc:
# Note: loads() is much (twice++???) faster than load()
idx = LicenseIndex.loads(ifc.read())
return idx
# Here, the cache is not consistent with the latest code and data:
# It is either stale or non-existing: we need to cleanup/regen
# regen the index
idx = LicenseIndex(get_rules())
with open(index_cache_file, 'wb') as ifc:
ifc.write(idx.dumps())
# save the new checksums tree
with open(tree_checksum_file, 'wb') as ctcs:
ctcs.write(current_checksum or tree_checksum())
return idx
except yg.lockfile.FileLockTimeout:
# TODO: unable to lock in a nicer way
raise
def load_index(cache_file, use_loads=False):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
try:
if use_loads:
return LicenseIndex.loads(ifc.read())
else:
return LicenseIndex.load(ifc)
except Exception as e:
import traceback
msg = (
'\n'
'ERROR: Failed to load license cache (the file may be corrupted ?).\n'
'Please delete "{cache_file}" and retry.\n'
'If the problem persists, copy this error message '
'and submit a bug report.\n'.format(**locals()))
msg += '\n' + traceback.format_exc()
raise Exception(msg)
def load_index(cache_file, use_loads=False):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
try:
if use_loads:
return LicenseIndex.loads(ifc.read())
else:
return LicenseIndex.load(ifc)
except:
ex_type, ex_msg, ex_traceback = sys.exc_info()
message = (
str(ex_msg) +
'\nERROR: Failed to load license cache (the file may be corrupted ?).\n'
'Please delete "{cache_file}" and retry.\n'
'If the problem persists, copy this error message '
'and submit a bug report.\n'.format(**locals()))
if py3:
raise ex_type(message).with_traceback(ex_traceback)
else:
six.reraise(ex_type, message, ex_traceback)
def get_or_build_index_from_cache():
"""
Return a LicenseIndex loaded from cache or build a new index and caches it.
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
try:
# acquire global lock file and wait until timeout to get a lock or die
with yg.lockfile.FileLock(index_lock_file, timeout=60 * 3):
# if we have a saved cached index
if exists(tree_checksum_file) and exists(index_cache_file):
# load saved tree_checksum and compare with current tree_checksum
with open(tree_checksum_file, 'rb') as etcs:
existing_checksum = etcs.read()
current_checksum = tree_checksum()
# if this cached index is current for the code and data
if current_checksum == existing_checksum:
# load index from cache
with open(index_cache_file, 'rb') as ifc:
idx = LicenseIndex.loads(ifc.read())
return idx
# here the cache is stale or non-existing: we need to regen the index
idx = LicenseIndex(get_rules())
with open(index_cache_file, 'wb') as ifc:
ifc.write(idx.dumps())
# and save the checksums
with open(tree_checksum_file, 'wb') as ctcs:
ctcs.write(tree_checksum())
return idx
except yg.lockfile.FileLockTimeout:
# handle unable to lock
raise
def load_index(cache_file):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
try:
return LicenseIndex.load(ifc)
except Exception as e:
msg = (
'ERROR: Failed to load license cache (the file may be corrupted ?).\n'
f'Please delete "{cache_file}" and retry.\n'
'If the problem persists, copy this error message '
'and submit a bug report at https://github.com/nexB/scancode-toolkit/issues/'
)
raise Exception(msg) from e
def build_index(licenses_db=None, licenses_data_dir=None, rules_data_dir=None):
"""
Return an index built from rules and licenses directories
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
from licensedcode.models import get_all_spdx_key_tokens
from licensedcode.models import licenses_data_dir as ldd
from licensedcode.models import rules_data_dir as rdd
licenses_data_dir = licenses_data_dir or ldd
rules_data_dir = rules_data_dir or rdd
licenses = licenses_db or build_licenses_db(
licenses_data_dir=licenses_data_dir)
spdx_tokens = set(get_all_spdx_key_tokens(licenses))
rules = get_rules(
licenses_data_dir=licenses_data_dir,
rules_data_dir=rules_data_dir)
return LicenseIndex(rules, _spdx_tokens=spdx_tokens)
def get_cached_index(cache_dir=scancode_cache_dir,
check_consistency=SCANCODE_DEV_MODE,
# used for testing only
timeout=LICENSE_INDEX_LOCK_TIMEOUT,
tree_base_dir=scancode_src_dir,
licenses_data_dir=None, rules_data_dir=None,):
"""
Return a LicenseIndex: either load a cached index or build and cache the
index.
- If the cache does not exist, a new index is built and cached.
- If `check_consistency` is True, the cache is checked for consistency and
rebuilt if inconsistent or stale.
- If `check_consistency` is False, the cache is NOT checked for consistency
If the cache files exist but ARE stale, the cache WILL NOT be rebuilt
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
from licensedcode.models import get_all_spdx_key_tokens
from licensedcode.models import licenses_data_dir as ldd
from licensedcode.models import rules_data_dir as rdd
licenses_data_dir = licenses_data_dir or ldd
rules_data_dir = rules_data_dir or rdd
lock_file, checksum_file, cache_file = get_license_cache_paths(cache_dir)
has_cache = exists(cache_file)
has_tree_checksum = exists(checksum_file)
# bypass check if no consistency check is needed
if has_cache and has_tree_checksum and not check_consistency:
return load_index(cache_file)
# here, we have no cache or we want a validity check: lock, check
# and build or rebuild as needed
try:
# acquire lock and wait until timeout to get a lock or die
with yg.lockfile.FileLock(lock_file, timeout=timeout):
current_checksum = None
# is the current cache consistent or stale?
if has_cache and has_tree_checksum:
# if we have a saved cached index
# load saved tree_checksum and compare with current tree_checksum
with open(checksum_file, 'rb') as etcs:
existing_checksum = etcs.read()
current_checksum = tree_checksum(tree_base_dir=tree_base_dir)
if current_checksum == existing_checksum:
# The cache is consistent with the latest code and data
# load and return
return load_index(cache_file)
# Here, the cache is not consistent with the latest code and
# data: It is either stale or non-existing: we need to
# rebuild the index and cache it
rules = get_rules(
licenses_data_dir=licenses_data_dir,
rules_data_dir=rules_data_dir)
license_db = get_licenses_db(licenses_data_dir=licenses_data_dir)
spdx_tokens = set(get_all_spdx_key_tokens(license_db))
idx = LicenseIndex(rules, _spdx_tokens=spdx_tokens)
with open(cache_file, 'wb') as ifc:
ifc.write(idx.dumps())
# save the new checksums tree
with open(checksum_file, 'wb') as ctcs:
ctcs.write(current_checksum
or tree_checksum(tree_base_dir=tree_base_dir))
return idx
except yg.lockfile.FileLockTimeout:
# TODO: handle unable to lock in a nicer way
raise
def get_or_build_index_through_cache(
check_consistency=DEV_MODE,
return_index=True,
# used for testing only
_tree_base_dir=src_dir,
_tree_checksum_file=tree_checksum_file,
_index_lock_file=index_lock_file,
_index_cache_file=index_cache_file,
_licenses_data_dir=None,
_rules_data_dir=None,
_timeout=LICENSE_INDEX_LOCK_TIMEOUT,
):
"""
Check and build or rebuild the LicenseIndex cache.
If the cache does not exist, a new index is built an cached.
Return the LicenseIndex if return_index is True.
If `check_consistency` is True, the cache is checked for consistency
and rebuilt if inconsistent or stale.
If `check_consistency` is False, the cache is NOT checked for consistency
If the cache files exist but stale, the cache WILL NOT be rebuilt
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
from licensedcode.models import licenses_data_dir
from licensedcode.models import rules_data_dir
_licenses_data_dir = _licenses_data_dir or licenses_data_dir
_rules_data_dir = _rules_data_dir or rules_data_dir
has_cache = exists(_index_cache_file)
has_tree_checksum = exists(_tree_checksum_file)
# bypass check if no consistency check is needed
if has_cache and has_tree_checksum and not check_consistency:
return return_index and _load_index(_index_cache_file)
# here, we have no cache or we want a validity check: lock, check
# and build or rebuild as needed
try:
# acquire lock and wait until timeout to get a lock or die
with yg.lockfile.FileLock(_index_lock_file, timeout=_timeout):
current_checksum = None
# is the current cache consistent or stale?
if has_cache and has_tree_checksum:
# if we have a saved cached index
# load saved tree_checksum and compare with current tree_checksum
with open(_tree_checksum_file, 'rb') as etcs:
existing_checksum = etcs.read()
current_checksum = tree_checksum(tree_base_dir=_tree_base_dir)
if current_checksum == existing_checksum:
# The cache is consistent with the latest code and data
# load and return
return return_index and _load_index(_index_cache_file)
# Here, the cache is not consistent with the latest code and
# data: It is either stale or non-existing: we need to
# rebuild the index and cache it
rules = get_rules(licenses_data_dir=_licenses_data_dir,
rules_data_dir=_rules_data_dir)
idx = LicenseIndex(rules)
with open(_index_cache_file, 'wb') as ifc:
ifc.write(idx.dumps())
# save the new checksums tree
with open(_tree_checksum_file, 'wb') as ctcs:
ctcs.write(current_checksum
or tree_checksum(tree_base_dir=_tree_base_dir))
return return_index and idx
except yg.lockfile.FileLockTimeout:
# TODO: handle unable to lock in a nicer way
raise
def get_or_build_index_through_cache(
check_consistency=DEV_MODE,
return_index=True,
# used for testing only
_tree_base_dir=src_dir,
_tree_checksum_file=tree_checksum_file,
_index_lock_file=index_lock_file,
_index_cache_file=index_cache_file,
_licenses_data_dir=None,
_rules_data_dir=None,
_timeout=LICENSE_INDEX_LOCK_TIMEOUT,
):
"""
Check and build or rebuild the LicenseIndex cache.
If the cache does not exist, a new index is built an cached.
Return the LicenseIndex if return_index is True.
If `check_consistency` is True, the cache is checked for consistency
and rebuilt if inconsistent or stale.
If `check_consistency` is False, the cache is NOT checked for consistency
If the cache files exist but stale, the cache WILL NOT be rebuilt
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
from licensedcode.models import licenses_data_dir
from licensedcode.models import rules_data_dir
_licenses_data_dir = _licenses_data_dir or licenses_data_dir
_rules_data_dir = _rules_data_dir or rules_data_dir
has_cache = exists(_index_cache_file)
has_tree_checksum = exists(_tree_checksum_file)
# bypass check if no consistency check is needed
if has_cache and has_tree_checksum and not check_consistency:
return return_index and _load_index(_index_cache_file)
# here, we have no cache or we want a validity check: lock, check
# and build or rebuild as needed
try:
# acquire lock and wait until timeout to get a lock or die
with yg.lockfile.FileLock(_index_lock_file, timeout=_timeout):
current_checksum = None
# is the current cache consistent or stale?
if has_cache and has_tree_checksum:
# if we have a saved cached index
# load saved tree_checksum and compare with current tree_checksum
with open(_tree_checksum_file, 'rb') as etcs:
existing_checksum = etcs.read()
current_checksum = tree_checksum(tree_base_dir=_tree_base_dir)
if current_checksum == existing_checksum:
# The cache is consistent with the latest code and data
# load and return
return return_index and _load_index(_index_cache_file)
# Here, the cache is not consistent with the latest code and
# data: It is either stale or non-existing: we need to
# rebuild the index and cache it
rules = get_rules(
licenses_data_dir=_licenses_data_dir,
rules_data_dir=_rules_data_dir)
idx = LicenseIndex(rules)
with open(_index_cache_file, 'wb') as ifc:
ifc.write(idx.dumps())
# save the new checksums tree
with open(_tree_checksum_file, 'wb') as ctcs:
ctcs.write(current_checksum or tree_checksum(tree_base_dir=_tree_base_dir))
return return_index and idx
except yg.lockfile.FileLockTimeout:
# TODO: handle unable to lock in a nicer way
raise
def get_cached_index(
licensedcode_cache_dir=licensedcode_cache_dir,
scancode_cache_dir=scancode_cache_dir,
check_consistency=SCANCODE_DEV_MODE,
# used for testing only
timeout=LICENSE_INDEX_LOCK_TIMEOUT,
tree_base_dir=scancode_src_dir,
licenses_data_dir=None,
rules_data_dir=None,
):
"""
Return a LicenseIndex: either load a cached index or build and cache the
index.
- If the cache does not exist, a new index is built and cached.
- If `check_consistency` is True, the cache is checked for consistency and
rebuilt if inconsistent or stale.
- If `check_consistency` is False, the cache is NOT checked for consistency
If the cache files exist but ARE stale, the cache WILL NOT be rebuilt
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
from licensedcode.models import get_all_spdx_key_tokens
from licensedcode.models import licenses_data_dir as ldd
from licensedcode.models import rules_data_dir as rdd
from scancode import lockfile
licenses_data_dir = licenses_data_dir or ldd
rules_data_dir = rules_data_dir or rdd
lock_file, checksum_file, cache_file = get_license_cache_paths(
licensedcode_cache_dir=licensedcode_cache_dir,
scancode_cache_dir=scancode_cache_dir,
)
has_cache = has_cache_index_file(cache_file)
# bypass check if no consistency check is needed
if has_cache and not check_consistency:
try:
return load_index(cache_file)
except Exception as e:
# work around some rare Windows quirks
import traceback
print(
'Inconsistent License index cache: checking and rebuilding index.'
)
print(str(e))
print(traceback.format_exc())
has_tree_checksum = os.path.exists(checksum_file)
# here, we have no cache or we want a validity check: lock, check
# and build or rebuild as needed
try:
# acquire lock and wait until timeout to get a lock or die
with lockfile.FileLock(lock_file).locked(timeout=timeout):
current_checksum = None
# is the current cache consistent or stale?
if has_cache and has_tree_checksum:
# if we have a saved cached index
# load saved tree_checksum and compare with current tree_checksum
with open(checksum_file) as etcs:
existing_checksum = etcs.read()
current_checksum = tree_checksum(tree_base_dir=tree_base_dir)
if current_checksum == existing_checksum:
# The cache is consistent with the latest code and data
# load and return
return load_index(cache_file)
# Here, the cache is not consistent with the latest code and
# data: It is either stale or non-existing: we need to
# rebuild the index and cache it
# FIXME: caching a pickle of this would be 10x times faster
license_db = get_licenses_db(licenses_data_dir=licenses_data_dir)
rules = get_rules(licenses_data_dir=licenses_data_dir,
rules_data_dir=rules_data_dir)
spdx_tokens = set(get_all_spdx_key_tokens(license_db))
idx = LicenseIndex(rules, _spdx_tokens=spdx_tokens)
with open(cache_file, 'wb') as ifc:
idx.dump(ifc)
# save the new tree checksum
current_checksum = tree_checksum(tree_base_dir=tree_base_dir)
with open(checksum_file, 'w') as ctcs:
ctcs.write(current_checksum)
return idx
except lockfile.LockTimeout:
# TODO: handle unable to lock in a nicer way
raise
