def build_index(licenses_db=None, licenses_data_dir=None, rules_data_dir=None): """ Return an index built from rules and licenses directories """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules from licensedcode.models import get_all_spdx_key_tokens from licensedcode.models import get_license_tokens from licensedcode.models import licenses_data_dir as ldd from licensedcode.models import rules_data_dir as rdd from licensedcode.models import load_licenses from licensedcode.legalese import common_license_words licenses_data_dir = licenses_data_dir or ldd rules_data_dir = rules_data_dir or rdd licenses_db = licenses_db or load_licenses( licenses_data_dir=licenses_data_dir) rules = get_rules(licenses_db=licenses_db, rules_data_dir=rules_data_dir) legalese = common_license_words spdx_tokens = set(get_all_spdx_key_tokens(licenses_db)) license_tokens = set(get_license_tokens()) return LicenseIndex( rules, _legalese=legalese, _spdx_tokens=spdx_tokens, _license_tokens=license_tokens, )
def _load_index(_index_cache_file=index_cache_file): """ Return a LicenseIndex loaded from cache. """ from licensedcode.index import LicenseIndex with open(_index_cache_file, 'rb') as ifc: # Note: weird but read() + loads() is much (twice++???) faster than load() idx = LicenseIndex.loads(ifc.read()) return idx
def _load_index(_index_cache_file=index_cache_file): """ Return a LicenseIndex loaded from cache. """ from licensedcode.index import LicenseIndex with open(_index_cache_file, 'rb') as ifc: # Note: weird but read() + loads() is much (twice++???) faster than load() idx = LicenseIndex.loads(ifc.read()) return idx
def get_or_build_index_from_cache(force_clear=False): """ Return a LicenseIndex loaded from cache. If the index is stale or does not exist, build a new index and caches it. Clear or purge the LicenseMatch cache as needed. """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules try: # acquire lock and wait until timeout to get a lock or die with yg.lockfile.FileLock(index_lock_file, timeout=LICENSE_INDEX_LOCK_TIMEOUT): current_checksum = None # if we have a saved cached index if exists(tree_checksum_file) and exists(index_cache_file): # load saved tree_checksum and compare with current tree_checksum with open(tree_checksum_file, 'rb') as etcs: existing_checksum = etcs.read() current_checksum = tree_checksum() if current_checksum == existing_checksum: # The cache is consistent with the latest code and data: # we load index from cache with open(index_cache_file, 'rb') as ifc: # Note: loads() is much (twice++???) faster than load() idx = LicenseIndex.loads(ifc.read()) return idx # Here, the cache is not consistent with the latest code and data: # It is either stale or non-existing: we need to cleanup/regen # regen the index idx = LicenseIndex(get_rules()) with open(index_cache_file, 'wb') as ifc: ifc.write(idx.dumps()) # save the new checksums tree with open(tree_checksum_file, 'wb') as ctcs: ctcs.write(current_checksum or tree_checksum()) return idx except yg.lockfile.FileLockTimeout: # TODO: unable to lock in a nicer way raise
def load_index(cache_file, use_loads=False): """ Return a LicenseIndex loaded from cache. """ from licensedcode.index import LicenseIndex with open(cache_file, 'rb') as ifc: # Note: weird but read() + loads() is much (twice++???) faster than load() try: if use_loads: return LicenseIndex.loads(ifc.read()) else: return LicenseIndex.load(ifc) except Exception as e: import traceback msg = ( '\n' 'ERROR: Failed to load license cache (the file may be corrupted ?).\n' 'Please delete "{cache_file}" and retry.\n' 'If the problem persists, copy this error message ' 'and submit a bug report.\n'.format(**locals())) msg += '\n' + traceback.format_exc() raise Exception(msg)
def load_index(cache_file, use_loads=False): """ Return a LicenseIndex loaded from cache. """ from licensedcode.index import LicenseIndex with open(cache_file, 'rb') as ifc: # Note: weird but read() + loads() is much (twice++???) faster than load() try: if use_loads: return LicenseIndex.loads(ifc.read()) else: return LicenseIndex.load(ifc) except: ex_type, ex_msg, ex_traceback = sys.exc_info() message = ( str(ex_msg) + '\nERROR: Failed to load license cache (the file may be corrupted ?).\n' 'Please delete "{cache_file}" and retry.\n' 'If the problem persists, copy this error message ' 'and submit a bug report.\n'.format(**locals())) if py3: raise ex_type(message).with_traceback(ex_traceback) else: six.reraise(ex_type, message, ex_traceback)
def get_or_build_index_from_cache(): """ Return a LicenseIndex loaded from cache or build a new index and caches it. """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules try: # acquire global lock file and wait until timeout to get a lock or die with yg.lockfile.FileLock(index_lock_file, timeout=60 * 3): # if we have a saved cached index if exists(tree_checksum_file) and exists(index_cache_file): # load saved tree_checksum and compare with current tree_checksum with open(tree_checksum_file, 'rb') as etcs: existing_checksum = etcs.read() current_checksum = tree_checksum() # if this cached index is current for the code and data if current_checksum == existing_checksum: # load index from cache with open(index_cache_file, 'rb') as ifc: idx = LicenseIndex.loads(ifc.read()) return idx # here the cache is stale or non-existing: we need to regen the index idx = LicenseIndex(get_rules()) with open(index_cache_file, 'wb') as ifc: ifc.write(idx.dumps()) # and save the checksums with open(tree_checksum_file, 'wb') as ctcs: ctcs.write(tree_checksum()) return idx except yg.lockfile.FileLockTimeout: # handle unable to lock raise
def load_index(cache_file): """ Return a LicenseIndex loaded from cache. """ from licensedcode.index import LicenseIndex with open(cache_file, 'rb') as ifc: # Note: weird but read() + loads() is much (twice++???) faster than load() try: return LicenseIndex.load(ifc) except Exception as e: msg = ( 'ERROR: Failed to load license cache (the file may be corrupted ?).\n' f'Please delete "{cache_file}" and retry.\n' 'If the problem persists, copy this error message ' 'and submit a bug report at https://github.com/nexB/scancode-toolkit/issues/' ) raise Exception(msg) from e
def build_index(licenses_db=None, licenses_data_dir=None, rules_data_dir=None): """ Return an index built from rules and licenses directories """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules from licensedcode.models import get_all_spdx_key_tokens from licensedcode.models import licenses_data_dir as ldd from licensedcode.models import rules_data_dir as rdd licenses_data_dir = licenses_data_dir or ldd rules_data_dir = rules_data_dir or rdd licenses = licenses_db or build_licenses_db( licenses_data_dir=licenses_data_dir) spdx_tokens = set(get_all_spdx_key_tokens(licenses)) rules = get_rules( licenses_data_dir=licenses_data_dir, rules_data_dir=rules_data_dir) return LicenseIndex(rules, _spdx_tokens=spdx_tokens)
def get_cached_index(cache_dir=scancode_cache_dir, check_consistency=SCANCODE_DEV_MODE, # used for testing only timeout=LICENSE_INDEX_LOCK_TIMEOUT, tree_base_dir=scancode_src_dir, licenses_data_dir=None, rules_data_dir=None,): """ Return a LicenseIndex: either load a cached index or build and cache the index. - If the cache does not exist, a new index is built and cached. - If `check_consistency` is True, the cache is checked for consistency and rebuilt if inconsistent or stale. - If `check_consistency` is False, the cache is NOT checked for consistency If the cache files exist but ARE stale, the cache WILL NOT be rebuilt """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules from licensedcode.models import get_all_spdx_key_tokens from licensedcode.models import licenses_data_dir as ldd from licensedcode.models import rules_data_dir as rdd licenses_data_dir = licenses_data_dir or ldd rules_data_dir = rules_data_dir or rdd lock_file, checksum_file, cache_file = get_license_cache_paths(cache_dir) has_cache = exists(cache_file) has_tree_checksum = exists(checksum_file) # bypass check if no consistency check is needed if has_cache and has_tree_checksum and not check_consistency: return load_index(cache_file) # here, we have no cache or we want a validity check: lock, check # and build or rebuild as needed try: # acquire lock and wait until timeout to get a lock or die with yg.lockfile.FileLock(lock_file, timeout=timeout): current_checksum = None # is the current cache consistent or stale? if has_cache and has_tree_checksum: # if we have a saved cached index # load saved tree_checksum and compare with current tree_checksum with open(checksum_file, 'rb') as etcs: existing_checksum = etcs.read() current_checksum = tree_checksum(tree_base_dir=tree_base_dir) if current_checksum == existing_checksum: # The cache is consistent with the latest code and data # load and return return load_index(cache_file) # Here, the cache is not consistent with the latest code and # data: It is either stale or non-existing: we need to # rebuild the index and cache it rules = get_rules( licenses_data_dir=licenses_data_dir, rules_data_dir=rules_data_dir) license_db = get_licenses_db(licenses_data_dir=licenses_data_dir) spdx_tokens = set(get_all_spdx_key_tokens(license_db)) idx = LicenseIndex(rules, _spdx_tokens=spdx_tokens) with open(cache_file, 'wb') as ifc: ifc.write(idx.dumps()) # save the new checksums tree with open(checksum_file, 'wb') as ctcs: ctcs.write(current_checksum or tree_checksum(tree_base_dir=tree_base_dir)) return idx except yg.lockfile.FileLockTimeout: # TODO: handle unable to lock in a nicer way raise
def get_or_build_index_through_cache( check_consistency=DEV_MODE, return_index=True, # used for testing only _tree_base_dir=src_dir, _tree_checksum_file=tree_checksum_file, _index_lock_file=index_lock_file, _index_cache_file=index_cache_file, _licenses_data_dir=None, _rules_data_dir=None, _timeout=LICENSE_INDEX_LOCK_TIMEOUT, ): """ Check and build or rebuild the LicenseIndex cache. If the cache does not exist, a new index is built an cached. Return the LicenseIndex if return_index is True. If `check_consistency` is True, the cache is checked for consistency and rebuilt if inconsistent or stale. If `check_consistency` is False, the cache is NOT checked for consistency If the cache files exist but stale, the cache WILL NOT be rebuilt """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules from licensedcode.models import licenses_data_dir from licensedcode.models import rules_data_dir _licenses_data_dir = _licenses_data_dir or licenses_data_dir _rules_data_dir = _rules_data_dir or rules_data_dir has_cache = exists(_index_cache_file) has_tree_checksum = exists(_tree_checksum_file) # bypass check if no consistency check is needed if has_cache and has_tree_checksum and not check_consistency: return return_index and _load_index(_index_cache_file) # here, we have no cache or we want a validity check: lock, check # and build or rebuild as needed try: # acquire lock and wait until timeout to get a lock or die with yg.lockfile.FileLock(_index_lock_file, timeout=_timeout): current_checksum = None # is the current cache consistent or stale? if has_cache and has_tree_checksum: # if we have a saved cached index # load saved tree_checksum and compare with current tree_checksum with open(_tree_checksum_file, 'rb') as etcs: existing_checksum = etcs.read() current_checksum = tree_checksum(tree_base_dir=_tree_base_dir) if current_checksum == existing_checksum: # The cache is consistent with the latest code and data # load and return return return_index and _load_index(_index_cache_file) # Here, the cache is not consistent with the latest code and # data: It is either stale or non-existing: we need to # rebuild the index and cache it rules = get_rules(licenses_data_dir=_licenses_data_dir, rules_data_dir=_rules_data_dir) idx = LicenseIndex(rules) with open(_index_cache_file, 'wb') as ifc: ifc.write(idx.dumps()) # save the new checksums tree with open(_tree_checksum_file, 'wb') as ctcs: ctcs.write(current_checksum or tree_checksum(tree_base_dir=_tree_base_dir)) return return_index and idx except yg.lockfile.FileLockTimeout: # TODO: handle unable to lock in a nicer way raise
def get_or_build_index_through_cache( check_consistency=DEV_MODE, return_index=True, # used for testing only _tree_base_dir=src_dir, _tree_checksum_file=tree_checksum_file, _index_lock_file=index_lock_file, _index_cache_file=index_cache_file, _licenses_data_dir=None, _rules_data_dir=None, _timeout=LICENSE_INDEX_LOCK_TIMEOUT, ): """ Check and build or rebuild the LicenseIndex cache. If the cache does not exist, a new index is built an cached. Return the LicenseIndex if return_index is True. If `check_consistency` is True, the cache is checked for consistency and rebuilt if inconsistent or stale. If `check_consistency` is False, the cache is NOT checked for consistency If the cache files exist but stale, the cache WILL NOT be rebuilt """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules from licensedcode.models import licenses_data_dir from licensedcode.models import rules_data_dir _licenses_data_dir = _licenses_data_dir or licenses_data_dir _rules_data_dir = _rules_data_dir or rules_data_dir has_cache = exists(_index_cache_file) has_tree_checksum = exists(_tree_checksum_file) # bypass check if no consistency check is needed if has_cache and has_tree_checksum and not check_consistency: return return_index and _load_index(_index_cache_file) # here, we have no cache or we want a validity check: lock, check # and build or rebuild as needed try: # acquire lock and wait until timeout to get a lock or die with yg.lockfile.FileLock(_index_lock_file, timeout=_timeout): current_checksum = None # is the current cache consistent or stale? if has_cache and has_tree_checksum: # if we have a saved cached index # load saved tree_checksum and compare with current tree_checksum with open(_tree_checksum_file, 'rb') as etcs: existing_checksum = etcs.read() current_checksum = tree_checksum(tree_base_dir=_tree_base_dir) if current_checksum == existing_checksum: # The cache is consistent with the latest code and data # load and return return return_index and _load_index(_index_cache_file) # Here, the cache is not consistent with the latest code and # data: It is either stale or non-existing: we need to # rebuild the index and cache it rules = get_rules( licenses_data_dir=_licenses_data_dir, rules_data_dir=_rules_data_dir) idx = LicenseIndex(rules) with open(_index_cache_file, 'wb') as ifc: ifc.write(idx.dumps()) # save the new checksums tree with open(_tree_checksum_file, 'wb') as ctcs: ctcs.write(current_checksum or tree_checksum(tree_base_dir=_tree_base_dir)) return return_index and idx except yg.lockfile.FileLockTimeout: # TODO: handle unable to lock in a nicer way raise
def get_cached_index( licensedcode_cache_dir=licensedcode_cache_dir, scancode_cache_dir=scancode_cache_dir, check_consistency=SCANCODE_DEV_MODE, # used for testing only timeout=LICENSE_INDEX_LOCK_TIMEOUT, tree_base_dir=scancode_src_dir, licenses_data_dir=None, rules_data_dir=None, ): """ Return a LicenseIndex: either load a cached index or build and cache the index. - If the cache does not exist, a new index is built and cached. - If `check_consistency` is True, the cache is checked for consistency and rebuilt if inconsistent or stale. - If `check_consistency` is False, the cache is NOT checked for consistency If the cache files exist but ARE stale, the cache WILL NOT be rebuilt """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules from licensedcode.models import get_all_spdx_key_tokens from licensedcode.models import licenses_data_dir as ldd from licensedcode.models import rules_data_dir as rdd from scancode import lockfile licenses_data_dir = licenses_data_dir or ldd rules_data_dir = rules_data_dir or rdd lock_file, checksum_file, cache_file = get_license_cache_paths( licensedcode_cache_dir=licensedcode_cache_dir, scancode_cache_dir=scancode_cache_dir, ) has_cache = has_cache_index_file(cache_file) # bypass check if no consistency check is needed if has_cache and not check_consistency: try: return load_index(cache_file) except Exception as e: # work around some rare Windows quirks import traceback print( 'Inconsistent License index cache: checking and rebuilding index.' ) print(str(e)) print(traceback.format_exc()) has_tree_checksum = os.path.exists(checksum_file) # here, we have no cache or we want a validity check: lock, check # and build or rebuild as needed try: # acquire lock and wait until timeout to get a lock or die with lockfile.FileLock(lock_file).locked(timeout=timeout): current_checksum = None # is the current cache consistent or stale? if has_cache and has_tree_checksum: # if we have a saved cached index # load saved tree_checksum and compare with current tree_checksum with open(checksum_file) as etcs: existing_checksum = etcs.read() current_checksum = tree_checksum(tree_base_dir=tree_base_dir) if current_checksum == existing_checksum: # The cache is consistent with the latest code and data # load and return return load_index(cache_file) # Here, the cache is not consistent with the latest code and # data: It is either stale or non-existing: we need to # rebuild the index and cache it # FIXME: caching a pickle of this would be 10x times faster license_db = get_licenses_db(licenses_data_dir=licenses_data_dir) rules = get_rules(licenses_data_dir=licenses_data_dir, rules_data_dir=rules_data_dir) spdx_tokens = set(get_all_spdx_key_tokens(license_db)) idx = LicenseIndex(rules, _spdx_tokens=spdx_tokens) with open(cache_file, 'wb') as ifc: idx.dump(ifc) # save the new tree checksum current_checksum = tree_checksum(tree_base_dir=tree_base_dir) with open(checksum_file, 'w') as ctcs: ctcs.write(current_checksum) return idx except lockfile.LockTimeout: # TODO: handle unable to lock in a nicer way raise