def _load_index(_index_cache_file=index_cache_file): """ Return a LicenseIndex loaded from cache. """ from licensedcode.index import LicenseIndex with open(_index_cache_file, 'rb') as ifc: # Note: weird but read() + loads() is much (twice++???) faster than load() idx = LicenseIndex.loads(ifc.read()) return idx
def load_index(cache_file): """ Return a LicenseIndex loaded from cache. """ from licensedcode.index import LicenseIndex with open(cache_file, 'rb') as ifc: # Note: weird but read() + loads() is much (twice++???) faster than load() try: return LicenseIndex.loads(ifc.read()) except: ex_type, ex_msg, ex_traceback = sys.exc_info() message = (str(ex_msg) + '\nERROR: Failed to load license cache (the file may be corrupted ?).\n' 'Please delete "{cache_file}" and retry.\n' 'If the problem persists, copy this error message ' 'and submit a bug report.\n'.format(**locals())) reraise(ex_type, message, ex_traceback)
def get_or_build_index_from_cache(force_clear=False): """ Return a LicenseIndex loaded from cache. If the index is stale or does not exist, build a new index and caches it. Clear or purge the LicenseMatch cache as needed. """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules try: # acquire lock and wait until timeout to get a lock or die with yg.lockfile.FileLock(index_lock_file, timeout=LICENSE_INDEX_LOCK_TIMEOUT): current_checksum = None # if we have a saved cached index if exists(tree_checksum_file) and exists(index_cache_file): # load saved tree_checksum and compare with current tree_checksum with open(tree_checksum_file, 'rb') as etcs: existing_checksum = etcs.read() current_checksum = tree_checksum() if current_checksum == existing_checksum: # The cache is consistent with the latest code and data: # we load index from cache with open(index_cache_file, 'rb') as ifc: # Note: loads() is much (twice++???) faster than load() idx = LicenseIndex.loads(ifc.read()) return idx # Here, the cache is not consistent with the latest code and data: # It is either stale or non-existing: we need to cleanup/regen # regen the index idx = LicenseIndex(get_rules()) with open(index_cache_file, 'wb') as ifc: ifc.write(idx.dumps()) # save the new checksums tree with open(tree_checksum_file, 'wb') as ctcs: ctcs.write(current_checksum or tree_checksum()) return idx except yg.lockfile.FileLockTimeout: # TODO: unable to lock in a nicer way raise
def load_index(cache_file, use_loads=False): """ Return a LicenseIndex loaded from cache. """ from licensedcode.index import LicenseIndex with open(cache_file, 'rb') as ifc: # Note: weird but read() + loads() is much (twice++???) faster than load() try: if use_loads: return LicenseIndex.loads(ifc.read()) else: return LicenseIndex.load(ifc) except Exception as e: import traceback msg = ( '\n' 'ERROR: Failed to load license cache (the file may be corrupted ?).\n' 'Please delete "{cache_file}" and retry.\n' 'If the problem persists, copy this error message ' 'and submit a bug report.\n'.format(**locals())) msg += '\n' + traceback.format_exc() raise Exception(msg)
def get_or_build_index_from_cache(): """ Return a LicenseIndex loaded from cache or build a new index and caches it. """ from licensedcode.index import LicenseIndex from licensedcode.models import get_rules try: # acquire global lock file and wait until timeout to get a lock or die with yg.lockfile.FileLock(index_lock_file, timeout=60 * 3): # if we have a saved cached index if exists(tree_checksum_file) and exists(index_cache_file): # load saved tree_checksum and compare with current tree_checksum with open(tree_checksum_file, 'rb') as etcs: existing_checksum = etcs.read() current_checksum = tree_checksum() # if this cached index is current for the code and data if current_checksum == existing_checksum: # load index from cache with open(index_cache_file, 'rb') as ifc: idx = LicenseIndex.loads(ifc.read()) return idx # here the cache is stale or non-existing: we need to regen the index idx = LicenseIndex(get_rules()) with open(index_cache_file, 'wb') as ifc: ifc.write(idx.dumps()) # and save the checksums with open(tree_checksum_file, 'wb') as ctcs: ctcs.write(tree_checksum()) return idx except yg.lockfile.FileLockTimeout: # handle unable to lock raise