def _load_index(_index_cache_file=index_cache_file):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(_index_cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
idx = LicenseIndex.loads(ifc.read())
return idx
def _load_index(_index_cache_file=index_cache_file):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(_index_cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
idx = LicenseIndex.loads(ifc.read())
return idx
def load_index(cache_file):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
try:
return LicenseIndex.loads(ifc.read())
except:
ex_type, ex_msg, ex_traceback = sys.exc_info()
message = (str(ex_msg) +
'\nERROR: Failed to load license cache (the file may be corrupted ?).\n'
'Please delete "{cache_file}" and retry.\n'
'If the problem persists, copy this error message '
'and submit a bug report.\n'.format(**locals()))
reraise(ex_type, message, ex_traceback)
def get_or_build_index_from_cache(force_clear=False):
"""
Return a LicenseIndex loaded from cache. If the index is stale or does not exist,
build a new index and caches it. Clear or purge the LicenseMatch cache as needed.
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
try:
# acquire lock and wait until timeout to get a lock or die
with yg.lockfile.FileLock(index_lock_file, timeout=LICENSE_INDEX_LOCK_TIMEOUT):
current_checksum = None
# if we have a saved cached index
if exists(tree_checksum_file) and exists(index_cache_file):
# load saved tree_checksum and compare with current tree_checksum
with open(tree_checksum_file, 'rb') as etcs:
existing_checksum = etcs.read()
current_checksum = tree_checksum()
if current_checksum == existing_checksum:
# The cache is consistent with the latest code and data:
# we load index from cache
with open(index_cache_file, 'rb') as ifc:
# Note: loads() is much (twice++???) faster than load()
idx = LicenseIndex.loads(ifc.read())
return idx
# Here, the cache is not consistent with the latest code and data:
# It is either stale or non-existing: we need to cleanup/regen
# regen the index
idx = LicenseIndex(get_rules())
with open(index_cache_file, 'wb') as ifc:
ifc.write(idx.dumps())
# save the new checksums tree
with open(tree_checksum_file, 'wb') as ctcs:
ctcs.write(current_checksum or tree_checksum())
return idx
except yg.lockfile.FileLockTimeout:
# TODO: unable to lock in a nicer way
raise
def load_index(cache_file, use_loads=False):
"""
Return a LicenseIndex loaded from cache.
"""
from licensedcode.index import LicenseIndex
with open(cache_file, 'rb') as ifc:
# Note: weird but read() + loads() is much (twice++???) faster than load()
try:
if use_loads:
return LicenseIndex.loads(ifc.read())
else:
return LicenseIndex.load(ifc)
except Exception as e:
import traceback
msg = (
'\n'
'ERROR: Failed to load license cache (the file may be corrupted ?).\n'
'Please delete "{cache_file}" and retry.\n'
'If the problem persists, copy this error message '
'and submit a bug report.\n'.format(**locals()))
msg += '\n' + traceback.format_exc()
raise Exception(msg)
def get_or_build_index_from_cache():
"""
Return a LicenseIndex loaded from cache or build a new index and caches it.
"""
from licensedcode.index import LicenseIndex
from licensedcode.models import get_rules
try:
# acquire global lock file and wait until timeout to get a lock or die
with yg.lockfile.FileLock(index_lock_file, timeout=60 * 3):
# if we have a saved cached index
if exists(tree_checksum_file) and exists(index_cache_file):
# load saved tree_checksum and compare with current tree_checksum
with open(tree_checksum_file, 'rb') as etcs:
existing_checksum = etcs.read()
current_checksum = tree_checksum()
# if this cached index is current for the code and data
if current_checksum == existing_checksum:
# load index from cache
with open(index_cache_file, 'rb') as ifc:
idx = LicenseIndex.loads(ifc.read())
return idx
# here the cache is stale or non-existing: we need to regen the index
idx = LicenseIndex(get_rules())
with open(index_cache_file, 'wb') as ifc:
ifc.write(idx.dumps())
# and save the checksums
with open(tree_checksum_file, 'wb') as ctcs:
ctcs.write(tree_checksum())
return idx
except yg.lockfile.FileLockTimeout:
# handle unable to lock
raise
