def put_files_cache(self):
"""
Used by :py:func:`waflib.Task.cache_outputs` to store the build files in the cache
"""
# file caching, if possible
# try to avoid data corruption as much as possible
if not isinstance(self.generator.bld, Build.BuildContext):
return
if hasattr(self, 'cached'):
return
sig = self.signature()
ssig = Utils.to_hex(self.uid()) + Utils.to_hex(sig)
dname = os.path.join(self.generator.bld.artifacts_cache, ssig)
dname_tmp = dname + '_tmp'
if os.path.exists(dname) or os.path.exists(dname_tmp):
return
else:
try:
os.makedirs(dname_tmp)
except OSError:
return
for node in self.outputs:
dest = os.path.join(dname_tmp, node.name)
dest = Utils.extended_path(dest)
try:
if Utils.is_win32:
os.system('copy {} {} /Y>nul'.format(node.abspath(), dest))
else:
os.system('cp -f {} {}'.format(node.abspath(), dest))
except Exception as e:
Logs.warn(
'[WARN] task: failed caching file {} due to exception\n {}\n'.
format(dest, e))
return
try:
os.utime(dname_tmp, None)
os.chmod(dname_tmp, Utils.O755)
if Utils.is_win32:
# For rename in Windows, target path cannot be full path, it will remain in the same folder with source path
os.system('rename {} {}'.format(dname_tmp, ssig))
else:
os.system('mv {} {}'.format(dname_tmp, dname))
except Exception as e:
Logs.warn(
'[WARN] task: failed updating timestamp/permission for cached outputs folder for task signature {} due to exception\n {}\n'
.format(dname, e))
pass
self.cached = True
def can_retrieve_cache(self):
"""
Used by :py:meth:`waflib.Task.cache_outputs`
Retrieve build nodes from the cache
update the file timestamps to help cleaning the least used entries from the cache
additionally, set an attribute 'cached' to avoid re-creating the same cache files
Suppose there are files in `cache/dir1/file1` and `cache/dir2/file2`:
#. read the timestamp of dir1
#. try to copy the files
#. look at the timestamp again, if it has changed, the data may have been corrupt (cache update by another process)
#. should an exception occur, ignore the data
"""
bld = self.generator.bld
if not isinstance(bld, Build.BuildContext):
return False
if not getattr(self, 'outputs', None):
return False
if not hasattr(self, 'can_retrieve_cache_checked'):
self.can_retrieve_cache_checked = True
else:
return False
bld.artifacts_cache_metrics.tasks_processed.add(self)
sig = self.signature()
ssig = Utils.to_hex(self.uid()) + Utils.to_hex(sig)
# first try to access the cache folder for the task
dname = os.path.join(bld.artifacts_cache, ssig)
if not os.path.exists(dname):
bld.artifacts_cache_metrics.tasks_missed.add(self)
return False
for node in self.outputs:
orig = os.path.join(dname, node.name)
# Maximum Path Length Limitation on Windows is 260 characters, starting from Windows 10, we can enable long path to remove this limitation.
# In case long path is not enabled, extended-length path to bypass this limitation.
orig = Utils.extended_path(orig)
try:
t1 = os.stat(orig).st_mtime
except OSError:
bld.artifacts_cache_metrics.tasks_missed.add(self)
return False
dir_name = os.path.dirname(node.abspath())
try:
os.makedirs(dir_name)
except Exception:
pass
try:
# Do not use shutil.copy2(orig, node.abspath()), otherwise, it will cause threading issue with compiler and linker.
# shutil.copy2() first calls shutil.copyfile() to copy the file contents, and then calls os.copystat() to copy the file stats, after the file contents are copied, waf is able to get the node's signature and might think the runnable status of a task is ready to run, but the copied file is then opened by os.copystat(), and compiler or linker who use the copied file as input file will fail.
if Utils.is_win32:
os.system('copy {} {} /Y>nul'.format(orig, node.abspath()))
else:
os.system('cp {} {}'.format(orig, node.abspath()))
# is it the same file?
try:
t2 = os.stat(orig).st_mtime
if t1 != t2:
bld.artifacts_cache_metrics.tasks_failed_to_retrieve.add(
self)
return False
except OSError:
bld.artifacts_cache_metrics.tasks_failed_to_retrieve.add(self)
return False
except Exception as e:
Logs.warn(
'[WARN] task: failed retrieving file {} due to exception\n{}\n'
.format(node.abspath(), e))
bld.artifacts_cache_metrics.tasks_failed_to_retrieve.add(self)
return False
for node in self.outputs:
node.sig = sig
if bld.progress_bar < 1:
bld.to_log('restoring from cache %r\n' % node.abspath())
# mark the cache file folder as used recently (modified)
os.utime(dname, None)
self.cached = True
return True
