def put_files_cache(self): """ Used by :py:func:`waflib.Task.cache_outputs` to store the build files in the cache """ # file caching, if possible # try to avoid data corruption as much as possible if not isinstance(self.generator.bld, Build.BuildContext): return if hasattr(self, 'cached'): return sig = self.signature() ssig = Utils.to_hex(self.uid()) + Utils.to_hex(sig) dname = os.path.join(self.generator.bld.artifacts_cache, ssig) dname_tmp = dname + '_tmp' if os.path.exists(dname) or os.path.exists(dname_tmp): return else: try: os.makedirs(dname_tmp) except OSError: return for node in self.outputs: dest = os.path.join(dname_tmp, node.name) dest = Utils.extended_path(dest) try: if Utils.is_win32: os.system('copy {} {} /Y>nul'.format(node.abspath(), dest)) else: os.system('cp -f {} {}'.format(node.abspath(), dest)) except Exception as e: Logs.warn( '[WARN] task: failed caching file {} due to exception\n {}\n'. format(dest, e)) return try: os.utime(dname_tmp, None) os.chmod(dname_tmp, Utils.O755) if Utils.is_win32: # For rename in Windows, target path cannot be full path, it will remain in the same folder with source path os.system('rename {} {}'.format(dname_tmp, ssig)) else: os.system('mv {} {}'.format(dname_tmp, dname)) except Exception as e: Logs.warn( '[WARN] task: failed updating timestamp/permission for cached outputs folder for task signature {} due to exception\n {}\n' .format(dname, e)) pass self.cached = True
def can_retrieve_cache(self): """ Used by :py:meth:`waflib.Task.cache_outputs` Retrieve build nodes from the cache update the file timestamps to help cleaning the least used entries from the cache additionally, set an attribute 'cached' to avoid re-creating the same cache files Suppose there are files in `cache/dir1/file1` and `cache/dir2/file2`: #. read the timestamp of dir1 #. try to copy the files #. look at the timestamp again, if it has changed, the data may have been corrupt (cache update by another process) #. should an exception occur, ignore the data """ bld = self.generator.bld if not isinstance(bld, Build.BuildContext): return False if not getattr(self, 'outputs', None): return False if not hasattr(self, 'can_retrieve_cache_checked'): self.can_retrieve_cache_checked = True else: return False bld.artifacts_cache_metrics.tasks_processed.add(self) sig = self.signature() ssig = Utils.to_hex(self.uid()) + Utils.to_hex(sig) # first try to access the cache folder for the task dname = os.path.join(bld.artifacts_cache, ssig) if not os.path.exists(dname): bld.artifacts_cache_metrics.tasks_missed.add(self) return False for node in self.outputs: orig = os.path.join(dname, node.name) # Maximum Path Length Limitation on Windows is 260 characters, starting from Windows 10, we can enable long path to remove this limitation. # In case long path is not enabled, extended-length path to bypass this limitation. orig = Utils.extended_path(orig) try: t1 = os.stat(orig).st_mtime except OSError: bld.artifacts_cache_metrics.tasks_missed.add(self) return False dir_name = os.path.dirname(node.abspath()) try: os.makedirs(dir_name) except Exception: pass try: # Do not use shutil.copy2(orig, node.abspath()), otherwise, it will cause threading issue with compiler and linker. # shutil.copy2() first calls shutil.copyfile() to copy the file contents, and then calls os.copystat() to copy the file stats, after the file contents are copied, waf is able to get the node's signature and might think the runnable status of a task is ready to run, but the copied file is then opened by os.copystat(), and compiler or linker who use the copied file as input file will fail. if Utils.is_win32: os.system('copy {} {} /Y>nul'.format(orig, node.abspath())) else: os.system('cp {} {}'.format(orig, node.abspath())) # is it the same file? try: t2 = os.stat(orig).st_mtime if t1 != t2: bld.artifacts_cache_metrics.tasks_failed_to_retrieve.add( self) return False except OSError: bld.artifacts_cache_metrics.tasks_failed_to_retrieve.add(self) return False except Exception as e: Logs.warn( '[WARN] task: failed retrieving file {} due to exception\n{}\n' .format(node.abspath(), e)) bld.artifacts_cache_metrics.tasks_failed_to_retrieve.add(self) return False for node in self.outputs: node.sig = sig if bld.progress_bar < 1: bld.to_log('restoring from cache %r\n' % node.abspath()) # mark the cache file folder as used recently (modified) os.utime(dname, None) self.cached = True return True