def _dump_built_library(self, library, builder): # TODO(wickman): Port this over to the Installer+Distiller and stop using ArtifactCache. absolute_sources = library.expand_files() absolute_sources.sort() cache_key = self._key_generator.key_for(library.id, absolute_sources) cache_dir = os.path.join(self._egg_cache_root, cache_key.hash) if os.path.exists(cache_dir): self.debug(' Generating (cached) %s...' % library) # We have no idea what the egg path is, so we simply walk the directory. for dir_name, _, filenames in os.walk(cache_dir): for filename in filenames: self._builder.add_egg(os.path.join(dir_name, filename)) else: self.debug(' Generating %s...' % library) egg_file = builder.build_egg() if not egg_file: raise PythonChroot.BuildFailureException("Failed to build %s!" % library) src_egg_file = egg_file dst_egg_file = os.path.join(os.path.dirname(egg_file), cache_key.hash + '_' + os.path.basename(egg_file)) self.debug(' %s => %s' % (src_egg_file, dst_egg_file)) os.rename(src_egg_file, dst_egg_file) cache_dir = os.path.join(self._egg_cache_root, cache_key.hash) cached_egg_file = os.path.join(cache_dir, os.path.relpath(dst_egg_file, self._root)) try: safe_mkdir_for(cached_egg_file) shutil.copy(dst_egg_file, cached_egg_file) except: safe_rmtree(cache_dir) raise self._builder.add_egg(dst_egg_file)
def __init__(self, info_file): self._info_file = info_file safe_mkdir_for(self._info_file) self._info = {} if os.path.exists(self._info_file): with open(self._info_file, 'r') as infile: info = infile.read() for m in re.finditer("""^([^:]+):(.*)$""", info, re.MULTILINE): self._info[m.group(1).strip()] = m.group(2).strip()
def extract(self): for dir_name, _, filenames in os.walk(self._directory): for filename in filenames: filename = os.path.join(dir_name, filename) relpath = os.path.relpath(filename, self._directory) dst = os.path.join(self._artifact_root, relpath) safe_mkdir_for(dst) shutil.copy(filename, dst) self._relpaths.add(relpath)
def output(self, name): """Returns the output buffer for the specified output name (e.g., 'stdout').""" m = WorkUnit._valid_name_re.match(name) if not m or m.group(0) != name: raise Exception('Invalid output name: %s' % name) if name not in self._outputs: path = os.path.join(self.run_tracker.info_dir, 'tool_outputs', '%s.%s' % (self.id, name)) safe_mkdir_for(path) self._outputs[name] = FileBackedRWBuf(path) return self._outputs[name]
def try_insert(self, cache_key, paths): tarfile = self._cache_file_for_key(cache_key) safe_mkdir_for(tarfile) # Write to a temporary name (on the same filesystem), and move it atomically, so if we # crash in the middle we don't leave an incomplete or missing artifact. tarfile_tmp = tarfile + '.' + str(uuid.uuid4()) + '.tmp' if os.path.exists(tarfile_tmp): os.unlink(tarfile_tmp) artifact = TarballArtifact(self.artifact_root, tarfile_tmp, self._compress) artifact.collect(paths) # Note: Race condition here if multiple pants runs (in different workspaces) # try to write the same thing at the same time. However since rename is atomic, # this should not result in corruption. It may however result in a missing artifact # If we crash between the unlink and the rename. But that's OK. if os.path.exists(tarfile): os.unlink(tarfile) os.rename(tarfile_tmp, tarfile)
def genlang(self, lang, targets): if lang != 'java': raise TaskError('Unrecognized ragel gen lang: {lang}'.format(lang=lang)) sources = self._calculate_sources(targets) output_dir = self._java_out lang_flag = '-J' for source in sources: output_file = os.path.join(output_dir, calculate_genfile(source)) safe_mkdir_for(output_file) args = [self.ragel_binary, lang_flag, '-o', output_file] args.append(source) self.context.log.debug('Executing: {args}'.format(args=' '.join(args))) process = subprocess.Popen(args) result = process.wait() if result != 0: raise TaskError('{binary} ... exited non-zero ({result})'.format(binary=self.ragel_binary, result=result))
def copy(src, rel_dst): dst = os.path.join(self.artifact_root, rel_dst) safe_mkdir_for(dst) shutil.copy(src, dst)
def __init__(self, path=None): # Map path -> timing in seconds (a float) self._timings_by_path = defaultdict(float) self._tool_labels = set() self._path = path safe_mkdir_for(self._path)