def _dump_built_library(self, library, builder):
# TODO(wickman): Port this over to the Installer+Distiller and stop using ArtifactCache.
absolute_sources = library.expand_files()
absolute_sources.sort()
cache_key = self._key_generator.key_for(library.id, absolute_sources)
cache_dir = os.path.join(self._egg_cache_root, cache_key.hash)
if os.path.exists(cache_dir):
self.debug(' Generating (cached) %s...' % library)
# We have no idea what the egg path is, so we simply walk the directory.
for dir_name, _, filenames in os.walk(cache_dir):
for filename in filenames:
self._builder.add_egg(os.path.join(dir_name, filename))
else:
self.debug(' Generating %s...' % library)
egg_file = builder.build_egg()
if not egg_file:
raise PythonChroot.BuildFailureException("Failed to build %s!" % library)
src_egg_file = egg_file
dst_egg_file = os.path.join(os.path.dirname(egg_file),
cache_key.hash + '_' + os.path.basename(egg_file))
self.debug(' %s => %s' % (src_egg_file, dst_egg_file))
os.rename(src_egg_file, dst_egg_file)
cache_dir = os.path.join(self._egg_cache_root, cache_key.hash)
cached_egg_file = os.path.join(cache_dir, os.path.relpath(dst_egg_file, self._root))
try:
safe_mkdir_for(cached_egg_file)
shutil.copy(dst_egg_file, cached_egg_file)
except:
safe_rmtree(cache_dir)
raise
self._builder.add_egg(dst_egg_file)
def __init__(self, info_file):
self._info_file = info_file
safe_mkdir_for(self._info_file)
self._info = {}
if os.path.exists(self._info_file):
with open(self._info_file, 'r') as infile:
info = infile.read()
for m in re.finditer("""^([^:]+):(.*)$""", info, re.MULTILINE):
self._info[m.group(1).strip()] = m.group(2).strip()
def extract(self):
for dir_name, _, filenames in os.walk(self._directory):
for filename in filenames:
filename = os.path.join(dir_name, filename)
relpath = os.path.relpath(filename, self._directory)
dst = os.path.join(self._artifact_root, relpath)
safe_mkdir_for(dst)
shutil.copy(filename, dst)
self._relpaths.add(relpath)
def output(self, name):
"""Returns the output buffer for the specified output name (e.g., 'stdout')."""
m = WorkUnit._valid_name_re.match(name)
if not m or m.group(0) != name:
raise Exception('Invalid output name: %s' % name)
if name not in self._outputs:
path = os.path.join(self.run_tracker.info_dir, 'tool_outputs', '%s.%s' % (self.id, name))
safe_mkdir_for(path)
self._outputs[name] = FileBackedRWBuf(path)
return self._outputs[name]
def try_insert(self, cache_key, paths):
tarfile = self._cache_file_for_key(cache_key)
safe_mkdir_for(tarfile)
# Write to a temporary name (on the same filesystem), and move it atomically, so if we
# crash in the middle we don't leave an incomplete or missing artifact.
tarfile_tmp = tarfile + '.' + str(uuid.uuid4()) + '.tmp'
if os.path.exists(tarfile_tmp):
os.unlink(tarfile_tmp)
artifact = TarballArtifact(self.artifact_root, tarfile_tmp, self._compress)
artifact.collect(paths)
# Note: Race condition here if multiple pants runs (in different workspaces)
# try to write the same thing at the same time. However since rename is atomic,
# this should not result in corruption. It may however result in a missing artifact
# If we crash between the unlink and the rename. But that's OK.
if os.path.exists(tarfile):
os.unlink(tarfile)
os.rename(tarfile_tmp, tarfile)
def genlang(self, lang, targets):
if lang != 'java':
raise TaskError('Unrecognized ragel gen lang: {lang}'.format(lang=lang))
sources = self._calculate_sources(targets)
output_dir = self._java_out
lang_flag = '-J'
for source in sources:
output_file = os.path.join(output_dir, calculate_genfile(source))
safe_mkdir_for(output_file)
args = [self.ragel_binary, lang_flag, '-o', output_file]
args.append(source)
self.context.log.debug('Executing: {args}'.format(args=' '.join(args)))
process = subprocess.Popen(args)
result = process.wait()
if result != 0:
raise TaskError('{binary} ... exited non-zero ({result})'.format(binary=self.ragel_binary, result=result))
def copy(src, rel_dst): dst = os.path.join(self.artifact_root, rel_dst) safe_mkdir_for(dst) shutil.copy(src, dst)
def __init__(self, path=None): # Map path -> timing in seconds (a float) self._timings_by_path = defaultdict(float) self._tool_labels = set() self._path = path safe_mkdir_for(self._path)
