def extract(self):
try:
with open_tar(self._tarfile, 'r', errorlevel=2) as tarin:
# Note: We create all needed paths proactively, even though extractall() can do this for us.
# This is because we may be called concurrently on multiple artifacts that share directories,
# and there will be a race condition inside extractall(): task T1 A) sees that a directory
# doesn't exist and B) tries to create it. But in the gap between A) and B) task T2 creates
# the same directory, so T1 throws "File exists" in B).
# This actually happened, and was very hard to debug.
# Creating the paths here up front allows us to squelch that "File exists" error.
paths = []
dirs = set()
for tarinfo in tarin.getmembers():
paths.append(tarinfo.name)
if tarinfo.isdir():
dirs.add(tarinfo.name)
else:
dirs.add(os.path.dirname(tarinfo.name))
for d in dirs:
try:
os.makedirs(os.path.join(self._artifact_root, d))
except OSError as e:
if e.errno != errno.EEXIST:
raise
tarin.extractall(self._artifact_root)
self._relpaths.update(paths)
except tarfile.ReadError as e:
raise ArtifactError(e.message)
def use_cached_files(self, cache_key):
path = self._path_for_key(cache_key)
response = self._request('GET', path)
if response is None:
return False
expected_size = int(response.getheader('content-length', -1))
if expected_size == -1:
raise Exception, 'No content-length header in HTTP response'
read_size = 4 * 1024 * 1024 # 4 MB
done = False
if self.context:
self.context.log.info('Reading %d bytes' % expected_size)
with temporary_file() as outfile:
total_bytes = 0
while not done:
data = response.read(read_size)
outfile.write(data)
if len(data) < read_size:
done = True
total_bytes += len(data)
if self.context:
self.context.log.debug('Read %d bytes' % total_bytes)
outfile.close()
if total_bytes != expected_size:
raise Exception, 'Read only %d bytes from %d expected' % (total_bytes, expected_size)
mode = 'r:bz2' if self.compress else 'r'
with open_tar(outfile.name, mode) as tarfile:
tarfile.extractall(self.artifact_root)
return True
def collect(self, paths):
# In our tests, gzip is slightly less compressive than bzip2 on .class files,
# but decompression times are much faster.
mode = 'w:gz' if self._compress else 'w'
with open_tar(self._tarfile, mode, dereference=True, errorlevel=2) as tarout:
for path in paths or ():
# Adds dirs recursively.
relpath = os.path.relpath(path, self._artifact_root)
tarout.add(path, relpath)
self._relpaths.add(relpath)
def try_insert(self, cache_key, build_artifacts):
path = self._path_for_key(cache_key)
with temporary_file() as tarfile:
mode = 'w:bz2' if self.compress else 'w'
with open_tar(tarfile, mode, dereference=True) as tarout:
for artifact in build_artifacts:
tarout.add(artifact, os.path.relpath(artifact, self.artifact_root)) # Adds dirs recursively.
tarfile.close()
with open(tarfile.name, 'rb') as infile:
if not self._request('PUT', path, body=infile):
raise Exception, 'Failed to PUT to %s. Error: 404' % self._url_string(path)
def try_insert(self, cache_key, build_artifacts):
with temporary_file_path() as tarfile:
# In our tests, gzip is slightly less compressive than bzip2 on .class files,
# but decompression times are much faster.
mode = 'w:gz' if self.compress else 'w'
with open_tar(tarfile, mode, dereference=True) as tarout:
for artifact in build_artifacts:
# Adds dirs recursively.
tarout.add(artifact, os.path.relpath(artifact, self.artifact_root))
with open(tarfile, 'rb') as infile:
path = self._path_for_key(cache_key)
if not self._request('PUT', path, body=infile):
raise self.CacheError('Failed to PUT to %s. Error: 404' % self._url_string(path))
def use_cached_files(self, cache_key):
# This implementation fetches the appropriate tarball and extracts it.
try:
# Send an HTTP request for the tarball.
path = self._path_for_key(cache_key)
response = self._request('GET', path)
if response is None:
return False
expected_size = int(response.getheader('content-length', -1))
if expected_size == -1:
raise ArtifactCacheError('No content-length header in HTTP response')
read_size = 4 * 1024 * 1024 # 4 MB
done = False
if self.context:
self.context.log.info('Reading %d bytes from artifact cache at %s' %
(expected_size, self._url_string(path)))
# Read the data in a loop.
with temporary_file() as outfile:
total_bytes = 0
while not done:
data = response.read(read_size)
outfile.write(data)
if len(data) < read_size:
done = True
total_bytes += len(data)
if self.context:
self.context.log.debug('Read %d bytes' % total_bytes)
outfile.close()
# Check the size.
if total_bytes != expected_size:
raise ArtifactCacheError('Read only %d bytes from %d expected' %
(total_bytes, expected_size))
# Extract the tarfile.
mode = 'r:bz2' if self.compress else 'r'
with open_tar(outfile.name, mode) as tarfile:
tarfile.extractall(self.artifact_root)
return True
except Exception, e:
if self.context:
self.context.log.warn('Error while reading from artifact cache: %s' % e)
return False
def archive(self, basedir, outdir, name):
tarpath = os.path.join(outdir, '%s.%s' % (name, self.extension))
with open_tar(tarpath, self.mode, dereference=True) as tar:
tar.add(basedir, arcname='')
return tarpath
def create(self, basedir, outdir, name, prefix=None):
tarpath = os.path.join(outdir, '%s.%s' % (name.decode('utf-8'), self.extension))
with open_tar(tarpath, self.mode, dereference=True, errorlevel=1) as tar:
basedir = basedir.decode('utf-8')
tar.add(basedir, arcname=prefix or '.')
return tarpath
def extract(cls, path, outdir):
with open_tar(path, errorlevel=1) as tar:
tar.extractall(outdir)
def extract(cls, path, outdir):
with open_tar(path, errorlevel=1) as tar:
tar.extractall(outdir)
def create(self, basedir, outdir, name, prefix=None):
tarpath = os.path.join(outdir, '%s.%s' % (name, self.extension))
with open_tar(tarpath, self.mode, dereference=True) as tar:
tar.add(basedir, arcname=prefix or '')
return tarpath
def extract(cls, path, outdir):
with open_tar(path) as tar:
tar.extractall(outdir)
def extract(cls, path, outdir):
with open_tar(path) as tar:
tar.extractall(outdir)
