def extract(self): try: with open_tar(self._tarfile, 'r', errorlevel=2) as tarin: # Note: We create all needed paths proactively, even though extractall() can do this for us. # This is because we may be called concurrently on multiple artifacts that share directories, # and there will be a race condition inside extractall(): task T1 A) sees that a directory # doesn't exist and B) tries to create it. But in the gap between A) and B) task T2 creates # the same directory, so T1 throws "File exists" in B). # This actually happened, and was very hard to debug. # Creating the paths here up front allows us to squelch that "File exists" error. paths = [] dirs = set() for tarinfo in tarin.getmembers(): paths.append(tarinfo.name) if tarinfo.isdir(): dirs.add(tarinfo.name) else: dirs.add(os.path.dirname(tarinfo.name)) for d in dirs: try: os.makedirs(os.path.join(self._artifact_root, d)) except OSError as e: if e.errno != errno.EEXIST: raise tarin.extractall(self._artifact_root) self._relpaths.update(paths) except tarfile.ReadError as e: raise ArtifactError(e.message)
def use_cached_files(self, cache_key): path = self._path_for_key(cache_key) response = self._request('GET', path) if response is None: return False expected_size = int(response.getheader('content-length', -1)) if expected_size == -1: raise Exception, 'No content-length header in HTTP response' read_size = 4 * 1024 * 1024 # 4 MB done = False if self.context: self.context.log.info('Reading %d bytes' % expected_size) with temporary_file() as outfile: total_bytes = 0 while not done: data = response.read(read_size) outfile.write(data) if len(data) < read_size: done = True total_bytes += len(data) if self.context: self.context.log.debug('Read %d bytes' % total_bytes) outfile.close() if total_bytes != expected_size: raise Exception, 'Read only %d bytes from %d expected' % (total_bytes, expected_size) mode = 'r:bz2' if self.compress else 'r' with open_tar(outfile.name, mode) as tarfile: tarfile.extractall(self.artifact_root) return True
def collect(self, paths): # In our tests, gzip is slightly less compressive than bzip2 on .class files, # but decompression times are much faster. mode = 'w:gz' if self._compress else 'w' with open_tar(self._tarfile, mode, dereference=True, errorlevel=2) as tarout: for path in paths or (): # Adds dirs recursively. relpath = os.path.relpath(path, self._artifact_root) tarout.add(path, relpath) self._relpaths.add(relpath)
def try_insert(self, cache_key, build_artifacts): path = self._path_for_key(cache_key) with temporary_file() as tarfile: mode = 'w:bz2' if self.compress else 'w' with open_tar(tarfile, mode, dereference=True) as tarout: for artifact in build_artifacts: tarout.add(artifact, os.path.relpath(artifact, self.artifact_root)) # Adds dirs recursively. tarfile.close() with open(tarfile.name, 'rb') as infile: if not self._request('PUT', path, body=infile): raise Exception, 'Failed to PUT to %s. Error: 404' % self._url_string(path)
def try_insert(self, cache_key, build_artifacts): with temporary_file_path() as tarfile: # In our tests, gzip is slightly less compressive than bzip2 on .class files, # but decompression times are much faster. mode = 'w:gz' if self.compress else 'w' with open_tar(tarfile, mode, dereference=True) as tarout: for artifact in build_artifacts: # Adds dirs recursively. tarout.add(artifact, os.path.relpath(artifact, self.artifact_root)) with open(tarfile, 'rb') as infile: path = self._path_for_key(cache_key) if not self._request('PUT', path, body=infile): raise self.CacheError('Failed to PUT to %s. Error: 404' % self._url_string(path))
def use_cached_files(self, cache_key): # This implementation fetches the appropriate tarball and extracts it. try: # Send an HTTP request for the tarball. path = self._path_for_key(cache_key) response = self._request('GET', path) if response is None: return False expected_size = int(response.getheader('content-length', -1)) if expected_size == -1: raise ArtifactCacheError('No content-length header in HTTP response') read_size = 4 * 1024 * 1024 # 4 MB done = False if self.context: self.context.log.info('Reading %d bytes from artifact cache at %s' % (expected_size, self._url_string(path))) # Read the data in a loop. with temporary_file() as outfile: total_bytes = 0 while not done: data = response.read(read_size) outfile.write(data) if len(data) < read_size: done = True total_bytes += len(data) if self.context: self.context.log.debug('Read %d bytes' % total_bytes) outfile.close() # Check the size. if total_bytes != expected_size: raise ArtifactCacheError('Read only %d bytes from %d expected' % (total_bytes, expected_size)) # Extract the tarfile. mode = 'r:bz2' if self.compress else 'r' with open_tar(outfile.name, mode) as tarfile: tarfile.extractall(self.artifact_root) return True except Exception, e: if self.context: self.context.log.warn('Error while reading from artifact cache: %s' % e) return False
def archive(self, basedir, outdir, name): tarpath = os.path.join(outdir, '%s.%s' % (name, self.extension)) with open_tar(tarpath, self.mode, dereference=True) as tar: tar.add(basedir, arcname='') return tarpath
def create(self, basedir, outdir, name, prefix=None): tarpath = os.path.join(outdir, '%s.%s' % (name.decode('utf-8'), self.extension)) with open_tar(tarpath, self.mode, dereference=True, errorlevel=1) as tar: basedir = basedir.decode('utf-8') tar.add(basedir, arcname=prefix or '.') return tarpath
def extract(cls, path, outdir): with open_tar(path, errorlevel=1) as tar: tar.extractall(outdir)
def create(self, basedir, outdir, name, prefix=None): tarpath = os.path.join(outdir, '%s.%s' % (name, self.extension)) with open_tar(tarpath, self.mode, dereference=True) as tar: tar.add(basedir, arcname=prefix or '') return tarpath
def extract(cls, path, outdir): with open_tar(path) as tar: tar.extractall(outdir)