def writeDataFile(self, filename, text, content_type, subdir=None): """ See IExportContext. """ mod_time = time.time() if subdir is not None: elements = subdir.split('/') parents = filter(None, elements) while parents: dirname = os.path.join(*parents) try: self._archive.getmember(dirname + '/') except KeyError: info = TarInfo(dirname) info.size = 0 info.mode = 509 info.mtime = mod_time info.type = DIRTYPE self._archive.addfile(info, StringIO()) parents = parents[:-1] filename = '/'.join((subdir, filename)) stream = StringIO(text) info = TarInfo(filename) info.size = len(text) info.mode = 436 info.mtime = mod_time self._archive.addfile(info, stream)
def writeDataFile( self, filename, text, content_type, subdir=None ): """ See IExportContext. """ mod_time = time.time() if subdir is not None: elements = subdir.split('/') parents = filter(None, elements) while parents: dirname = os.path.join(*parents) try: self._archive.getmember(dirname+'/') except KeyError: info = TarInfo(dirname) info.size = 0 info.mode = 509 info.mtime = mod_time info.type = DIRTYPE self._archive.addfile(info, StringIO()) parents = parents[:-1] filename = '/'.join( ( subdir, filename ) ) stream = StringIO( text ) info = TarInfo( filename ) info.size = len( text ) info.mode = 436 info.mtime = mod_time self._archive.addfile( info, stream )
def _add_entry(self, name, type, mode, size, data): info = TarInfo(name) info.type = type info.mode = mode info.size = size info.mtime = time.time() self._tarfile.addfile(info, data)
def writeDataFile(self, filename, text, content_type, subdir=None): """ See IExportContext. """ if subdir is not None: filename = '/'.join((subdir, filename)) parents = filename.split('/')[:-1] while parents: path = '/'.join(parents) + '/' if path not in self._archive.getnames(): info = TarInfo(path) info.type = DIRTYPE # tarfile.filemode(0755) == '-rwxr-xr-x' info.mode = 0755 info.mtime = time.time() self._archive.addfile(info) parents.pop() info = TarInfo(filename) if isinstance(text, str): stream = StringIO(text) info.size = len(text) elif isinstance(text, unicode): raise ValueError("Unicode text is not supported, even if it only " "contains ascii. Please encode your data") else: # Assume text is a an instance of a class like # Products.Archetypes.WebDAVSupport.PdataStreamIterator, # as in the case of ATFile stream = text.file info.size = text.size info.mtime = time.time() self._archive.addfile(info, stream)
def upload(self): logger.info("Loading artifacts") t = ts() tar_data = io.BytesIO() with tarfile.open(fileobj=tar_data, mode="w|") as tar: for root, _, files in os.walk(self._artifact_directory): for af in files: full_path = os.path.join(root, af) relpath = os.path.relpath(full_path, self._artifact_directory) ti = TarInfo(relpath) stat = os.stat(full_path) ti.size = stat.st_size ti.mode = stat.st_mode with open(full_path, "rb") as f: tar.addfile(ti, f) res = self._container.put_archive(config.build_dir, tar_data.getvalue()) if not res: raise Exception(f"Error loading artifact: {af}") t = ts() - t logger.info("Artifacts loaded in %.3fs", t)
def writeDataFile( self, filename, text, content_type, subdir=None ): """ See IExportContext. """ if subdir is not None: filename = '/'.join( ( subdir, filename ) ) parents = filename.split('/')[:-1] while parents: path = '/'.join(parents) + '/' if path not in self._archive.getnames(): info = TarInfo(path) info.type = DIRTYPE # tarfile.filemode(0755) == '-rwxr-xr-x' info.mode = 0755 info.mtime = time.time() self._archive.addfile(info) parents.pop() info = TarInfo(filename) if isinstance(text, basestring): stream = StringIO(text) info.size = len(text) else: # Assume text is a an instance of a class like # Products.Archetypes.WebDAVSupport.PdataStreamIterator, # as in the case of ATFile stream = text.file info.size = text.size info.mtime = time.time() self._archive.addfile( info, stream )
def writeDataFile( self, filename, text, content_type, subdir=None ): """ See IExportContext. """ if subdir is not None: filename = '/'.join( ( subdir, filename ) ) parents = filename.split('/')[:-1] while parents: path = '/'.join(parents) + '/' if path not in self._archive.getnames(): info = TarInfo(path) info.type = DIRTYPE # tarfile.filemode(0755) == '-rwxr-xr-x' info.mode = 0755 info.mtime = time.time() self._archive.addfile(info) parents.pop() info = TarInfo(filename) if isinstance(text, str): stream = StringIO(text) info.size = len(text) elif isinstance(text, unicode): raise ValueError("Unicode text is not supported, even if it only " "contains ascii. Please encode your data. See " "GS 1.7.0 changes for more") else: # Assume text is a an instance of a class like # Products.Archetypes.WebDAVSupport.PdataStreamIterator, # as in the case of ATFile stream = text.file info.size = text.size info.mtime = time.time() self._archive.addfile( info, stream )
def GetTarInfo(filename, filetype=DIRTYPE, mode=0755): """Create information for tar files""" tarinfo = TarInfo(path.basename(filename)) tarinfo.type = filetype tarinfo.mode = mode tarinfo.mtime = time() return tarinfo
def add_filter(info: tarfile.TarInfo) -> Optional[tarfile.TarInfo]: """Filter files targeted to be added to tarfile. Args: info: Information on the file targeted to be added Returns: None: if file is not to be added TarInfo: when file is to be added. Modified as needed. Notes: exclude is captured from parent """ if not (info.isfile() or info.isdir() or info.issym()): return None if _exclude_matcher(info.name, exclude): return None # Workaround https://bugs.python.org/issue32713. Fixed in Python 3.7 if info.mtime < 0 or info.mtime > 8 ** 11 - 1: info.mtime = int(info.mtime) # do not leak client information to service info.uid = 0 info.uname = info.gname = "root" if sys.platform == "win32": info.mode = info.mode & 0o755 | 0o111 return info
def zip2tar(zip_file: str, tar_file, tar_mode: Optional[str] = 'w:gz'): """ :param zip_file: zip file path :param tar_file: IO(_io.IOBase): file obj :param tar_mode: ref `tarfile.TarFile.open` :return: """ zip_file = ZipFile(file=zip_file, mode='r') tar_file = TarFile.open(fileobj=tar_file, mode=tar_mode) try: for zip_info in zip_file.infolist(): tar_info = TarInfo(name=zip_info.filename) tar_info.size = zip_info.file_size tar_info.mtime = datetime.now().timestamp() # https://stackoverflow.com/a/434689/11722440 tar_info.mode = zip_info.external_attr >> 16 # https://stackoverflow.com/a/18432983/11722440 # https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT # TODO whg fix other file (like symbolic link) in zip to regular file in tar file if zip_info.filename.endswith('/'): tar_info.type = tarfile.DIRTYPE else: tar_info.type = tarfile.REGTYPE infile = zip_file.open(zip_info.filename) tar_file.addfile(tar_info, infile) except Exception as e: raise finally: tar_file.close() zip_file.close()
def writeDataFile(self, filename, text, content_type, subdir=None): """ See IExportContext. """ if subdir is not None: filename = '/'.join((subdir, filename)) parents = filename.split('/')[:-1] while parents: path = '/'.join(parents) + '/' if path not in self._archive.getnames(): info = TarInfo(path) info.type = DIRTYPE # tarfile.filemode(0o755) == '-rwxr-xr-x' info.mode = 0o755 info.mtime = time.time() self._archive.addfile(info) parents.pop() info = TarInfo(filename) if isinstance(text, six.text_type): encoding = self.getEncoding() or 'utf-8' text = text.encode(encoding) if isinstance(text, six.binary_type): stream = BytesIO(text) info.size = len(text) else: # Assume text is a an instance of a class like # Products.Archetypes.WebDAVSupport.PdataStreamIterator, # as in the case of ATFile stream = text.file info.size = text.size info.mtime = time.time() self._archive.addfile(info, stream)
def move_certs(self, paths): self.log.info("Staging internal ssl certs for %s", self._log_name) yield self.pull_image(self.move_certs_image) # create the volume volume_name = self.format_volume_name(self.certs_volume_name, self) # create volume passes even if it already exists self.log.info("Creating ssl volume %s for %s", volume_name, self._log_name) yield self.docker('create_volume', volume_name) # create a tar archive of the internal cert files # docker.put_archive takes a tarfile and a running container # and unpacks the archive into the container nb_paths = {} tar_buf = BytesIO() archive = TarFile(fileobj=tar_buf, mode='w') for key, hub_path in paths.items(): fname = os.path.basename(hub_path) nb_paths[key] = '/certs/' + fname with open(hub_path, 'rb') as f: content = f.read() tarinfo = TarInfo(name=fname) tarinfo.size = len(content) tarinfo.mtime = os.stat(hub_path).st_mtime tarinfo.mode = 0o644 archive.addfile(tarinfo, BytesIO(content)) archive.close() tar_buf.seek(0) # run a container to stage the certs, # mounting the volume at /certs/ host_config = self.client.create_host_config( binds={ volume_name: {"bind": "/certs", "mode": "rw"}, }, ) container = yield self.docker('create_container', self.move_certs_image, volumes=["/certs"], host_config=host_config, ) container_id = container['Id'] self.log.debug( "Container %s is creating ssl certs for %s", container_id[:12], self._log_name, ) # start the container yield self.docker('start', container_id) # stage the archive to the container try: yield self.docker( 'put_archive', container=container_id, path='/certs', data=tar_buf, ) finally: yield self.docker('remove_container', container_id) return nb_paths
def addFile(tar, dest, file, file_size): if dest not in written_files: info = TarInfo(dest) info.size = file_size info.mtime = now info.mode = 0o777 tar.addfile(info, fileobj=file) written_files.add(dest)
def file_filter(info: tarfile.TarInfo): info.mode = 0o00777 if executable else 0o00666 info.mtime = 0 info.type = tarfile.REGTYPE info.uid = info.gid = 0 info.uname = info.gname = "root" info.pax_headers = {} return info
def addFile(tar, dest, file, file_size): if dest not in written_files: info = TarInfo(dest) info.size = file_size info.mtime = now info.mode = 0777 tar.addfile(info, fileobj=file) written_files.add(dest)
def create_dir(self, path): """Create a directory within the tarfile. :param path: the path to put the directory at. """ tarinfo = TarInfo(name=path) tarinfo.type = DIRTYPE tarinfo.mode = 0755 self._set_defaults(tarinfo) self.addfile(tarinfo)
def add_to_tar(tar: TarFile, data: bytes, filename: str): tarinfo = TarInfo(name=filename) tarinfo.size = len(data) tarinfo.mtime = int(datetime.timestamp(datetime.utcnow())) tarinfo.mode = 436 tarinfo.type = b'0' tarinfo.uid = tarinfo.gid = 0 tarinfo.uname = tarinfo.gname = "0" tar.addfile(tarinfo, BytesIO(data))
def addString(tar, dest, string): if dest not in written_files: print dest, string info = TarInfo(dest) info.size = len(string) info.mtime = now info.mode = 0777 file = StringIO(string) tar.addfile(info, fileobj=file) file.close() written_files.add(dest)
def addBytes(tar, dest, bytes): if dest not in written_files: # print dest, string info = TarInfo(dest) info.size = len(bytes) info.mtime = now info.mode = 0o777 file = BytesIO(bytes) tar.addfile(info, fileobj=file) file.close() written_files.add(dest)
def _set_uid_gid(tarinfo: TarInfo) -> TarInfo: tarinfo.uid = tarinfo.gid = 0 tarinfo.uname = tarinfo.gname = '' # Set 644 permissions, leaving higher bits of st_mode unchanged new_mode = (tarinfo.mode | 0o644) & ~0o133 if tarinfo.mode & 0o100: new_mode |= 0o111 # Executable: 644 -> 755 tarinfo.mode = new_mode return tarinfo
def write_package(): tbs = ForgeClient.UPLOAD_TAR_BUFFER_SIZE with TarFile.open(mode="w|gz", fileobj=body, bufsize=tbs, dereference=True) as tar: for file in files: self.debug("Sending %s", file) ti = TarInfo(file) fp = os.path.join(self.path, file) ti.size = os.path.getsize(fp) ti.mode = 0o666 with open(fp, "rb") as fd: tar.addfile(ti, fileobj=fd) body.close()
def _sanitize_tar(tarinfo: TarInfo) -> TarInfo or None: if '__pycache__' in tarinfo.name: return None tarinfo.uid = tarinfo.gid = 0 tarinfo.uname = tarinfo.gname = '' # Set 644 permissions, leaving higher bits of st_mode unchanged new_mode = (tarinfo.mode | 0o644) & ~0o133 if tarinfo.mode & 0o100: new_mode |= 0o111 # Executable: 644 -> 755 tarinfo.mode = new_mode return tarinfo
def uploadDF(dataflowName): dataflowStr = None udfs = {} dataflowPath = os.path.join(path, "dataflows", dataflowName) with open(os.path.join(dataflowPath, "dataflowInfo.json"), 'r') as df: dataflowStr = df.read() if os.path.exists(dataflowPath + "/udfs/"): for udf in os.listdir(os.path.join(dataflowPath, "udfs")): with open(os.path.join(dataflowPath, "udfs", udf), 'r') as udfFile: udfs[udf] = udfFile.read() retinaBuf = io.BytesIO() with tarfile.open(fileobj=retinaBuf, mode="w:gz") as tar: info = TarInfo("dataflowInfo.json") info.size = len(dataflowStr) tar.addfile(info, io.BytesIO(bytearray(dataflowStr, "utf-8"))) # # ##udfs directory if udfs: info = TarInfo("udfs") info.type = tarfile.DIRTYPE info.mode = 0o755 tar.addfile(info) # ##Add udf to the above dir for udfName, udfCode in udfs.items(): info = TarInfo(name="udfs/" + udfName) info.size = len(udfCode) info.mode = 0o755 tar.addfile(info, io.BytesIO(bytearray(udfCode, "utf-8"))) try: retina.delete(dataflowName) except: print("Dataflow deletion failed!", dataflowName, availableRetinas) retina.add(dataflowName, retinaBuf.getvalue())
def _add_entry( self, name: str, type: bytes, mode: int, mtime: int, size: int, data: Optional[IO[bytes]], linkname: str = "", ) -> None: info = TarInfo(name) info.type = type info.mode = mode info.size = size info.mtime = mtime info.linkname = linkname return self._inner.addfile(info, data)
def file_write(self, path, content, mode=None, owner=None, group=None, append=False, hide=False, sudo=False): """ Writes a file to the container @param path: path of the file @param content: content to be put in the file @param mode: file mode @param owner: owner of the file @param group: group of the file @param append: append content to the file @param hide: hide (debug) logs @raise runtimeError: path for file couldn't be created """ if append and self.exists(path): content = self.file_read(path) + content file_name = os.path.basename(path) dir_name = os.path.dirname(path) buf = BytesIO() with TarFile("write_file", mode='w', fileobj=buf) as tarf: f = BytesIO() length = f.write(content.encode('utf8')) f.seek(0) tari = TarInfo(name=file_name) tari.size = length if not mode is None: tari.mode = mode if not owner is None: tari.uname = owner if not group is None: tari.gname = group tarf.addfile(tari, f) if not self.exists(dir_name): result = self.container.exec_run("mkdir -p %s" % dir_name) if result.exit_code != 0: raise RuntimeError("Could not create path %s!\n%s" % (dir_name, result.output)) self.container.put_archive(dir_name, buf.getvalue())
def get_image(self, image): if not image: raise APIError(HTTPError('500 Server Error'), None, explanation='Usage: image_export IMAGE [IMAGE...]') layers = [] next_layer_id = image while next_layer_id: layer = normalizeimage(self._findlayer(next_layer_id), copy=True) layers.append(layer) next_layer_id = layers[-1][':parent_id'] image_file = BytesIO() mtime = time() with tarfile_open(mode='w', fileobj=image_file) as image_tar_file: for layer in layers: ti_dir = TarInfo(layer[':id']) ti_dir.mtime = mtime ti_dir.mode = 0o755 ti_dir.type = DIRTYPE image_tar_file.addfile(ti_dir) layer_tar_src_path = ospath_join(self._my_dir, 'data', layer[':short_id'], 'layer.tar') with open(layer_tar_src_path, 'rb') as layer_tar_src_file: layer_tar_dst_path = '{}/layer.tar'.format(layer[':id']) ti_layer = image_tar_file.gettarinfo( layer_tar_src_path, layer_tar_dst_path) ti_layer.mtime = mtime ti_layer.mode = 0o644 ti_layer.uid = ti_layer.gid = 0 ti_layer.uname = ti_layer.gname = '' image_tar_file.addfile(ti_layer, fileobj=layer_tar_src_file) image_file.seek(0) return image_file
def _reset_entry_attrs(tarinfo: tarfile.TarInfo): """ filter function for tar creation that will remove all file attributes (uid, gid, mtime) from the file added to tar would can make the build of package not reproducible. Args: tarinfo: entry being added to the tar Returns: the modified tarinfo that will be actually added to tar """ tarinfo.uid = tarinfo.gid = 0 tarinfo.uname = tarinfo.gname = 'root' tarinfo.mtime = 0 if tarinfo.name.lower().endswith('.dll'): tarinfo.mode = 0o755 return tarinfo
def writeDataFile(self, filename, text, content_type, subdir=None): """ See IExportContext. """ if subdir is not None: filename = '/'.join((subdir, filename)) parents = filename.split('/')[:-1] while parents: path = '/'.join(parents) + '/' if path not in self._archive.getnames(): info = TarInfo(path) info.type = DIRTYPE # tarfile.filemode(0755) == '-rwxr-xr-x' info.mode = 0755 info.mtime = time.time() self._archive.addfile(info) parents.pop() stream = StringIO(text) info = TarInfo(filename) info.size = len(text) info.mtime = time.time() self._archive.addfile(info, stream)
def get_image(self, image): if not image: raise APIError(HTTPError('500 Server Error'), None, explanation='Usage: image_export IMAGE [IMAGE...]') layers = [] next_layer_id = image while next_layer_id: layer = normalizeimage(self._findlayer(next_layer_id), copy=True) layers.append(layer) next_layer_id = layers[-1][':parent_id'] image_file = BytesIO() mtime = time() with tarfile_open(mode='w', fileobj=image_file) as image_tar_file: for layer in layers: ti_dir = TarInfo(layer[':id']) ti_dir.mtime = mtime ti_dir.mode = 0o755 ti_dir.type = DIRTYPE image_tar_file.addfile(ti_dir) layer_tar_src_path = ospath_join(self._my_dir, 'data', layer[':short_id'], 'layer.tar') with open(layer_tar_src_path, 'rb') as layer_tar_src_file: layer_tar_dst_path = '{}/layer.tar'.format(layer[':id']) ti_layer = image_tar_file.gettarinfo(layer_tar_src_path, layer_tar_dst_path) ti_layer.mtime = mtime ti_layer.mode = 0o644 ti_layer.uid = ti_layer.gid = 0 ti_layer.uname = ti_layer.gname = '' image_tar_file.addfile(ti_layer, fileobj=layer_tar_src_file) image_file.seek(0) return image_file
def tar(host, backup, share, path): binary_stdout = stdout.buffer fbak = Fruitbak(confdir = Path('/dev/shm/conf')) backup = fbak[host][backup] if path is None: share, path = backup.locate_path(share) else: share = backup[share] def iterator(): for dentry in share.find(path): if dentry.is_file and not dentry.is_hardlink: yield from dentry.hashes with fbak.pool.agent().readahead(iterator()) as reader: for dentry in share.find(path): name = dentry.name or b'.' i = TarInfo(fsdecode(bytes(name))) i.mode = dentry.mode & 0o7777 i.uid = dentry.uid i.gid = dentry.gid i.mtime = dentry.mtime // 1000000000 if dentry.is_hardlink: i.type = LNKTYPE hardlink = dentry.hardlink or b'.' i.linkname = fsdecode(bytes(hardlink)) elif dentry.is_file: i.type = REGTYPE i.size = dentry.size elif dentry.is_symlink: i.type = SYMTYPE i.linkname = fsdecode(bytes(dentry.symlink)) elif dentry.is_chardev: i.type = CHRTYPE i.devmajor = dentry.major i.devminor = dentry.minor elif dentry.is_blockdev: i.type = BLKTYPE i.devmajor = dentry.major i.devminor = dentry.minor elif dentry.is_directory: i.type = DIRTYPE elif dentry.is_fifo: i.type = FIFOTYPE else: continue binary_stdout.write(i.tobuf(GNU_FORMAT)) if dentry.is_file and not dentry.is_hardlink: for hash in dentry.hashes: action = next(reader) if action.exception: raise action.exception[1] binary_stdout.write(action.value) padding = -i.size % BLOCKSIZE if padding: binary_stdout.write(bytes(padding)) binary_stdout.write(b'\0' * (BLOCKSIZE*2))
def compute(self, conn, data=None): tarinfo = TarInfo() tarinfo.name = self.name tarinfo.mode = 0o700 tarinfo.uid = 0 tarinfo.gid = 0 tarinfo.type = REGTYPE tarinfo.linkname = "" if self.name == CONTAINER_PROPERTIES: meta = data or conn.container_get_properties(self.acct, self.ref) tarinfo.size = len(json.dumps(meta['properties'], sort_keys=True)) self._filesize = tarinfo.size self._buf = tarinfo.tobuf(format=PAX_FORMAT, encoding='utf-8') return elif self.name == CONTAINER_MANIFEST: tarinfo.size = len(json.dumps(data, sort_keys=True)) self._filesize = tarinfo.size self._buf = tarinfo.tobuf(format=PAX_FORMAT, encoding='utf-8') return entry = conn.object_get_properties(self.acct, self.ref, self.name) properties = entry['properties'] # x-static-large-object if properties.get(SLO, False): tarinfo.size = int(properties.get(SLO_SIZE)) _, slo = conn.object_fetch(self.acct, self.ref, self.name, properties=False) self._slo = json.loads(b''.join(slo).decode('utf-8'), object_pairs_hook=OrderedDict) self._checksums = {} # format MD5 to share same format as multi chunks object offset = 0 for idx, ck in enumerate(self._slo): self._checksums[idx] = { 'hash': ck['hash'].upper(), 'size': ck['bytes'], 'offset': offset } offset += ck['bytes'] else: tarinfo.size = int(entry['length']) meta, chunks = conn.object_locate(self.acct, self.ref, self.name, properties=False) storage_method = STORAGE_METHODS.load(meta['chunk_method']) chunks = _sort_chunks(chunks, storage_method.ec) for idx in chunks: chunks[idx] = chunks[idx][0] del chunks[idx]['url'] del chunks[idx]['score'] del chunks[idx]['pos'] self._checksums = chunks self._filesize = tarinfo.size # XATTR # do we have to store basic properties like policy, ... ? for key, val in properties.items(): assert isinstance(val, string_types), \ "Invalid type for %s:%s:%s" % (self.acct, self.name, key) if self.slo and key in SLO_HEADERS: continue tarinfo.pax_headers[SCHILY + key] = val tarinfo.pax_headers['mime_type'] = entry['mime_type'] self._buf = tarinfo.tobuf(format=PAX_FORMAT, encoding='utf-8')