def compress_files(files, symlinks, name, dest_dir): """Compress the package and returns the new dict (name => content) of files, only with the conanXX files and the compressed file""" t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) with open(tgz_path, "wb") as tgz_handle: # tgz_contents = BytesIO() tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) for filename, dest in sorted(symlinks.items()): info = tarfile.TarInfo(name=filename) info.type = tarfile.SYMTYPE info.linkname = dest tgz.addfile(tarinfo=info) mask = ~(stat.S_IWOTH | stat.S_IWGRP) for filename, abs_path in sorted(files.items()): info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode & mask if os.path.islink(abs_path): info.type = tarfile.SYMTYPE info.linkname = os.readlink(abs_path) # @UndefinedVariable tgz.addfile(tarinfo=info) else: with open(abs_path, 'rb') as file_handler: tgz.addfile(tarinfo=info, fileobj=file_handler) tgz.close() duration = time.time() - t1 log_compressed_files(files, duration, tgz_path) return tgz_path
def compress_files(files, symlinks, name, dest_dir, output=None): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) for filename, dest in sorted(symlinks.items()): info = tarfile.TarInfo(name=filename) info.type = tarfile.SYMTYPE info.linkname = dest info.size = 0 # A symlink shouldn't have size tgz.addfile(tarinfo=info) mask = ~(stat.S_IWOTH | stat.S_IWGRP) with progress_bar.iterate_list_with_progress(sorted(files.items()), output, "Compressing %s" % name) as pg_file_list: for filename, abs_path in pg_file_list: info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode & mask if os.path.islink(abs_path): info.type = tarfile.SYMTYPE info.size = 0 # A symlink shouldn't have size info.linkname = os.readlink(abs_path) # @UndefinedVariable tgz.addfile(tarinfo=info) else: with open(abs_path, 'rb') as file_handler: tgz.addfile(tarinfo=info, fileobj=file_handler) tgz.close() duration = time.time() - t1 log_compressed_files(files, duration, tgz_path) return tgz_path
def compress_files(files, symlinks, name, dest_dir): """Compress the package and returns the new dict (name => content) of files, only with the conanXX files and the compressed file""" t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) with open(tgz_path, "wb") as tgz_handle: # tgz_contents = BytesIO() tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) for filename, dest in sorted(symlinks.items()): info = tarfile.TarInfo(name=filename) info.type = tarfile.SYMTYPE info.linkname = dest tgz.addfile(tarinfo=info) mask = ~(stat.S_IWOTH | stat.S_IWGRP) for filename, abs_path in sorted(files.items()): info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode & mask if os.path.islink(abs_path): info.type = tarfile.SYMTYPE info.linkname = os.readlink(abs_path) # @UndefinedVariable tgz.addfile(tarinfo=info) else: with open(abs_path, 'rb') as file_handler: tgz.addfile(tarinfo=info, fileobj=file_handler) tgz.close() duration = time.time() - t1 log_compressed_files(files, duration, tgz_path) return tgz_path
def compress_files(files, symlinks, name, dest_dir, output=None): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) with set_dirty_context_manager(tgz_path), open(tgz_path, "wb") as tgz_handle: tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) for filename, dest in sorted(symlinks.items()): info = tarfile.TarInfo(name=filename) info.type = tarfile.SYMTYPE info.linkname = dest info.size = 0 # A symlink shouldn't have size tgz.addfile(tarinfo=info) mask = ~(stat.S_IWOTH | stat.S_IWGRP) i_file = 0 n_files = len(files) last_progress = None if output and n_files > 1 and not output.is_terminal: output.write("[") elif output and n_files > 1 and output.is_terminal: progress_bar = tqdm(total=len(files), desc="Compressing %s" % name, unit="files", leave=True, dynamic_ncols=False, ascii=True, file=output) for filename, abs_path in sorted(files.items()): info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode & mask if os.path.islink(abs_path): info.type = tarfile.SYMTYPE info.size = 0 # A symlink shouldn't have size info.linkname = os.readlink(abs_path) # @UndefinedVariable tgz.addfile(tarinfo=info) else: with open(abs_path, 'rb') as file_handler: tgz.addfile(tarinfo=info, fileobj=file_handler) if output and n_files > 1: i_file = i_file + 1 units = min(50, int(50 * i_file / n_files)) if last_progress != units: # Avoid screen refresh if nothing has change if not output.is_terminal: output.write('=' * (units - (last_progress or 0))) last_progress = units if output.is_terminal: progress_bar.update() if output and n_files > 1: if output.is_terminal: progress_bar.close() else: output.writeln("]") tgz.close() duration = time.time() - t1 log_compressed_files(files, duration, tgz_path) return tgz_path
def compress_files(files, symlinks, name, dest_dir, output=None): t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) set_dirty(tgz_path) with open(tgz_path, "wb") as tgz_handle: # tgz_contents = BytesIO() tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) for filename, dest in sorted(symlinks.items()): info = tarfile.TarInfo(name=filename) info.type = tarfile.SYMTYPE info.linkname = dest info.size = 0 # A symlink shouldn't have size tgz.addfile(tarinfo=info) mask = ~(stat.S_IWOTH | stat.S_IWGRP) i_file = 0 n_files = len(files) last_progress = None if output and n_files > 1 and not output.is_terminal: output.write("[") for filename, abs_path in sorted(files.items()): info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode & mask if os.path.islink(abs_path): info.type = tarfile.SYMTYPE info.size = 0 # A symlink shouldn't have size info.linkname = os.readlink(abs_path) # @UndefinedVariable tgz.addfile(tarinfo=info) else: with open(abs_path, 'rb') as file_handler: tgz.addfile(tarinfo=info, fileobj=file_handler) if output and n_files > 1: i_file = i_file + 1 units = min(50, int(50 * i_file / n_files)) if last_progress != units: # Avoid screen refresh if nothing has change if output.is_terminal: text = "%s/%s files" % (i_file, n_files) output.rewrite_line("[%s%s] %s" % ('=' * units, ' ' * (50 - units), text)) else: output.write('=' * (units - (last_progress or 0))) last_progress = units if output and n_files > 1: if output.is_terminal: output.writeln("") else: output.writeln("]") tgz.close() clean_dirty(tgz_path) duration = time.time() - t1 log_compressed_files(files, duration, tgz_path) return tgz_path
def compress_files(files, symlinks, name, dest_dir): """Compress the package and returns the new dict (name => content) of files, only with the conanXX files and the compressed file""" t1 = time.time() # FIXME, better write to disk sequentially and not keep tgz contents in memory tgz_path = os.path.join(dest_dir, name) is_export_sources = (name == EXPORT_SOURCES_TGZ_NAME) with open(tgz_path, "wb") as tgz_handle: # tgz_contents = BytesIO() tgz = gzopen_without_timestamps(name, mode="w", fileobj=tgz_handle) for filename, dest in symlinks.items(): info = tarfile.TarInfo(name=filename) info.type = tarfile.SYMTYPE info.linkname = dest tgz.addfile(tarinfo=info) for filename, abs_path in files.items(): if is_export_sources: # temporary backwards compat TGZ creation filename = ".c_src/%s" % filename info = tarfile.TarInfo(name=filename) info.size = os.stat(abs_path).st_size info.mode = os.stat(abs_path).st_mode if os.path.islink(abs_path): info.type = tarfile.SYMTYPE info.linkname = os.readlink(abs_path) # @UndefinedVariable tgz.addfile(tarinfo=info) else: with open(abs_path, 'rb') as file_handler: tgz.addfile(tarinfo=info, fileobj=file_handler) tgz.close() duration = time.time() - t1 log_compressed_files(files, duration, tgz_path) return tgz_path