def extract_orig_tarball(tarball_filename,
component,
target,
strip_components=None):
"""Extract an orig tarball.
:param tarball: Path to the tarball
:param component: Component name (or None for top-level)
:param target: Target path
"""
from tarfile import TarFile
tar_args = ["tar"]
if tarball_filename.endswith(".tar.bz2"):
tar_args.append('xjf')
tf = TarFile.bz2open(tarball_filename)
elif (tarball_filename.endswith(".tar.lzma")
or tarball_filename.endswith(".tar.xz")):
tar_args.append('xJf')
tf = TarFile.xzopen(tarball_filename)
elif tarball_filename.endswith(".tar"):
tar_args.append('xf')
tf = TarFile.open(tarball_filename)
elif (tarball_filename.endswith(".tar.gz")
or tarball_filename.endswith(".tgz")):
tf = TarFile.gzopen(tarball_filename)
tar_args.append('xzf')
else:
note('Unable to figure out type of %s, '
'assuming .tar.gz', tarball_filename)
tf = TarFile.gzopen(tarball_filename)
tar_args.append('xzf')
try:
if strip_components is None:
if needs_strip_components(tf):
strip_components = 1
else:
strip_components = 0
finally:
tf.close()
if component is not None:
target_path = os.path.join(target, component)
os.mkdir(target_path)
else:
target_path = target
tar_args.extend([tarball_filename, "-C", target_path])
if strip_components is not None:
tar_args.extend(["--strip-components", str(strip_components)])
proc = subprocess.Popen(tar_args,
preexec_fn=subprocess_setup,
stderr=subprocess.PIPE)
(stdout, stderr) = proc.communicate()
if proc.returncode != 0:
raise TarFailed("extract", tarball_filename, error=stderr)
def __init__(self, path):
self.fd = TarFile.gzopen(path)
self.pkg_info = defaultdict(list)
self.members = None
if self.parse_pkginfo():
self.parse_contents()
def archive_add_dependency(archived_prefix: Path,
dependency_path: Path,
strip_components: int) -> None:
print(f"Adding files from {dependency_path.name}")
with TarFile.gzopen(dependency_path) as dependency_archive:
for file_info in dependency_archive:
archived_path = str(
Path(
archived_prefix,
*Path(file_info.name).parts[strip_components:],
))
if file_info.isreg():
file_reader = dependency_archive.extractfile(file_info)
assert file_reader is not None
archive.add_file_entry(archived_path,
file_reader.read(),
mtime=file_info.mtime)
elif file_info.issym():
archive.add_symlink_entry(archived_path,
file_info.linkname,
mtime=file_info.mtime)
elif file_info.isdir():
# Directories are deliberately ignored because the previous setup
# didn't put them into resulting archives, and their entries are
# useless to us anyway.
pass
else:
# Other file types (character devices, block devices and named
# pipes) are UNIX-specific and can't be handled by Zip, but it's
# not like they are used in dependencies anyway. Correction: well,
# after checking APPNOTE.TXT section 4.5.7 I noticed that these
# exotic file types may be supported, but it's not like any modding
# projects use those.
pass
def getControl(filename):
from cStringIO import StringIO
from tarfile import TarFile
file = open(filename)
if file.read(8) != "!<arch>\n":
raise ValueError, "Invalid file"
while True:
name = file.read(16)
date = file.read(12)
uid = file.read(6)
gid = file.read(6)
mode = file.read(8)
size = file.read(10)
magic = file.read(2)
if name == "control.tar.gz ":
data = file.read(int(size))
sio = StringIO(data)
tf = TarFile.gzopen("", fileobj=sio)
try:
control = tf.extractfile("./control")
except KeyError:
control = tf.extractfile("control")
return control.read()
else:
file.seek(int(size), 1)
return None
def importData(self, archive):
if not archive:
raise SwineException(self.tr("File name cannot be empty"))
os.chdir(self.getPath())
with TarFile.gzopen(archive, "r") as tar:
tar.extractall(".")
self.loadConfig()
def test_attrsFromFile(self):
"""
Extracting a simple AppleDouble file representing some extended
attributes should result in a dictionary of those attributes.
"""
tarfile = TarFile.gzopen("sample.tgz", fileobj=StringIO(simpleTarWithXattrs))
self.assertEqual(attrsFromFile(tarfile.extractfile("./._f")), {"alpha": "beta", "gamma": "delta"})
def getControl(filename):
from cStringIO import StringIO
from tarfile import TarFile
file = open(filename)
if file.read(8) != "!<arch>\n":
raise ValueError, "Invalid file"
while True:
name = file.read(16)
date = file.read(12)
uid = file.read(6)
gid = file.read(6)
mode = file.read(8)
size = file.read(10)
magic = file.read(2)
if name == "control.tar.gz ":
data = file.read(int(size))
sio = StringIO(data)
tf = TarFile.gzopen("", fileobj=sio)
try:
control = tf.extractfile("./control")
except KeyError:
control = tf.extractfile("control")
return control.read()
else:
file.seek(int(size), 1)
return None
def get_local_file_deps(self, fname):
from os import mkdir, system
from os.path import exists
from tarfile import TarFile
from time import asctime, localtime
if exists("/tmp/gtkpacman"):
system("rm -rf /tmp/gtkpacman")
mkdir("/tmp/gtkpacman", 0755)
archive = TarFile.gzopen(fname)
for member in archive.getmembers():
archive.extract(member, "/tmp/gtkpacman")
continue
info_file = file("/tmp/gtkpacman/.PKGINFO")
infos = info_file.read()
info_file.close()
infos_lines = infos.splitlines()
deps = []
conflicts = []
for line in infos_lines:
sides = line.split(" = ")
if sides[0] == "depend":
deps.append(sides[1])
elif sides[0] == "conflict":
conflicts.append(sides[1])
continue
system("rm -rf /tmp/gtkpacman")
return deps, conflicts
def test_attrsFromFile(self):
"""
Extracting a simple AppleDouble file representing some extended
attributes should result in a dictionary of those attributes.
"""
tarfile = TarFile.gzopen('sample.tgz',
fileobj=StringIO(simpleTarWithXattrs))
self.assertEqual(attrsFromFile(tarfile.extractfile("./._f")),
{"alpha": "beta", "gamma": "delta"})
def test_pack_plugin():
data = pack_plugin(skeleton_path, 'skeleton')
files = find_files(skeleton_path, 'skeleton')
io = StringIO(data)
tarfile = TarFile.gzopen(None,fileobj=io)
tar_files = [info.name for info in tarfile]
for norm, tar in zip(files, tar_files):
tar = path.normpath(tar)
assert norm == tar
def create_mqtt_gzip(f: BinaryIO, root: str = ROOT) -> None:
def exclude_bad(ti: TarInfo) -> Union[TarInfo, None]:
for r in EXCLUDE_REGEXES:
if r.search(ti.name):
return None
return ti
log.debug("Compressing MQTT deployment files")
with TarFile.gzopen("mqtt-squeeze", mode="w", fileobj=f) as tf:
tf.add(path.join(ROOT, 'mqtt_squeeze.py'), arcname='mqtt_squeeze.py')
for d in ['etc', 'squeezealexa']:
tf.add(path.join(root, d), arcname=d, filter=exclude_bad)
if not [fn for fn in tf.getnames() if fn.endswith('.pem.crt')]:
raise Error("Can't find any certs (.pem.crt files). "
"Make sure you create these first in /etc/certs")
log.debug("All files: %s", ", ".join(tf.getnames()))
def test_propertiesFromTarball(self):
"""
Extracting a tarball with included AppleDouble WebDAV property
information should allow properties to be retrieved using
L{PropertyStore}.
"""
tf = TarFile.gzopen("sample.tgz", fileobj=StringIO(samplePropertyTar))
tmpdir = self.mktemp()
# Note that 'tarfile' doesn't know anything about xattrs, so while OS
# X's 'tar' will restore these as actual xattrs, the 'tarfile' module
# will drop "._" parallel files into the directory structure on all
# platforms.
tf.extractall(tmpdir)
props = PropertyStore("bob", lambda: FilePath(tmpdir).child("f"))
self.assertEqual(props[PropertyName.fromElement(HRef)], HRef("http://sample.example.com/"))
self.assertEqual(props[PropertyName.fromElement(Depth)], Depth("1"))
self.assertEqual(props[PropertyName.fromElement(GETContentType)], GETContentType("text/example"))
def test_propertiesFromTarball(self):
"""
Extracting a tarball with included AppleDouble WebDAV property
information should allow properties to be retrieved using
L{PropertyStore}.
"""
tf = TarFile.gzopen('sample.tgz', fileobj=StringIO(samplePropertyTar))
tmpdir = self.mktemp()
# Note that 'tarfile' doesn't know anything about xattrs, so while OS
# X's 'tar' will restore these as actual xattrs, the 'tarfile' module
# will drop "._" parallel files into the directory structure on all
# platforms.
tf.extractall(tmpdir)
props = PropertyStore("bob", lambda: FilePath(tmpdir).child('f'))
self.assertEqual(props[PropertyName.fromElement(HRef)],
HRef("http://sample.example.com/"))
self.assertEqual(props[PropertyName.fromElement(Depth)], Depth("1"))
self.assertEqual(props[PropertyName.fromElement(GETContentType)],
GETContentType("text/example"))
def main(self, options, args):
if options.quiet:
logging.getLogger().setLevel(logging.WARNING)
if options.install_rdflib:
logging.info("downloading rdflib")
import os
from subprocess import Popen
from tarfile import TarFile
from urllib2 import urlopen, Request
from StringIO import StringIO
url = "http://rdflib.net/rdflib.tgz"
headers = {'User-agent': 'redfoot.py (%s)' % __version__}
f = urlopen(Request(url, None, headers))
sio = StringIO(f.read())
sio.seek(0)
tar = TarFile.gzopen("rdflib.tgz", fileobj=sio)
logging.info("extracting rdflib")
for member in tar:
if member.name.endswith("setup.py"):
setup = member.name
tar.extract(member)
dir, file = os.path.split(setup)
os.chdir(dir)
logging.info("installing rdflib")
p = Popen([sys.executable, "setup.py", "install"])
p.wait()
if "rdflib" in sys.modules:
del sys.modules["rdflib"]
try:
import rdflib
logging.info("rdflib %s installed" % rdflib.__version__)
except ImportError, e:
logging.info("rdflib not installed: %s" % e)
sys.exit()
def __init__(self, path):
if path.endswith('.gz'):
self.fd = TarFile.gzopen(path)
elif path.endswith('.xz'):
self.fd = TarFile.open(fileobj=LZMAFile(path))
else:
raise Exception('Unsupported file type %s' % path)
self.pkg_info = defaultdict(list)
self.members = []
# Extract most used information
if self.parse_pkginfo():
self.parse_contents()
self.name = self.pkg_info.get('pkgname')
self.desc = self.pkg_info.get('pkgdesc')[0]
self.depends = self.pkg_info.get('depend') or []
self.groups = self.pkg_info.get('group') or []
if isinstance(self.name, (list, tuple)) and len(self.name) == 1:
self.name = self.name[0]
from tarfile import TarFile
from zipfile import ZipFile
path = 'g:/geek/'
for i in range(0, 300)[::-1]:
extract_path = path + str(i)
#put_path =path +str(i + 1)
file = open(extract_path, 'rb')
strr = file.read()
try:
if "\x50\x4B\x03\x04" in strr:
tar = ZipFile(extract_path)
tar.extract(str(i - 1), path)
tar.close()
file.close()
elif "\x1F\x8B\x08" in strr:
tar = TarFile.gzopen(extract_path)
tar.extract(str(i - 1), path)
tar.close()
file.close()
elif "\x42\x5A\x68\x39" in strr:
tar = TarFile.bz2open(extract_path)
tar.extract(str(i - 1), path)
tar.close()
file.close()
file.close()
except:
print "something error!"
def package(target, source, env):
"""Builder action for packaging the distribution archives."""
# Print out.
print('')
print("#######################")
print("# Packaging the files #")
print("#######################")
# List of distribution files.
type_list = [env['DIST_TYPE']]
if type_list[0] == 'ALL':
type_list = ['zip', 'tar']
# Loop over the distribution files.
for dist_type in type_list:
# The file name.
if dist_type == 'zip':
file = env['DIST_FILE'] + '.zip'
elif dist_type == 'tar':
file = env['DIST_FILE'] + '.tar.bz2'
elif dist_type == 'dmg':
file = env['DIST_FILE'] + '.dmg'
# Print out.
print("\n\nCreating the package distribution " + repr(file) + ".\n")
# Create the special Mac OS X DMG file and then stop execution.
if dist_type == 'dmg':
# Create the Mac OS X universal application.
print("\n# Creating the Mac OS X universal application.\n\n")
cmd = '%s setup.py py2app' % sys.executable
print("%s\n" % cmd)
pipe = Popen(cmd, shell=True, stdin=PIPE, close_fds=False)
waitpid(pipe.pid, 0)
# Create the dmg image.
print("\n\n# Creating the DMG image.\n\n")
cmd = 'hdiutil create -ov -fs HFS+ -volname "relax" -srcfolder dist/relax.app ../%s' % file
print("%s\n" % cmd)
pipe = Popen(cmd, shell=True, stdin=PIPE, close_fds=False)
waitpid(pipe.pid, 0)
# Stop executing.
return
# Open the Zip distribution file.
if dist_type == 'zip':
archive = ZipFile(path.pardir + path.sep + file,
'w',
compression=8)
# Open the Tar distribution file.
elif dist_type == 'tar':
if search('.bz2$', file):
archive = TarFile.bz2open(path.pardir + path.sep + file, 'w')
elif search('.gz$', file):
archive = TarFile.gzopen(path.pardir + path.sep + file, 'w')
else:
archive = TarFile.open(path.pardir + path.sep + file, 'w')
# Base directory.
base = getcwd() + sep
# Walk through the directories.
for root, dirs, files in walk(getcwd()):
# Skip the subversion directories.
if search("\.svn", root):
continue
# Add the files in the current directory to the archive.
for i in range(len(files)):
# Skip any '.sconsign' files, hidden files, byte-compiled '*.pyc' files, or binary objects '.o', '.os', 'obj', 'lib', and 'exp'.
if search("\.sconsign",
files[i]) or search("^\.", files[i]) or search(
"\.pyc$", files[i]) or search(
"\.o$", files[i]) or search(
"\.os$", files[i]) or search(
"\.obj$", files[i]) or search(
"\.lib$", files[i]) or search(
"\.exp$", files[i]):
continue
# Create the file name (without the base directory).
name = path.join(root, files[i])
name = name[len(base):]
print('relax-' + version + path.sep + name)
# The archive file name.
arcname = 'relax-' + version + path.sep + name
# Zip archives.
if dist_type == 'zip':
archive.write(filename=name, arcname=arcname)
# Tar archives.
if dist_type == 'tar':
archive.add(name=name, arcname=arcname)
# Close the archive.
archive.close()
# Final printout.
print("\n\n\n")
description='Make a multi-file tex documents into a single tex file.')
parser.add_argument(
'master_tex', metavar='master', type=unicode,
help='master .tex file to be converted.')
parser.add_argument(
'archive', metavar='archive', type=unicode,
help='output archive file (tar.gz).')
args = parser.parse_args()
archive_base = re.sub(ur'\.tar\.gz$',u'',args.archive)
archive_file = archive_base + '.tar.gz'
master = args.master_tex
with TarFile.gzopen(archive_file, 'w') as arv:
LOGGER.info('Create archive "{}":'.format(archive_file))
with NamedTemporaryFile(prefix='texarv_') as tempfile:
with tempfile.file as tmp:
recursive_print(master, tmp)
arv.add(tempfile.name, arcname='ms.tex')
LOGGER.info(' append manuscript file "ms.tex"')
for items in FIGURE_PATH:
for figpath,arcname in items.iteritems():
arv.add(figpath, arcname=arcname)
LOGGER.info(
' append figure file "{}" as "{}"'.format(figpath, arcname))
for items in MISC_PATH:
for miscpath,arcname in items.iteritems():
if os.path.exists(miscpath):
arv.add(miscpath, arcname=arcname)
# Find the mailbox in which to store the mail
md = None
if re.match('Inbox(\![0-9]+)?$', folder):
md = Maildir(maildir)
elif re.match('Inbox/', folder):
# Nested folders under Inbox, put them under a folder named
# 'INBOX'.
folder = folder.replace('/', '.').replace('Inbox', 'INBOX')
md = Maildir(path.join(maildir, '.' + folder), factory=None)
elif re.match('Sent(\ Items.*)?', folder):
md = Maildir(path.join(maildir, '.' + 'Sent'), factory=None)
else:
md = Maildir(path.join(maildir, '.' + folder), factory=None)
# Store the mail
md.add(msg)
if __name__ == '__main__':
if len(sys.argv) < 3:
print("Usage: {} /path/to/zmmailbox.tgz /path/to/Maildir".format(sys.argv[0]))
exit(1)
with TarFile.gzopen(sys.argv[1]) as tf:
print("Building metadata...")
metadata = get_metadata(tf)
maildir_path = sys.argv[2]
# Create the top Maildir
Maildir(maildir_path)
for m in get_mails(tf):
print("{}".format(m['name'][:20]))
store_mail(tf, m, maildir_path, metadata)
logging.getLogger().setLevel(logging.WARNING)
if options.install_rdflib:
logging.info("downloading rdflib")
import os
from subprocess import Popen
from tarfile import TarFile
from urllib2 import urlopen, Request
from StringIO import StringIO
url = "http://rdflib.net/rdflib.tgz"
headers = {'User-agent': 'redfoot.py (%s)' % __version__}
f = urlopen(Request(url, None, headers))
sio = StringIO(f.read())
sio.seek(0)
tar = TarFile.gzopen("rdflib.tgz", fileobj=sio)
logging.info("extracting rdflib")
for member in tar:
if member.name.endswith("setup.py"):
setup = member.name
tar.extract(member)
dir, file = os.path.split(setup)
os.chdir(dir)
logging.info("installing rdflib")
p = Popen([sys.executable, "setup.py", "install"])
p.wait()
if "rdflib" in sys.modules:
del sys.modules["rdflib"]
try:
def exportData(self, archive):
if not archive:
raise SwineException(self.tr("File name cannot be empty"))
os.chdir(self.getPath())
with TarFile.gzopen(archive, "w") as tar:
tar.add(".") #recursive
def package(target, source, env):
"""Builder action for packaging the distribution archives."""
# Print out.
print('')
print("#######################")
print("# Packaging the files #")
print("#######################")
# List of distribution files.
type_list = [env['DIST_TYPE']]
if type_list[0] == 'ALL':
type_list = ['zip', 'tar']
# Loop over the distribution files.
for dist_type in type_list:
# The file name.
if dist_type == 'zip':
file = env['DIST_FILE'] + '.zip'
elif dist_type == 'tar':
file = env['DIST_FILE'] + '.tar.bz2'
elif dist_type == 'dmg':
file = env['DIST_FILE'] + '.dmg'
# Print out.
print("\n\nCreating the package distribution " + repr(file) + ".\n")
# Create the special Mac OS X DMG file and then stop execution.
if dist_type == 'dmg':
# Create the Mac OS X universal application.
print("\n# Creating the Mac OS X universal application.\n\n")
cmd = '%s setup.py py2app' % sys.executable
print("%s\n" % cmd)
pipe = Popen(cmd, shell=True, stdin=PIPE, close_fds=False)
waitpid(pipe.pid, 0)
# Create the dmg image.
print("\n\n# Creating the DMG image.\n\n")
cmd = 'hdiutil create -ov -fs HFS+ -volname "relax" -srcfolder dist/relax.app ../%s' % file
print("%s\n" % cmd)
pipe = Popen(cmd, shell=True, stdin=PIPE, close_fds=False)
waitpid(pipe.pid, 0)
# Stop executing.
return
# Open the Zip distribution file.
if dist_type == 'zip':
archive = ZipFile(path.pardir + path.sep + file, 'w', compression=8)
# Open the Tar distribution file.
elif dist_type == 'tar':
if search('.bz2$', file):
archive = TarFile.bz2open(path.pardir + path.sep + file, 'w')
elif search('.gz$', file):
archive = TarFile.gzopen(path.pardir + path.sep + file, 'w')
else:
archive = TarFile.open(path.pardir + path.sep + file, 'w')
# Base directory.
base = getcwd() + sep
# Walk through the directories.
for root, dirs, files in walk(getcwd()):
# Skip the subversion directories.
if search("\.svn", root):
continue
# Add the files in the current directory to the archive.
for i in range(len(files)):
# Skip any '.sconsign' files, hidden files, byte-compiled '*.pyc' files, or binary objects '.o', '.os', 'obj', 'lib', and 'exp'.
if search("\.sconsign", files[i]) or search("^\.", files[i]) or search("\.pyc$", files[i]) or search("\.o$", files[i]) or search("\.os$", files[i]) or search("\.obj$", files[i]) or search("\.lib$", files[i]) or search("\.exp$", files[i]):
continue
# Create the file name (without the base directory).
name = path.join(root, files[i])
name = name[len(base):]
print('relax-' + version + path.sep + name)
# The archive file name.
arcname = 'relax-' + version + path.sep + name
# Zip archives.
if dist_type == 'zip':
archive.write(filename=name, arcname=arcname)
# Tar archives.
if dist_type == 'tar':
archive.add(name=name, arcname=arcname)
# Close the archive.
archive.close()
# Final printout.
print("\n\n\n")
def package(target, source, env):
"""Builder action for packaging the distribution archives."""
# Print out.
print('')
print("#######################")
print("# Packaging the files #")
print("#######################")
# List of distribution files.
type_list = [env['DIST_TYPE']]
if type_list[0] == 'ALL':
type_list = ['zip', 'tar']
# Loop over the distribution files.
for dist_type in type_list:
# The file name.
if dist_type == 'zip':
file = env['DIST_FILE'] + '.zip'
elif dist_type == 'tar':
file = env['DIST_FILE'] + '.tar.bz2'
elif dist_type == 'dmg':
file = env['DIST_FILE'] + '.dmg'
# Print out.
print("\n\nCreating the package distribution " + repr(file) + ".\n")
# Create the special Mac OS X DMG file and then stop execution.
if dist_type == 'dmg':
# Create the Mac OS X universal application.
print("\n# Creating the Mac OS X universal application.\n\n")
cmd = '%s setup.py py2app' % sys.executable
print("%s\n" % cmd)
pipe = Popen(cmd, shell=True, stdin=PIPE, close_fds=False)
waitpid(pipe.pid, 0)
# Create the dmg image.
print("\n\n# Creating the DMG image.\n\n")
cmd = 'hdiutil create -ov -fs HFS+ -volname "relax" -srcfolder dist/relax.app ../%s' % file
print("%s\n" % cmd)
pipe = Popen(cmd, shell=True, stdin=PIPE, close_fds=False)
waitpid(pipe.pid, 0)
# Stop executing.
return
# Open the Zip distribution file.
if dist_type == 'zip':
archive = ZipFile(path.pardir + path.sep + file,
'w',
compression=8)
# Open the Tar distribution file.
elif dist_type == 'tar':
if search('.bz2$', file):
archive = TarFile.bz2open(path.pardir + path.sep + file, 'w')
elif search('.gz$', file):
archive = TarFile.gzopen(path.pardir + path.sep + file, 'w')
else:
archive = TarFile.open(path.pardir + path.sep + file, 'w')
# Base directory.
base = getcwd() + sep
# Find all files untracked by the VC and not ignored.
untracked = []
if path.isdir(getcwd() + path.sep + ".git"):
cmd = "git ls-files --others --exclude-standard"
pipe = Popen(cmd, shell=True, stdout=PIPE, close_fds=False)
for file_name in pipe.stdout.readlines():
untracked.append(file_name.strip())
# Walk through the directories.
for root, dirs, files in walk(getcwd()):
# Skip the version control directories.
skip = False
for vc in VERSION_CONTROL_DIRS:
if search(vc, root):
skip = True
if skip:
continue
# Add the files in the current directory to the archive.
for i in range(len(files)):
# Skip all blacklisted files.
skip = False
for file_name in BLACKLISTED_FILES:
if search(file_name, files[i]):
skip = True
# Create the file name (without the base directory).
name = path.join(root, files[i])
name = name[len(base):]
# Skip all untracked files.
if name in untracked:
skip = True
# Nothing to do.
if skip:
continue
# The archive file name.
arcname = 'relax-' + version + path.sep + name
print(arcname)
# Zip archives.
if dist_type == 'zip':
archive.write(filename=name, arcname=arcname)
# Tar archives.
if dist_type == 'tar':
archive.add(name=name, arcname=arcname)
# Close the archive.
archive.close()
# Final printout.
print("\n\n\n")
def from_bytes(cls, data):
tar = TarFile.gzopen(name='delta.tar.gz', fileobj=BytesIO(data))
return cls(tar)
