def inspect(self, deb):
"""Perform a couple of check on the given .deb file."""
with TemporaryDirectory() as tmp:
with cwd(tmp):
check_call(["ar", "x", deb], stdout=DEVNULL, stderr=DEVNULL)
self.assertTrue(isfile("debian-binary"))
self.assertTrue(isfile("control.tar.gz"))
self.assertTrue(isfile("data.tar.xz"))
with open("debian-binary", "rb") as f:
self.assertEqual(f.read(), b"2.0\x0a")
with tarOpen("data.tar.xz", "r") as f:
members = f.getmembers()
self.assertEqual(members[0].name, ".")
self.assertEqual(members[1].name, "./etc")
self.assertEqual(members[2].name, "./etc/conf")
self.assertEqual(members[3].name, "./usr")
self.assertEqual(members[4].name, "./usr/file")
self.assertEqual(members[4].uname, "root")
self.assertEqual(members[4].gname, "root")
self.assertEqual(members[5].name, "./usr/dir")
self.assertEqual(members[6].name, "./usr/dir/file")
self.assertEqual(len(members), 7)
with tarOpen("control.tar.gz", "r") as f:
members = f.getmembers()
self.assertEqual(members[0].name, ".")
self.assertEqual(members[1].name, "./md5sums")
md5sums = f.extractfile(members[1])
md5sum = md5(b"testfile1").hexdigest()
self.assertIn("%s %s" % (md5sum, "usr/file"),
md5sums.read().decode())
self.assertEqual(members[2].name, "./control")
control = f.extractfile(members[2]).read().decode()
self.assertTrue(control.startswith("Package: helloworld"))
self.assertIn("Version: 0.1", control)
self.assertIn("Maintainer: None", control)
self.assertIn("Installed-Size: 1", control)
self.assertTrue(control.endswith("Description: None\n"))
def prepareDatabases(outFormat, configuration, xdgCache):
"""
Prepare the databases to be parsed.
"""
if not outFormat:
print("=> Parsing Databases")
#Create the packages dictionary
packagesDictionary = {"Manjaro": {}, "Arch": {}}
if not outFormat:
print(" => Parsing Manjaro databases")
#Generate Manjaro package list.
for repository in configuration["Repositories"]:
database = tarOpen(xdgCache + "/compare-mirrors/manjaro/" +
repository.split("-")[0] + "-" +
repository.split("-")[1] + ".db")
packageList = [
package for package in database.getnames() if "/" not in package
]
packagesDictionary["Manjaro"][repository] = packageList
if not outFormat:
print(" => Parsing Arch databases")
#Generate Arch package list
for repository in configuration["Repositories"]:
database = tarOpen(xdgCache + "/compare-mirrors/arch/" +
repository.split("-")[0] + "-" +
repository.split("-")[1] + ".db")
packageList = [
package for package in database.getnames() if "/" not in package
]
packagesDictionary["Arch"][repository] = packageList
return packagesDictionary
def ExportCalc(
arg): # python 2.7 multiprocessing cant handle argument more than one
[
geomId, geom, calcId, expDir, template, calcName, startGId, startDir,
ind, writer
] = arg
writer('Exporting Job No {} with GeomId {}'.format(ind, geomId))
baseName = "{}{}-geom{}-{}".format(calcName, calcId, geomId, ind)
exportDir = expDir + "/" + baseName
os.makedirs(exportDir)
fCalc = '{}/{}.calc_'.format(
exportDir, baseName
) # `_` at the end means job not yet done, will be removed after successful run
fXYZ = '{}/{}.xyz'.format(exportDir, baseName)
genCalcFile(calcId, geomId, calcName, baseName, startGId, fCalc)
geomObj.createXYZfile(
geom, fXYZ,
ddr=calcId != 1) #< -- geometry file is created from outside
if startGId: # copy wavefunc if start id is present
_, a, b = startDir.split(
"/"
) # StartDir -> GeomData/geom1/multi1; StartBaseName -> multi1-geom1
startBaseName = "{}-{}".format(b, a)
if os.path.isdir(
startDir): # not in zipped format, copy it to a new name
shutil.copy("{}/{}.wfu".format(startDir, startBaseName),
"{}/{}.wfu".format(exportDir, baseName))
else: # file is in tar
with tarOpen(startDir + ".tar.bz2") as tar:
tar.extract("./%s.wfu" % startBaseName,
path=exportDir) # open tar file and rename it
os.rename(exportDir + "/%s.wfu" % startBaseName,
exportDir + "/%s.wfu" % baseName)
txt = template.replace("$F$", baseName)
# generate molpro input file
fInp = '{}/{}.com'.format(exportDir, baseName)
with open(fInp, 'w') as f:
f.write(txt)
return baseName
def _makeDataTar(data_root):
"""Create a data.tar.xz containing all the data."""
data_file = "data.tar.xz"
data_path = join(data_root, data_file)
with tarOpen(data_path, "x:xz") as data:
with cwd(data_root):
for dir_, _, files in walk(curdir):
data.add(dir_, recursive=False, filter=_chownTarInfo)
for file_ in files:
file_path = join(dir_, file_)
# We need to prevent the data archive itself from being
# packaged.
if file_path == curdir + sep + data_file:
continue
data.add(file_path, filter=_chownTarInfo)
return data_path
def _makeControlTar(debian, control_files, ctrl_root):
"""Create a control.tar.gz containing all the DEBIAN files."""
path = join(ctrl_root, "control.tar.gz")
with tarOpen(path, "x:gz") as control:
with cwd(debian):
control.add(curdir, filter=_chownTarInfo)
if control_files is not None:
for src, dst in control_files:
if isabs(dst):
raise RuntimeError("Destination path (%s) must not be absolute" % dst)
# We could impose additional checks on the source/destination
# files here. E.g., using a directory as a source or placing a
# file not in the root are unlikely to make much sense. However,
# we do not want to get into too much business of enforcing
# policies. A whitelist-style approach (check for things
# allowed, deny everything else) does not seem very future proof
# while a blacklist based method is prone to missing certain
# bits.
control.add(src, arcname=join(curdir, dst), filter=_chownTarInfo)
return path
def testAdditionalControlFileSupport(self):
"""Test that we support adding of other control files properly."""
with NamedTemporaryFile("w+") as preinst, \
NamedTemporaryFile("w+") as postrm:
preinst.write(
dedent("""\
#!/bin/sh
echo preinst
"""))
postrm.write(
dedent("""\
#!/bin/sh
echo postrm
"""))
preinst.flush()
postrm.flush()
fsync(preinst)
fsync(postrm)
with TemporaryDirectory() as pkg:
debian = join(pkg, "DEBIAN")
mkdir(debian)
control_files = [
(preinst.name, "preinst"),
(postrm.name, "postrm"),
]
_makeControlTar(debian, control_files, pkg)
with cwd(pkg):
with tarOpen("control.tar.gz", "r") as f:
members = f.getmembers()
self.assertEqual(len(members), 3)
self.assertEqual(members[0].name, ".")
self.assertEqual(members[1].name, "./preinst")
self.assertEqual(members[2].name, "./postrm")
#! /usr/bin/env python
from urllib2 import urlopen
from StringIO import StringIO
from tarfile import open as tarOpen
from os.path import sep, join
from os import mkdir
DIR="/root/.vim"
try:
mkdir(DIR)
except OSError:
pass
tar = tarOpen(fileobj=StringIO(urlopen("https://github.com/plasticboy/vim-markdown/archive/master.tar.gz").read()),
mode="r:gz")
# This function imitates tar's --strip=1 option and filters out unneccessary stuff
def strip_1(members):
for x in members:
sp = x.name.split(sep, 2)
if len(sp) <= 1:
continue
n = sp[1]
if n.endswith(".md") or n in ("Makefile", "test"):
continue
x.name = apply(join, sp[1:])
yield x
tar.extractall(members=strip_1(tar), path=DIR)
#! /usr/bin/env python
from urllib2 import urlopen
from StringIO import StringIO
from tarfile import open as tarOpen
from os.path import sep, join
from os import mkdir
DIR = "/root/.vim"
try:
mkdir(DIR)
except OSError:
pass
tar = tarOpen(fileobj=StringIO(
urlopen("https://github.com/plasticboy/vim-markdown/archive/master.tar.gz"
).read()),
mode="r:gz")
# This function imitates tar's --strip=1 option and filters out unneccessary stuff
def strip_1(members):
for x in members:
sp = x.name.split(sep, 2)
if len(sp) <= 1:
continue
n = sp[1]
if n.endswith(".md") or n in ("Makefile", "test"):
continue
x.name = apply(join, sp[1:])
yield x
def unZipOne(path, extn = '.tar.bz2'):
print("Extracting %s"%path)
with tarOpen(path) as tar:
tar.extractall(path=path.replace(extn,''))
os.remove(path)