def _download_kickstarts(self, project, configurations, ignore):
"""Downloads RPM and extrack .ks files."""
kickstarts = []
errors = []
rpms = set()
with Lab(prefix="get_kickstarts") as lab:
# Download binaries
for package in configurations:
for target in configurations[package]:
for binary in configurations[package][target]:
rpms.add(
self.download_binary(project, package, target,
binary, lab.path))
for rpm in rpms:
# Extract kickstart files
found = False
for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
# Read ks contents in images array
basedir, basename = os.path.split(fname)
if [True for pattern in ignore if pattern.match(basename)]:
continue
kickstarts.append({
"basedir": basedir,
"basename": basename,
"contents": lab.open(fname).read()
})
found = True
if not found:
errors.append("%s did not contain .ks files" %
os.path.basename(rpm))
return kickstarts, errors
def _download_kickstarts(self, project, configurations):
"""Downloads RPM and extrack .ks files."""
rpms = set()
with Lab(prefix="get_kickstarts") as lab:
# Download binaries
for package in configurations:
for target in configurations[package]:
for binary in configurations[package][target]:
rpms.add(self.download_binary(project, package,
target, binary, lab.path))
for rpm in rpms:
# deploy dir is the name of the rpm without the versions
deploy_dir = os.path.join(self.deploy_prefix,
".%s" % os.path.basename(rpm).rsplit("-", 2)[0])
shutil.rmtree(deploy_dir, ignore_errors=True)
os.mkdir(deploy_dir)
# Extract kickstart files and copy to the deploy dir
for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
shutil.copy(os.path.join(lab.path, fname), deploy_dir)
symlink_src = os.path.join(deploy_dir, os.path.basename(fname))
symlink_dst = os.path.join(self.deploy_prefix, os.path.basename(fname))
if os.path.lexists(symlink_dst):
os.unlink(symlink_dst)
os.symlink(symlink_src, symlink_dst)
return
def _download_kickstarts(self, project, configurations):
"""Downloads RPM and extrack .ks files."""
rpms = set()
with Lab(prefix="get_kickstarts") as lab:
# Download binaries
if isinstance(project, unicode):
project = project.encode('utf8')
for package in configurations:
if isinstance(package, unicode):
package = package.encode('utf8')
for target in configurations[package]:
if isinstance(target, unicode):
target = target.encode('utf8')
for binary in configurations[package][target]:
if isinstance(binary, unicode):
binary = binary.encode('utf8')
rpms.add(self.download_binary(project, package,
target, binary, lab.path))
for rpm in rpms:
# deploy dir is the name of the rpm without the versions
deploy_dir = os.path.join(self.deploy_prefix,
".%s" % os.path.basename(rpm).rsplit("-", 2)[0])
# Both release and devel ks share the same directory
if not os.path.exists(deploy_dir):
def __update_meta(self, project, providers, metatype):
"""Extracts a meta xml from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
for package, targets in providers.items():
for target, binaries in targets.items():
for binary in binaries:
with Lab(prefix=metatype) as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package,
binary, lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" %
(binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path,
["*.xml"]):
meta = os.path.basename(xml)
try:
with open(lab.real_path(xml), 'r') as fd:
metadata = [
line.replace("@PROJECT@", project)
for line in fd.readlines()
]
# Update meta
core.edit_meta(metatype,
project,
data=metadata)
uploaded.append(meta)
except HTTPError as exc:
errors.append(
"Failed to upload %s:\nHTTP %s %s\n%s" %
(meta, exc.code, exc.filename,
exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" %
(meta, exc))
return uploaded, errors
def __update_patterns(self, project, package, target, binary):
"""Extracts patterns from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
with Lab(prefix="update_patterns") as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package, binary,
lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" % (binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
print lab.real_path(binary)
for xml in extract_rpm(lab.real_path(binary), lab.path, ["*.xml"]):
pattern = os.path.basename(xml)
try:
# chop .xml from name
if pattern.endswith(".xml"):
pattern = pattern[:-4]
# Update pattern to project
self.obs.setProjectPattern(project,
lab.real_path(xml),
name=pattern)
uploaded.append(pattern)
except HTTPError as exc:
errors.append(
"Failed to upload %s:\nHTTP %s %s\n%s" %
(pattern, exc.code, exc.filename, exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" % (pattern, exc))
return uploaded, errors
def __update_meta(self, project, providers, metatype):
"""Extracts a meta xml from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
for package, targets in providers.items():
for target, binaries in targets.items():
for binary in binaries:
with Lab(prefix=metatype) as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package,
binary, lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" %
(binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path,
["*.xml"]):
meta = os.path.basename(xml)
submetatype = os.path.basename(
os.path.dirname(xml))
print(meta, metatype, submetatype)
try:
with open(lab.real_path(xml), 'r') as fd:
metadata = [
line.replace("@PROJECT@", project)
for line in fd.readlines()
]
# Update meta
if submetatype == "aggregates":
pkgname = os.path.splitext(meta)[0]
core.edit_meta(metatype='pkg',
path_args=(project,
pkgname),
template_args=({
'name':
pkgname,
'user':
'******'
}),
apiurl=self.obs.apiurl)
u = core.makeurl(self.obs.apiurl, [
'source', project, pkgname,
'_aggregate'
])
print u
print metadata
core.http_PUT(u, data="\n".join(metadata))
else:
core.edit_meta(metatype,
project,
data=metadata)
uploaded.append(metatype + '/' + meta)
except HTTPError as exc:
errors.append(
"Failed to upload %s:\nHTTP %s %s\n%s" %
(meta, exc.code, exc.filename,
exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" %
(meta, exc))
return uploaded, errors
def setUp(self):
self.lab = Lab()
class TestLab(unittest.TestCase):
def setUp(self):
self.lab = Lab()
def tearDown(self):
self.lab.cleanup()
def test_snapshot_and_cleanup(self):
snap = self.lab.take_snapshot()
self.assertEqual(len(self.lab._history), 2)
for path in self.lab._history:
self.assertTrue(os.path.exists(path))
self.lab.real_path("test", snap)
self.assertRaises(ValueError, self.lab.real_path, "test", 5)
def test_store(self):
self.lab.store("test", "testing")
self.assertTrue(os.path.exists(self.lab.real_path("test")))
self.assertEqual(self.lab.open("test").read(), "testing")
def test_get_diff(self):
self.lab.store("test", "testing")
self.assertRaises(ValueError, self.lab.get_diff, "test", 5)
self.assertRaises(ValueError, self.lab.get_diff, "test", 0, 3)
self.assertEquals(len(self.lab.get_diff("test", 0, 0)), 0)
snap = self.lab.take_snapshot()
self.assertEquals(len(self.lab.get_diff("test", snap)), 0)
self.lab.open("test", "w").write("newstuff")
self.assertEquals(len(self.lab.get_diff("test", snap)), 2)
self.lab.store("newfile", "newcontent")
self.assertEquals(len(self.lab.get_diff("newfile", snap)), 1)
os.remove(self.lab.real_path("test"))
self.assertEquals(len(self.lab.get_diff("test", snap)), 1)
def test_dir_creation(self):
self.assertFalse(os.path.isdir(self.lab.real_path("test")))
self.lab.mkdir("test")
self.assertTrue(os.path.isdir(self.lab.real_path("test")))
self.assertFalse(os.path.isdir(self.lab.real_path("foo/bar")))
self.lab.makedirs("foo/bar")
self.assertTrue(os.path.isdir(self.lab.real_path("foo/bar")))
def test_open(self):
self.lab.open(name="test", mode="w").write("testing")
self.assertEqual(self.lab.open("test").read(), "testing")
sid = self.lab.take_snapshot()
self.assertEqual(self.lab.open("test", sid=sid).read(), "testing")
self.assertRaises(ValueError, self.lab.open, "foo", "w", sid=sid)
self.assertRaises(ValueError, self.lab.open, "test", "a", sid=sid)
def test_bad_path(self):
self.assertRaises(ValueError, self.lab.open, "../foo", "w")
self.assertRaises(ValueError, self.lab.open, "/../foo", "w")
self.assertRaises(ValueError, self.lab.mkdir, "../foo")
self.assertRaises(ValueError, self.lab.makedirs, "foo/../bar")
def test_context(self):
class MyException(Exception):
pass
path = None
try:
with self.lab:
path = self.lab.path
raise MyException()
except MyException:
self.assertFalse(os.path.exists(path))
def setUp(self):
self.lab = Lab()
class TestLab(unittest.TestCase):
def setUp(self):
self.lab = Lab()
def tearDown(self):
self.lab.cleanup()
def test_snapshot_and_cleanup(self):
snap = self.lab.take_snapshot()
self.assertEqual(len(self.lab._history), 2)
for path in self.lab._history:
self.assertTrue(os.path.exists(path))
self.lab.real_path("test", snap)
self.assertRaises(ValueError, self.lab.real_path, "test", 5)
def test_store(self):
self.lab.store("test", "testing")
self.assertTrue(os.path.exists(self.lab.real_path("test")))
self.assertEqual(self.lab.open("test").read(), "testing")
def test_get_diff(self):
self.lab.store("test", "testing")
self.assertRaises(ValueError, self.lab.get_diff, "test", 5)
self.assertRaises(ValueError, self.lab.get_diff, "test", 0, 3)
self.assertEquals(len(self.lab.get_diff("test", 0, 0)), 0)
snap = self.lab.take_snapshot()
self.assertEquals(len(self.lab.get_diff("test", snap)), 0)
self.lab.open("test", "w").write("newstuff")
self.assertEquals(len(self.lab.get_diff("test", snap)), 2)
self.lab.store("newfile", "newcontent")
self.assertEquals(len(self.lab.get_diff("newfile", snap)), 1)
os.remove(self.lab.real_path("test"))
self.assertEquals(len(self.lab.get_diff("test", snap)), 1)
def test_dir_creation(self):
self.assertFalse(os.path.isdir(self.lab.real_path("test")))
self.lab.mkdir("test")
self.assertTrue(os.path.isdir(self.lab.real_path("test")))
self.assertFalse(os.path.isdir(self.lab.real_path("foo/bar")))
self.lab.makedirs("foo/bar")
self.assertTrue(os.path.isdir(self.lab.real_path("foo/bar")))
def test_open(self):
self.lab.open(name="test", mode="w").write("testing")
self.assertEqual(self.lab.open("test").read(), "testing")
sid = self.lab.take_snapshot()
self.assertEqual(self.lab.open("test", sid=sid).read(), "testing")
self.assertRaises(ValueError, self.lab.open, "foo", "w", sid=sid)
self.assertRaises(ValueError, self.lab.open, "test", "a", sid=sid)
def test_bad_path(self):
self.assertRaises(ValueError, self.lab.open, "../foo", "w")
self.assertRaises(ValueError, self.lab.open, "/../foo", "w")
self.assertRaises(ValueError, self.lab.mkdir, "../foo")
self.assertRaises(ValueError, self.lab.makedirs, "foo/../bar")
def test_context(self):
class MyException(Exception):
pass
path = None
try:
with self.lab:
path = self.lab.path
raise MyException()
except MyException:
self.assertFalse(os.path.exists(path))
def __handle_action(self, action, _wid):
"""Process single action from OBS event info.
:param action: Single dictionary from OBS event actions list
:returns: True if all good, False otherwise
"""
project = action["sourceproject"]
package = action["sourcepackage"]
revision = action["sourcerevision"]
files = self.obs.getPackageFileList(project, package, revision)
with Lab(prefix="check_yaml_spec_") as lab:
spec = None
yaml = None
for name in files:
if name.endswith(".spec"):
lab.store(name, self.obs.getFile(project, package, name,
revision))
spec = name
elif name.endswith(".yaml"):
lab.store(name, self.obs.getFile(project, package, name,
revision))
yaml = name
if not (spec and self.spec_re.search(lab.open(spec).read())):
# No spec file or spec not from spectacle, skip
return True, None
if not yaml:
return False, "SPEC file generated with spectacle, " \
"but yaml not present"
snapshot = lab.take_snapshot()
# Download rest of the files
files.remove(spec)
files.remove(yaml)
for name in files:
lab.store(name, self.obs.getFile(project, package, name,
revision))
# Run specify
specify = subprocess.Popen(["specify", "-n", "-N",
lab.real_path(yaml)], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env={"ANSI_COLORS_DISABLED":"1"})
rcode = specify.wait()
if rcode != 0:
return False, "Running specify failed:\n%s" \
% specify.stdout.read()
# Get the diff
diff = lab.get_diff(spec, snapshot)
clean_diff = []
for line in diff:
# ignore the ? seperator lines
if line[0] == "?":
continue
# Remove diff markers and white space
stripped = line[2:].strip()
# skip empty lines
if not stripped:
continue
# skip comments
if stripped[0] == "#":
continue
# effective change
clean_diff.append(line)
if clean_diff:
return False, "Spec file changed by specify:\n%s" \
% "".join(clean_diff)
return True, None