def _download_kickstarts(self, project, configurations, ignore):
        """Downloads RPM and extrack .ks files."""
        kickstarts = []
        errors = []
        rpms = set()
        with Lab(prefix="get_kickstarts") as lab:
            # Download binaries
            for package in configurations:
                for target in configurations[package]:
                    for binary in configurations[package][target]:
                        rpms.add(
                            self.download_binary(project, package, target,
                                                 binary, lab.path))

            for rpm in rpms:
                # Extract kickstart files
                found = False
                for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
                    # Read ks contents in images array
                    basedir, basename = os.path.split(fname)
                    if [True for pattern in ignore if pattern.match(basename)]:
                        continue
                    kickstarts.append({
                        "basedir": basedir,
                        "basename": basename,
                        "contents": lab.open(fname).read()
                    })
                    found = True
                if not found:
                    errors.append("%s did not contain .ks files" %
                                  os.path.basename(rpm))

        return kickstarts, errors
    def _download_kickstarts(self, project, configurations):
        """Downloads RPM and extrack .ks files."""
        rpms = set()
        with Lab(prefix="get_kickstarts") as lab:
            # Download binaries
            for package in configurations:
                for target in configurations[package]:
                    for binary in configurations[package][target]:
                        rpms.add(self.download_binary(project, package,
                                target, binary, lab.path))

            for rpm in rpms:
                # deploy dir is the name of the rpm without the versions
                deploy_dir = os.path.join(self.deploy_prefix,
                             ".%s" % os.path.basename(rpm).rsplit("-", 2)[0])
                shutil.rmtree(deploy_dir, ignore_errors=True)
                os.mkdir(deploy_dir)
                # Extract kickstart files and copy to the deploy dir
                for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
                    shutil.copy(os.path.join(lab.path, fname), deploy_dir)
                    symlink_src = os.path.join(deploy_dir, os.path.basename(fname))
                    symlink_dst = os.path.join(self.deploy_prefix, os.path.basename(fname))
                    if os.path.lexists(symlink_dst):
                        os.unlink(symlink_dst)
                    os.symlink(symlink_src, symlink_dst)

        return
Пример #3
0
    def _download_kickstarts(self, project, configurations):
        """Downloads RPM and extrack .ks files."""
        rpms = set()
        with Lab(prefix="get_kickstarts") as lab:
            # Download binaries
            if isinstance(project, unicode):
                project = project.encode('utf8')
            for package in configurations:
                if isinstance(package, unicode):
                    package = package.encode('utf8')
                for target in configurations[package]:
                    if isinstance(target, unicode):
                        target = target.encode('utf8')
                    for binary in configurations[package][target]:
                        if isinstance(binary, unicode):
                            binary = binary.encode('utf8')
                        rpms.add(self.download_binary(project, package,
                                target, binary, lab.path))

            for rpm in rpms:
                # deploy dir is the name of the rpm without the versions
                deploy_dir = os.path.join(self.deploy_prefix,
                             ".%s" % os.path.basename(rpm).rsplit("-", 2)[0])
                # Both release and devel ks share the same directory
                if not os.path.exists(deploy_dir):
Пример #4
0
    def __update_meta(self, project, providers, metatype):
        """Extracts a meta xml from rpm and uploads them to project.

        :returns: uploaded pattern names and error messages
        :rtype: tuple(list, list)
        """
        uploaded = []
        errors = []
        for package, targets in providers.items():
            for target, binaries in targets.items():
                for binary in binaries:
                    with Lab(prefix=metatype) as lab:
                        # Download the rpm
                        try:
                            self.obs.getBinary(project, target, package,
                                               binary, lab.real_path(binary))
                        except HTTPError as exc:
                            errors.append("Failed to download %s: HTTP %s %s" %
                                          (binary, exc.code, exc.filename))
                        except Exception as exc:
                            errors.append("Failed to download %s: %s" %
                                          (binary, exc))
                        if errors:
                            return uploaded, errors
                        # Extract pattern (xml) files from the rpm
                        for xml in extract_rpm(lab.real_path(binary), lab.path,
                                               ["*.xml"]):
                            meta = os.path.basename(xml)
                            try:
                                with open(lab.real_path(xml), 'r') as fd:
                                    metadata = [
                                        line.replace("@PROJECT@", project)
                                        for line in fd.readlines()
                                    ]
                                # Update meta
                                core.edit_meta(metatype,
                                               project,
                                               data=metadata)
                                uploaded.append(meta)
                            except HTTPError as exc:
                                errors.append(
                                    "Failed to upload %s:\nHTTP %s %s\n%s" %
                                    (meta, exc.code, exc.filename,
                                     exc.fp.read()))
                            except Exception as exc:
                                errors.append("Failed to upload %s: %s" %
                                              (meta, exc))
                    return uploaded, errors
    def __update_patterns(self, project, package, target, binary):
        """Extracts patterns from rpm and uploads them to project.

        :returns: uploaded pattern names and error messages
        :rtype: tuple(list, list)
        """
        uploaded = []
        errors = []
        with Lab(prefix="update_patterns") as lab:
            # Download the rpm
            try:
                self.obs.getBinary(project, target, package, binary,
                                   lab.real_path(binary))
            except HTTPError as exc:
                errors.append("Failed to download %s: HTTP %s %s" %
                              (binary, exc.code, exc.filename))
            except Exception as exc:
                errors.append("Failed to download %s: %s" % (binary, exc))
            if errors:
                return uploaded, errors
            # Extract pattern (xml) files from the rpm
            print lab.real_path(binary)
            for xml in extract_rpm(lab.real_path(binary), lab.path, ["*.xml"]):
                pattern = os.path.basename(xml)
                try:
                    # chop .xml from name
                    if pattern.endswith(".xml"):
                        pattern = pattern[:-4]

                    # Update pattern to project
                    self.obs.setProjectPattern(project,
                                               lab.real_path(xml),
                                               name=pattern)
                    uploaded.append(pattern)
                except HTTPError as exc:
                    errors.append(
                        "Failed to upload %s:\nHTTP %s %s\n%s" %
                        (pattern, exc.code, exc.filename, exc.fp.read()))
                except Exception as exc:
                    errors.append("Failed to upload %s: %s" % (pattern, exc))
        return uploaded, errors
Пример #6
0
    def __update_meta(self, project, providers, metatype):
        """Extracts a meta xml from rpm and uploads them to project.

        :returns: uploaded pattern names and error messages
        :rtype: tuple(list, list)
        """
        uploaded = []
        errors = []
        for package, targets in providers.items():
            for target, binaries in targets.items():
                for binary in binaries:
                    with Lab(prefix=metatype) as lab:
                        # Download the rpm
                        try:
                            self.obs.getBinary(project, target, package,
                                               binary, lab.real_path(binary))
                        except HTTPError as exc:
                            errors.append("Failed to download %s: HTTP %s %s" %
                                          (binary, exc.code, exc.filename))
                        except Exception as exc:
                            errors.append("Failed to download %s: %s" %
                                          (binary, exc))
                        if errors:
                            return uploaded, errors
                        # Extract pattern (xml) files from the rpm
                        for xml in extract_rpm(lab.real_path(binary), lab.path,
                                               ["*.xml"]):
                            meta = os.path.basename(xml)
                            submetatype = os.path.basename(
                                os.path.dirname(xml))
                            print(meta, metatype, submetatype)
                            try:
                                with open(lab.real_path(xml), 'r') as fd:
                                    metadata = [
                                        line.replace("@PROJECT@", project)
                                        for line in fd.readlines()
                                    ]
                                # Update meta
                                if submetatype == "aggregates":
                                    pkgname = os.path.splitext(meta)[0]
                                    core.edit_meta(metatype='pkg',
                                                   path_args=(project,
                                                              pkgname),
                                                   template_args=({
                                                       'name':
                                                       pkgname,
                                                       'user':
                                                       '******'
                                                   }),
                                                   apiurl=self.obs.apiurl)
                                    u = core.makeurl(self.obs.apiurl, [
                                        'source', project, pkgname,
                                        '_aggregate'
                                    ])
                                    print u
                                    print metadata
                                    core.http_PUT(u, data="\n".join(metadata))
                                else:
                                    core.edit_meta(metatype,
                                                   project,
                                                   data=metadata)
                                uploaded.append(metatype + '/' + meta)
                            except HTTPError as exc:
                                errors.append(
                                    "Failed to upload %s:\nHTTP %s %s\n%s" %
                                    (meta, exc.code, exc.filename,
                                     exc.fp.read()))
                            except Exception as exc:
                                errors.append("Failed to upload %s: %s" %
                                              (meta, exc))
                    return uploaded, errors
 def setUp(self):
     self.lab = Lab()
class TestLab(unittest.TestCase):
    def setUp(self):
        self.lab = Lab()

    def tearDown(self):
        self.lab.cleanup()

    def test_snapshot_and_cleanup(self):
        snap = self.lab.take_snapshot()
        self.assertEqual(len(self.lab._history), 2)
        for path in self.lab._history:
            self.assertTrue(os.path.exists(path))
        self.lab.real_path("test", snap)
        self.assertRaises(ValueError, self.lab.real_path, "test", 5)

    def test_store(self):
        self.lab.store("test", "testing")
        self.assertTrue(os.path.exists(self.lab.real_path("test")))
        self.assertEqual(self.lab.open("test").read(), "testing")

    def test_get_diff(self):
        self.lab.store("test", "testing")
        self.assertRaises(ValueError, self.lab.get_diff, "test", 5)
        self.assertRaises(ValueError, self.lab.get_diff, "test", 0, 3)
        self.assertEquals(len(self.lab.get_diff("test", 0, 0)), 0)
        snap = self.lab.take_snapshot()
        self.assertEquals(len(self.lab.get_diff("test", snap)), 0)

        self.lab.open("test", "w").write("newstuff")
        self.assertEquals(len(self.lab.get_diff("test", snap)), 2)

        self.lab.store("newfile", "newcontent")
        self.assertEquals(len(self.lab.get_diff("newfile", snap)), 1)

        os.remove(self.lab.real_path("test"))
        self.assertEquals(len(self.lab.get_diff("test", snap)), 1)

    def test_dir_creation(self):
        self.assertFalse(os.path.isdir(self.lab.real_path("test")))
        self.lab.mkdir("test")
        self.assertTrue(os.path.isdir(self.lab.real_path("test")))

        self.assertFalse(os.path.isdir(self.lab.real_path("foo/bar")))
        self.lab.makedirs("foo/bar")
        self.assertTrue(os.path.isdir(self.lab.real_path("foo/bar")))

    def test_open(self):
        self.lab.open(name="test", mode="w").write("testing")
        self.assertEqual(self.lab.open("test").read(), "testing")

        sid = self.lab.take_snapshot()
        self.assertEqual(self.lab.open("test", sid=sid).read(), "testing")
        self.assertRaises(ValueError, self.lab.open, "foo", "w", sid=sid)
        self.assertRaises(ValueError, self.lab.open, "test", "a", sid=sid)

    def test_bad_path(self):
        self.assertRaises(ValueError, self.lab.open, "../foo", "w")
        self.assertRaises(ValueError, self.lab.open, "/../foo", "w")
        self.assertRaises(ValueError, self.lab.mkdir, "../foo")
        self.assertRaises(ValueError, self.lab.makedirs, "foo/../bar")

    def test_context(self):
        class MyException(Exception):
            pass

        path = None
        try:
            with self.lab:
                path = self.lab.path
                raise MyException()
        except MyException:
            self.assertFalse(os.path.exists(path))
 def setUp(self):
     self.lab = Lab()
class TestLab(unittest.TestCase):
    def setUp(self):
        self.lab = Lab()

    def tearDown(self):
        self.lab.cleanup()

    def test_snapshot_and_cleanup(self):
        snap = self.lab.take_snapshot()
        self.assertEqual(len(self.lab._history), 2)
        for path in self.lab._history:
            self.assertTrue(os.path.exists(path))
        self.lab.real_path("test", snap)
        self.assertRaises(ValueError, self.lab.real_path, "test", 5)

    def test_store(self):
        self.lab.store("test", "testing")
        self.assertTrue(os.path.exists(self.lab.real_path("test")))
        self.assertEqual(self.lab.open("test").read(), "testing")

    def test_get_diff(self):
        self.lab.store("test", "testing")
        self.assertRaises(ValueError, self.lab.get_diff, "test", 5)
        self.assertRaises(ValueError, self.lab.get_diff, "test", 0, 3)
        self.assertEquals(len(self.lab.get_diff("test", 0, 0)), 0)
        snap = self.lab.take_snapshot()
        self.assertEquals(len(self.lab.get_diff("test", snap)), 0)

        self.lab.open("test", "w").write("newstuff")
        self.assertEquals(len(self.lab.get_diff("test", snap)), 2)

        self.lab.store("newfile", "newcontent")
        self.assertEquals(len(self.lab.get_diff("newfile", snap)), 1)

        os.remove(self.lab.real_path("test"))
        self.assertEquals(len(self.lab.get_diff("test", snap)), 1)

    def test_dir_creation(self):
        self.assertFalse(os.path.isdir(self.lab.real_path("test")))
        self.lab.mkdir("test")
        self.assertTrue(os.path.isdir(self.lab.real_path("test")))

        self.assertFalse(os.path.isdir(self.lab.real_path("foo/bar")))
        self.lab.makedirs("foo/bar")
        self.assertTrue(os.path.isdir(self.lab.real_path("foo/bar")))

    def test_open(self):
        self.lab.open(name="test", mode="w").write("testing")
        self.assertEqual(self.lab.open("test").read(), "testing")

        sid = self.lab.take_snapshot()
        self.assertEqual(self.lab.open("test", sid=sid).read(), "testing")
        self.assertRaises(ValueError, self.lab.open, "foo", "w", sid=sid)
        self.assertRaises(ValueError, self.lab.open, "test", "a", sid=sid)

    def test_bad_path(self):
        self.assertRaises(ValueError, self.lab.open, "../foo", "w")
        self.assertRaises(ValueError, self.lab.open, "/../foo", "w")
        self.assertRaises(ValueError, self.lab.mkdir, "../foo")
        self.assertRaises(ValueError, self.lab.makedirs, "foo/../bar")


    def test_context(self):
        class MyException(Exception):
            pass
        path = None
        try:
            with self.lab:
                path = self.lab.path
                raise MyException()
        except MyException:
            self.assertFalse(os.path.exists(path))
    def __handle_action(self, action, _wid):
        """Process single action from OBS event info.

        :param action: Single dictionary from OBS event actions list
        :returns: True if all good, False otherwise
        """
        project = action["sourceproject"]
        package = action["sourcepackage"]
        revision = action["sourcerevision"]
        files = self.obs.getPackageFileList(project, package, revision)

        with Lab(prefix="check_yaml_spec_") as lab:
            spec = None
            yaml = None
            for name in files:
                if name.endswith(".spec"):
                    lab.store(name, self.obs.getFile(project, package, name,
                        revision))
                    spec = name
                elif name.endswith(".yaml"):
                    lab.store(name, self.obs.getFile(project, package, name,
                        revision))
                    yaml = name

            if not (spec and self.spec_re.search(lab.open(spec).read())):
                # No spec file or spec not from spectacle, skip
                return True, None
            if not yaml:
                return False, "SPEC file generated with spectacle, " \
                              "but yaml not present"

            snapshot = lab.take_snapshot()
            # Download rest of the files
            files.remove(spec)
            files.remove(yaml)
            for name in files:
                lab.store(name, self.obs.getFile(project, package, name,
                        revision))

            # Run specify
            specify = subprocess.Popen(["specify", "-n", "-N",
                lab.real_path(yaml)], stdout=subprocess.PIPE,
                stderr=subprocess.STDOUT, env={"ANSI_COLORS_DISABLED":"1"})
            rcode = specify.wait()
            if rcode != 0:
                return False, "Running specify failed:\n%s" \
                        % specify.stdout.read()
            # Get the diff
            diff = lab.get_diff(spec, snapshot)
            clean_diff = []
            for line in diff:
                # ignore the ? seperator lines
                if line[0] == "?":
                    continue
                # Remove diff markers and white space
                stripped = line[2:].strip()
                # skip empty lines
                if not stripped:
                    continue
                # skip comments
                if stripped[0] == "#":
                    continue
                # effective change
                clean_diff.append(line)
            if clean_diff:
                return False, "Spec file changed by specify:\n%s" \
                        % "".join(clean_diff)
        return True, None