def setup_obs(self, namespace): """Setup the Buildservice instance Using namespace as an alias to the apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace)
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def check_and_mark_project(self, project, attribute): """ Checks an OBS project for the existence of attribute needs_nightly_build. Return True if the project didn't have one and create the attibute.False otherwise. """ if self.obs.projectAttributeExists(project, attribute): return False else: self.obs.createProjectAttribute(project, attribute) return True def handle_wi(self, wid): """ actual job thread """ wid.status = False wid.fields.needs_build = False self.setup_obs(wid.fields.ev.namespace) if wid.params.delete: stat = self.obs.deleteProjectAttribute(wid.fields.project, wid.params.attribute) if stat: wid.status = True else: wid.status = False else: if self.check_and_mark_project(wid.fields.project, wid.params.attribute): wid.fields.needs_build = True else: wid.fields.needs_build = False
def get_repositories(self, wid, project): if not wid.fields.ev or not wid.fields.ev.namespace: raise RuntimeError("Missing field: ev.namespace") obs = BuildService(oscrc=self.oscrc, apiurl=wid.fields.ev.namespace) try: repositories = obs.getProjectRepositories(project) except HTTPError, exobj: if exobj.code == 404: raise RuntimeError("Project %s not found in OBS" % project) raise
class ParticipantHandler(object): """Participant class as defined by the SkyNET API""" def __init__(self): self.obs = None self.oscrc = None self.validator = Validator() def handle_wi_control(self, ctrl): """Job control thread""" pass def handle_lifecycle_control(self, ctrl): """Handle messages for the participant itself, like start and stop.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance :param namespace: Alias to the OBS apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def _get_spec_file(self, prj, pkg, rev): file_list = self.obs.getPackageFileList(prj, pkg, revision=rev) specs = [ fil for fil in file_list if fil.endswith(".spec")] if len(specs) > 1: specs = [ fil for fil in specs if (fil == "%s.spec" % pkg or fil.endswith(":%s.spec" % pkg))] if not specs: return "No specfile in %s" % pkg, None elif len(specs) > 1: return "Couldn't determine which spec file to use for %s " % pkg, None print specs fil = specs[0] spec = self.obs.getFile(prj, pkg, fil, revision=rev) specob = None with NamedTemporaryFile() as specf: specf.write(spec) specf.flush() try: specob = parse_spec(specf.name) except ValueError, exobj: return "Could not parse spec in %s: %s" % (pkg, exobj), None return None, specob
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def get_changes_file(self, prj, pkg, rev=None): """ Get a package's changes file """ changelog = "" file_list = self.obs.getPackageFileList(prj, pkg, revision=rev) for fil in file_list: if fil.endswith(".changes"): changelog = self.obs.getFile(prj, pkg, fil) return changelog def handle_wi(self, wid): """ actual job thread """ wid.result = False missing = [name for name in ["project", "package"] if not getattr(wid.fields, name, None)] if missing: raise RuntimeError("Missing mandatory field(s): %s" % ", ".join(missing)) self.setup_obs(wid.fields.ev.namespace) wid.fields.changelog = self.get_changes_file( wid.fields.project, wid.fields.package) wid.result = True
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def get_changes_file(self, prj, pkg, rev=None): """ Get a package's changes file """ changelog = "" file_list = self.obs.getPackageFileList(prj, pkg, revision=rev) for fil in file_list: if fil.endswith(".changes"): changelog = self.obs.getFile(prj, pkg, fil) return changelog def handle_wi(self, wid): """ actual job thread """ wid.result = False missing = [ name for name in ["project", "package"] if not getattr(wid.fields, name, None) ] if missing: raise RuntimeError("Missing mandatory field(s): %s" % ", ".join(missing)) self.setup_obs(wid.fields.ev.namespace) wid.fields.changelog = self.get_changes_file(wid.fields.project, wid.fields.package) wid.result = True
class ParticipantHandler(object): """Participant class as defined by the SkyNET API""" def __init__(self): self.obs = None self.oscrc = None self.validator = Validator() def handle_wi_control(self, ctrl): """Job control thread""" pass def handle_lifecycle_control(self, ctrl): """Handle messages for the participant itself, like start and stop.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance :param namespace: Alias to the OBS apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def check_spec_version_match(self, version, prj, pkg, rev=None): """Check that spec version matches given version""" spec = "" file_list = self.obs.getPackageFileList(prj, pkg, revision=rev) for fil in file_list: if fil.endswith(".spec"): spec = self.obs.getFile(prj, pkg, fil, revision=rev) break if not spec: return False, "No specfile in %s" % pkg with NamedTemporaryFile() as specf: specf.write(spec) specf.flush() try: specob = parse_spec(specf.name) except ValueError, exobj: return False, "Could not parse spec in %s: %s" % (pkg, exobj) src_hdrs = [pkg for pkg in specob.packages if pkg.header.isSource()][0] spec_version = src_hdrs.header[rpm.RPMTAG_VERSION] if spec_version != version.split('-')[0]: return False, "Last changelog version %s does not match" \ " version %s in spec file." % (version, spec_version) return True, None
def handle_wi(self, wid): """Actual job thread.""" wid.result = False if not wid.fields.build_trial or not wid.fields.build_trial.project: raise RuntimeError("Missing mandatory field 'build_trial.project'") obs = BuildService(oscrc=self.oscrc, apiurl=wid.fields.ev.namespace) try: wid.result = False core.delete_project(obs.apiurl, wid.fields.build_trial.project, force=True, msg="Removed by BOSS") self.log.info("Trial area %s removed" % wid.fields.build_trial.project) del (wid.fields.build_trial.as_dict()["project"]) wid.result = True except HTTPError as err: if err.code == 403: self.log.info("Is the BOSS user (see /etc/skynet/oscrc) enabled as a"\ " maintainer in %s or its parent?" \ % wid.fields.build_trial.project) if err.code == 404: self.log.info("HTTPError 404 : The project is already gone") wid.result = True return raise err
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.reposerver = "" self.ksstore = "" self.oscrc = "" self.namespace = None self.obs = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": self.reposerver = ctrl.config.get("build_ks", "reposerver") self.ksstore = ctrl.config.get("build_ks", "ksstore") if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def get_repositories(self, wid, project): if not (wid.fields.ev and wid.fields.ev.namespace): raise RuntimeError("Missing field: ev.namespace") if self.obs is None or self.namespace != wid.fields.ev.namespace: self.namespace = wid.fields.ev.namespace self.obs = BuildService(oscrc=self.oscrc, apiurl=self.namespace) try: repositories = self.obs.getProjectRepositories(project) except HTTPError, exobj: if exobj.code == 404: raise RuntimeError("Project %s not found in OBS" % project) raise return repositories
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def handle_wi(self, wid): """ """ self.setup_obs(wid.fields.ev.namespace) user = Verify.assertMandatoryParameter(wid, "user") field = Verify.assertMandatoryParameter(wid, "field") user_realname, user_email = self.obs.getUserData(user, "realname", "email") wid.set_field(field + ".realname", user_realname) wid.set_field(field + ".email", user_email)
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def handle_wi(self, wid): """ actual job thread """ wid.result = False if not wid.fields.msg: wid.fields.msg = [] rid = wid.fields.ev.rid actions = wid.fields.ev.actions test_project = wid.fields.test_project if not rid or not actions or not test_project: raise RuntimeError("Missing one of the mandatory fields 'ev.rid', " "'ev.actions' or 'test_project'") self.setup_obs(wid.fields.ev.namespace) in_testing = [] message = "" for action in actions: if action['type'] != "submit": continue # Check if packages are already in testing if not self.obs.hasChanges(action['sourceproject'], action['sourcepackage'], action['sourcerevision'], test_project, action['targetpackage']): in_testing.append(action['sourcepackage']) if not in_testing: message = "Request %s packages not already under testing." % rid wid.result = True else: message = "Request %s packages %s are already under testing in \ %s" % (rid, " ".join(in_testing), test_project) wid.fields.status = "FAILED" wid.fields.msg.append(message)
def __get_obs(self): """Lazy BuildService property.""" if self.__obs is None: if self.__oscrc is None or self.__obs_alias is None: raise RuntimeError("BuildService conf values not set. " "Use get_oscrc and setup_obs decorators.") self.__obs = BuildService(oscrc=self.__oscrc, apiurl=self.__obs_alias) return self.__obs
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def handle_wi(self, wid): """ actual job thread """ wid.result = False if not wid.fields.msg: wid.fields.msg = [] if not wid.fields.ev: raise RuntimeError("Missing mandatory field 'ev'") if not wid.fields.ev.namespace: raise RuntimeError("Missing mandatory field 'ev.namespace'") if not wid.fields.ev.actions: raise RuntimeError("Missing mandatory field 'ev.actions'") self.setup_obs(wid.fields.ev.namespace) all_ok = True for action in wid.fields.ev.actions: if action['type'] != 'submit': continue if not self.obs.hasChanges(action['sourceproject'], action['sourcepackage'], action['sourcerevision'], action['targetproject'], action['targetpackage']): wid.fields.msg.append( "Package %(sourceproject)s %(sourcepackage)s" " does not introduce any changes compared to" " %(targetproject)s %(targetpackage)s" % action) all_ok = False wid.result = all_ok
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def get_changes_file(self, prj, pkg, rev=None): """ Get a package's changes file """ changelog = "" try: file_list = self.obs.getPackageFileList(prj, pkg, revision=rev) for fil in file_list: if fil.endswith(".changes"): changelog = self.obs.getFile(prj, pkg, fil, revision=rev) except HTTPError, e: if e.code == 404: pass return changelog
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def handle_wi(self, wid): """ actual job thread """ wid.result = False if not wid.fields.msg: wid.fields.msg = [] if not wid.fields.ev or not wid.fields.ev.actions: raise RuntimeError("Missing mandatory field 'ev.actions'") self.setup_obs(wid.fields.ev.namespace) for action in wid.fields.ev.actions: if not self.obs.isMaintainer(action["sourceproject"], wid.fields.ev.who): wid.fields.status = "FAILED" wid.fields.msg.append("%s who submitted request %s "\ "from project %s is not allowed to do "\ "so." % (wid.fields.ev.who, wid.fields.ev.rid, action["sourceproject"])) return wid.result = True
def handle_wi(self, wid): """Actual job thread.""" if not wid.fields.build_trial or not wid.fields.build_trial.project: raise RuntimeError("Missing mandatory field 'build_trial.project'") self.obs = BuildService(oscrc=self.oscrc, apiurl=wid.fields.ev.namespace) for prj in wid.fields.build_trial.as_dict().get("subprojects", []): if prj == wid.fields.build_trial.project: continue self._delete_project(prj) self._delete_project(wid.fields.build_trial.project)
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def handle_wi(self, wid): """ actual job thread """ wid.result = False if not wid.fields.msg: wid.fields.msg = [] if not wid.fields.mail_to: wid.fields.mail_to = [] if not wid.fields.ev or not wid.fields.ev.who: raise RuntimeError("Missing mandatory field 'ev.who'") who = wid.fields.ev.who self.setup_obs(wid.fields.ev.namespace) user_email = self.obs.getUserEmail(who) if user_email: wid.fields.mail_to.append(user_email) wid.result = True else: wid.fields.msg.append("User %s doesn't have an email" % who)
apiurl = os.environ.get('OBSAPI') oscrc = os.environ.get('OSCRC', home+"/.config/osc/oscrc") testprj = os.environ.get('OBSTESTPRJ') testprj2 = os.environ.get('OBSTESTPRJ2') testpkg = os.environ.get('OBSTESTPKG') testfile = os.environ.get('OBSTESTFILE') if not apiurl: raise ValueError('You must have "OBSAPI" variable') if not oscrc: raise ValueError('You must have "OSCRC" variable') if not testprj: raise ValueError('You must have "OBSTESTPRJ" variable') if not testprj2: raise ValueError('You must have "OBSTESTPRJ2" variable') if not testpkg: raise ValueError('You must have "OBSTESTPKG" variable') if not testfile: raise ValueError('You must have "OBSTESTFILE" variable') bs = BuildService(apiurl,oscrc) print "**********************************" print "Our test settings:" print "API: "+apiurl print "OSCRC: "+oscrc print "Test project 1: "+testprj print "Test project 2: "+testprj2 print "Test package: "+testpkg print "Test file: "+testfile print "**********************************"
class ParticipantHandler(object): """Participant class as defined by the SkyNET API.""" def __init__(self): self.oscrc = None self.obs = None def handle_wi_control(self, ctrl): """Job control thread.""" pass def handle_lifecycle_control(self, ctrl): """Participant control thread.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance Using namespace as an alias to the apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def build_results(self, wid): """Main function to get new failures related to a build trial.""" wid.result = False if not wid.fields.msg: wid.fields.msg = [] if wid.params.project: prj = wid.params.project else: prj = wid.fields.project if wid.params.packages: pkgs = wid.params.package else: pkgs = wid.fields.packages or [] if wid.params.exclude_repos: exclude_repos = wid.params.exclude_repos else: exclude_repos = wid.fields.exclude_repos or [] if wid.params.exclude_archs: exclude_archs = wid.params.exclude_archs else: exclude_archs = wid.fields.exclude_archs or [] failures = set() for repo in self.obs.getProjectRepositories(prj): if repo in exclude_repos: continue archs = self.obs.getRepositoryArchs(prj, repo) archs = [arch for arch in archs if arch not in exclude_archs] # Get results results = self.obs.getRepoResults(prj, repo) failures.update(get_failures(results, archs)) #filter results if pkgs: failures = failures & set(pkgs) if len(failures): wid.fields.msg.append("%s failed to"\ " build in %s" % (" ".join(failures), prj)) wid.fields.failures = list(failures) def handle_wi(self, wid): """Actual job thread.""" self.setup_obs(wid.fields.ev.namespace) self.build_results(wid)
def _setup_obs(self, namespace): """Initialize buildservice instance.""" self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace)
class ParticipantHandler(object): """Participant class as defined by the SkyNET API.""" def __init__(self): self.oscrc = None self.namespace = None self.obs = None self.spec_re = None def handle_lifecycle_control(self, ctrl): """Participant life cycle control.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") else: raise RuntimeError("Participant config missing " "[obs] oscrc option") if ctrl.config.has_option("check_yaml", "spec_pattern"): pat = ctrl.config.get("check_yaml", "spec_pattern") else: pat = DEFAULT_SPEC_PATTERN self.spec_re = re.compile(pat) self.log.info("oscrc: %s" % self.oscrc) self.log.info("spec_pattern: %s" % pat) def handle_wi_control(self, ctrl): """Job control.""" pass def setup_obs(self, namespace): """Set up OBS instance.""" if not self.obs or self.namespace != namespace: self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) self.namespace = namespace def handle_wi(self, wid): """Job thread.""" wid.result = False if not isinstance(wid.fields.msg, list): wid.fields.msg = [] if not wid.fields.ev: raise RuntimeError("Missing mandatory field 'ev'") if not isinstance(wid.fields.ev.actions, list): raise RuntimeError("Mandatory field ev.actions not a list") if not isinstance(wid.fields.ev.namespace, basestring): raise RuntimeError("Mandatory field ev.namespace not a string") self.setup_obs(wid.fields.ev.namespace) result = True for action in wid.fields.ev.actions: pkg_result, _ = self.__handle_action(action, wid) result = result and pkg_result wid.result = result @CheckActionProcessor("check_yaml_matches_spec") def __handle_action(self, action, _wid): """Process single action from OBS event info. :param action: Single dictionary from OBS event actions list :returns: True if all good, False otherwise """ project = action["sourceproject"] package = action["sourcepackage"] revision = action["sourcerevision"] files = self.obs.getPackageFileList(project, package, revision) with Lab(prefix="check_yaml_spec_") as lab: spec = None yaml = None for name in files: if name.endswith(".spec"): lab.store(name, self.obs.getFile(project, package, name, revision)) spec = name elif name.endswith(".yaml"): lab.store(name, self.obs.getFile(project, package, name, revision)) yaml = name if not (spec and self.spec_re.search(lab.open(spec).read())): # No spec file or spec not from spectacle, skip return True, None if not yaml: return False, "SPEC file generated with spectacle, " \ "but yaml not present" snapshot = lab.take_snapshot() # Download rest of the files files.remove(spec) files.remove(yaml) for name in files: lab.store(name, self.obs.getFile(project, package, name, revision)) # Run specify specify = subprocess.Popen(["specify", "-n", "-N", lab.real_path(yaml)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env={"ANSI_COLORS_DISABLED":"1"}) rcode = specify.wait() if rcode != 0: return False, "Running specify failed:\n%s" \ % specify.stdout.read() # Get the diff diff = lab.get_diff(spec, snapshot) clean_diff = [] for line in diff: # ignore the ? seperator lines if line[0] == "?": continue # Remove diff markers and white space stripped = line[2:].strip() # skip empty lines if not stripped: continue # skip comments if stripped[0] == "#": continue # effective change clean_diff.append(line) if clean_diff: return False, "Spec file changed by specify:\n%s" \ % "".join(clean_diff) return True, None
#!/usr/bin/python from pprint import pprint from buildservice import BuildService bs = BuildService(apiurl='http://api.meego.com', oscrc='/etc/boss/oscrc') print 'devel project of Trunk:' pprint(bs.getProjectDevel('Trunk')) print 'devel package of Trunk/bash:' pprint(bs.getPackageDevel('Trunk', 'bash'))
#!/usr/bin/python from buildservice import BuildService bs = BuildService(apiurl='http://api.meego.com', oscrc='/etc/boss/oscrc' ) print bs.getRepoState('Trunk')
class ParticipantHandler(object): """Participant class as defined by the SkyNET API.""" def __init__(self): self.oscrc = None self.obs = None def handle_wi_control(self, ctrl): """Job control thread.""" pass def handle_lifecycle_control(self, ctrl): """Participant control thread.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance. Using the namespace as an alias to the apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def handle_request(self, wid): """Request handling implementation.""" wid.result = False if not wid.fields.msg : wid.fields.msg = [] rid = str(wid.fields.ev.id) # https://api.opensuse.org/apidocs/request.xsd obj_type = "request" newstate = None if wid.params.action == 'accept': newstate = "accepted" elif wid.params.action == 'reject' or wid.params.action == 'decline' : newstate = "declined" elif wid.params.action == 'add review': obj_type = "review" newstate = "add" elif wid.params.action == 'accept review': obj_type = "review" newstate = "accepted" elif wid.params.action == 'decline review': obj_type = "review" newstate = "declined" Verify.assertNotNull("Request ID field must not be empty", rid) Verify.assertNotNull("Participant action should be one of accept, "\ "decline, add review, accept review, "\ "decline review", newstate) try: if obj_type == "review": reviewer = wid.params.reviewer extra_msg = "" if wid.params.comment: extra_msg = "%s\n" % wid.params.comment if not reviewer: reviewer = self.obs.getUserName() if newstate == "add": res = self.obs.addReview(rid, extra_msg, reviewer) else: res = self.obs.setReviewState(rid, newstate, extra_msg, reviewer) elif obj_type == "request": extra_msg = "" if wid.params.comment: extra_msg = "%s\n" % wid.params.comment msgstring = "%sBOSS %s this %s because:\n %s" % ( extra_msg, newstate, obj_type, "\n ".join(wid.fields.msg) ) res = self.obs.setRequestState(rid, newstate, msgstring) if res: self.log.info("%s %s %s" % (newstate , obj_type, rid)) wid.result = True else: self.log.info("Failed to %s %s %s" % (wid.params.action , obj_type, rid)) except HTTPError, exc: if exc.code == 403: wid.fields.msg.append("Applying the actions required to "\ "automate this process has failed, "\ "because the %s user was not authorized "\ "to do so. "\ "Please add %s as a maintainer in the "\ "target projet %s" % (self.obs.getUserName(), self.obs.getUserName(), wid.fields.project)) self.log.info("Forbidden to %s %s %s" % (wid.params.action, obj_type, rid)) elif exc.code == 401: wid.fields.msg.append("Credentials for the '%s' user were "\ "refused. Please update the skynet "\ "configuration." % self.obs.getUserName()) self.log.info(exc) self.log.info("User is %s" % self.obs.getUserName()) else: import traceback self.log.info(exc) traceback.print_exc() raise
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) @CheckActionProcessor("check_package_is_complete") def is_complete(self, action, wid): """ Package file completeness check """ filelist = self.obs.getPackageFileList( action['sourceproject'], action['sourcepackage'], action['sourcerevision']) if "_service" in filelist: filelist.remove("_service") if "_ccache" in filelist: filelist.remove("_ccache") if "_src" in filelist: filelist.remove("_src") spec = self.has_spec_file(action, wid, filelist)[0] changes = self.has_changes_file(action, wid, filelist)[0] sources = spec and self.check_source_files(action, wid, filelist)[0] return (spec and changes and sources), "" def get_rpm_sources(self, action, filelist): """Extract source file list from package spec. :parma action: OBS request action dictionary :param filelist: List of package files :returns: List of source file names :raises SourceError: If something goes wrong """ try: spec_name = [name for name in filelist if name.endswith(".spec")][0] except IndexError: # raise SourceError("No spec file found") return [] try: spec = self.obs.getFile(action["sourceproject"], action["sourcepackage"], spec_name, action["sourcerevision"]) except Exception, exobj: raise SourceError("Failed to fetch spec file %s/%s/%s rev %s: %s" % (action["sourceproject"], action["sourcepackage"], spec_name, action["sourcerevision"], exobj)) try: tmp_spec = NamedTemporaryFile(mode="w") tmp_spec.file.write(spec) tmp_spec.file.flush() spec_obj = parse_spec(tmp_spec.name) sources = [os.path.basename(name) for name, _, _ in spec_obj.sources] tmp_spec.close() except ValueError, exobj: raise SourceError("Failed to parse spec file: %s" % exobj)
#!/usr/bin/python from pprint import pprint from buildservice import BuildService bs = BuildService(apiurl='http://api.meego.com', oscrc='/etc/boss/oscrc') print 'Bugowners of Trunk/bash:' pprint(bs.getPackagePersons('Trunk', 'bash', 'bugowner')) print 'Maintainers of Trunk/bash:' pprint(bs.getPackagePersons('Trunk', 'bash', 'maintainers'))
class ParticipantHandler(object): """Participant class as defined by the SkyNET API.""" def __init__(self): self.oscrc = None self.obs = None def handle_wi_control(self, ctrl): """Job control thread.""" pass def handle_lifecycle_control(self, ctrl): """Participant control thread.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance Using the namespace as an alias to the apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def revert_trial(self, wid): """Copy packages from trunk to testing. * The revert notes the intended request destination and copies the latest version from there (usually Trunk) back into the build_in project. * Later, if/when parallel build_in projects are used it may be important to correctly sync build_in from a potentially updated Trunk * If build_in is a project link we remove the packages and wipebinaries """ wid.result = False rid = wid.fields.ev.id actions = wid.fields.ev.actions build_in = wid.params.build_in for act in actions: if act['type'] != 'submit': continue if wid.params.linked: self.obs.deletePackage(build_in, act['targetpackage']) # wipeBinaries errors if there are no packages to wipe if self.obs.getPackageList(build_in): self.obs.wipeBinaries(build_in) else: try: self.obs.copyPackage(self.obs.apiurl, act['targetproject'], act['targetpackage'], self.obs.apiurl, build_in, act['targetpackage'], client_side_copy=False, keep_maintainers=False, keep_develproject=False, expand=False, comment="Trial revert for \ request %s" % rid) except HTTPError, exp: # If the package is not found in target, reverting is # done by deleting it from build_in. if exp.code == 404: self.obs.deletePackage(build_in, act['targetpackage']) else: raise
#!/usr/bin/python from buildservice import BuildService from osc import conf, core bs = BuildService(apiurl = 'https://api.meego.com', oscrc = "/etc/oscrc") #print bs.getRepoState('Trunk') #print bs.getProjectDiff('Trunk:Testing', 'Trunk') packages = bs.getPackageList('Trunk:Testing') for src_package in packages: print src_package diff = core.server_diff(bs.apiurl, 'Trunk', src_package, None, 'Trunk:Testing', src_package, None, False) p = open(src_package, 'w') p.write(diff) p.close()
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) @CheckActionProcessor("check_package_is_complete") def is_complete(self, action, wid): """ Package file completeness check """ filelist = self.obs.getPackageFileList( action['sourceproject'], action['sourcepackage'], action['sourcerevision']) # Remove all files starting with _ which don't have a : in # them (_constraints, _service but not _service:filename) filelist = [f for f in filelist if not re.match(r"_[^:]*$", f)] spec = self.has_spec_file(action, wid, filelist)[0] changes = self.has_changes_file(action, wid, filelist)[0] sources = spec and self.check_source_files(action, wid, filelist)[0] return (spec and changes and sources), "" def get_rpm_sources(self, action, filelist): """Extract source file list from package spec. :parma action: OBS request action dictionary :param filelist: List of package files :returns: List of source file names :raises SourceError: If something goes wrong """ try: specs = [name for name in filelist if name.endswith(".spec")] if len(specs) > 1: specs = [name for name in filelist if name.endswith("%s.spec" % action['sourcepackage'])] if len(specs) > 1: specs = [name for name in filelist if name.endswith(":%s.spec" % action['sourcepackage'])] spec_name = specs[0] except IndexError: # raise SourceError("No spec file found") return [] print spec_name try: spec = self.obs.getFile(action["sourceproject"], action["sourcepackage"], spec_name, action["sourcerevision"]) except Exception, exobj: raise SourceError("Failed to fetch spec file %s/%s/%s rev %s: %s" % (action["sourceproject"], action["sourcepackage"], spec_name, action["sourcerevision"], exobj)) import hashlib print "Spec file retrieved from", action["sourceproject"], action["sourcepackage"], action["sourcerevision"], ": ", hashlib.md5(spec).hexdigest() try: tmp_spec = NamedTemporaryFile(mode="w", delete=False) tmp_spec.file.write(spec) tmp_spec.file.flush() print "Parsing spec file from", tmp_spec.name spec_obj = parse_spec(tmp_spec.name) sources = [os.path.basename(name) for name, _, _ in spec_obj.sources] tmp_spec.close() except ValueError, exobj: raise SourceError("Failed to parse spec file: %s" % exobj)
#!/usr/bin/python from pprint import pprint from buildservice import BuildService bs = BuildService(apiurl='http://api.meego.com', oscrc='/etc/boss/oscrc' ) print 'devel project of Trunk:' pprint(bs.getProjectDevel('Trunk')) print 'devel package of Trunk/bash:' pprint(bs.getPackageDevel('Trunk', 'bash'))
class ParticipantHandler(object): """ Class implementation as required by the API""" def __init__(self): self.oscrc = None self.obs = None def handle_lifecycle_control(self, ctrl): """Participant lifecycle control.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def handle_wi(self, wid): """Participant workitem handler.""" wid.result = False if not wid.fields.msg: wid.fields.msg = [] if not wid.fields.ev: raise RuntimeError("Mandatory field 'ev' missing") if not wid.fields.ev.actions: raise RuntimeError("Mandatory field 'ev.actions' missing") if not wid.fields.ev.namespace: raise RuntimeError("Mandatory field 'ev.namespace' missing") wid.fields.package_conf = {} # We need direct access to the dictionary as DictAttrProxy does not # have __getitem__ package_conf = wid.fields.package_conf.as_dict() self._setup_obs(wid.fields.ev.namespace) for action in wid.fields.ev.actions: self._process_action(action, package_conf) wid.result = True def _setup_obs(self, namespace): """Initialize buildservice instance.""" self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def _get_boss_conf(self, project, package, revision=None): """Fetch boss.conf contents for package. :returns: boss.conf contents as string or None if package does not have boss.conf """ try: self.log.info("getting %s %s" % (project, package)) contents = self.obs.getFile(project, package, "boss.conf", revision) except HTTPError, exobj: if exobj.getcode() == 404: # Package does not have boss.conf contents = None else: # something else failed on OBS self.log.info("WTF!\n%s" % exobj) contents = None except Exception: # buildservice raises all kinds of weird exceptions self.log.info("Failed to get boss.conf for %s %s revision %s" % \ (project, package, revision)) raise
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def get_l10n_stats(self, source_project, target_project, package): tmp_dir_old = mkdtemp() tmp_dir_new = mkdtemp() old_ts_dir = tmp_dir_old + "/ts" new_ts_dir = tmp_dir_new + "/ts" target = self.obs.getTargets(str(source_project))[0] #get src.rpm as it contains all .ts files src_rpm = [ rpm for rpm in self.obs.getBinaryList(source_project, target, package) if "src.rpm" in rpm ] target_rpm = [ rpm for rpm in self.obs.getBinaryList(target_project, target, package) if "src.rpm" in rpm ] #download source and target rpms self.obs.getBinary(target_project, target, package, target_rpm[0], tmp_dir_old + "/old.rpm") self.obs.getBinary(source_project, target, package, src_rpm[0], tmp_dir_new + "/new.rpm") # extract rpms old_file = extract_rpm(tmp_dir_old + "/old.rpm", tmp_dir_old) new_file = extract_rpm(tmp_dir_new + "/new.rpm", tmp_dir_new) #rpm contains tar.bz2 and .spec file. Open and extract tar.bz2 old_tar = tarfile.open(tmp_dir_old + '/' + old_file[0]) old_tar.extractall(old_ts_dir) new_tar = tarfile.open(tmp_dir_new + '/' + new_file[0]) new_tar.extractall(new_ts_dir) old_ts_files = {} for member in old_tar.members: # rpm directrory has .spec file if member.name.split('/')[1] == 'rpm': continue # "lang : path_to_ts_file" pair old_ts_files.update({member.name.split('/')[1]: member.name}) new_ts_files = {} for member in new_tar.members: # rpm directrory has .spec file if member.name.split('/')[1] == 'rpm': continue # "lang : path_to_ts_file" pair new_ts_files.update({member.name.split('/')[1]: member.name}) l10n_stats = {} for key in set(new_ts_files.keys()) & set(old_ts_files.keys()): _old_path = tmp_dir_old + "/ts/" + old_ts_files[key] _new_path = tmp_dir_new + "/ts/" + new_ts_files[key] unit_diff = _make_ts_diff(_old_path, _new_path) l10n_stats.update({key: unit_diff}) l10n_stats.update({ "removed_langs": list(set(old_ts_files.keys()) - set(new_ts_files.keys())) }) l10n_stats.update({ "added_langs": list(set(new_ts_files.keys()) - set(old_ts_files.keys())) }) # possible removed strings l10n_stats.update({"removed_strings": []}) #check that -ts-devel package is not going out of sync src_pkg = package.replace("-l10n", "") #is there a package that is using -l10n pakcage already src_pkg = [ rpm for rpm in self.obs.getPackageList(target_project) if src_pkg == rpm ] if len(src_pkg) > 0: #get -ts-devel rpm src_ts_devel_rpm = [ rpm for rpm in self.obs.getBinaryList( target_project, target, src_pkg[0]) if "-ts-devel" in rpm ] if len(src_ts_devel_rpm) > 0: tmp_dir_ts = mkdtemp() self.obs.getBinary(target_project, target, src_pkg[0], src_ts_devel_rpm[0], tmp_dir_ts + "/orig.rpm") orig_ts_file = extract_rpm(tmp_dir_ts + "/orig.rpm", tmp_dir_ts, patterns="*.ts") original_units = factory.getobject(tmp_dir_ts + "/" + orig_ts_file[0]) new_units = factory.getobject(tmp_dir_new + "/ts/" + new_ts_files['templates']) removed_units = set(original_units.getids()) - set( new_units.getids()) l10n_stats.update({"removed_strings": list(removed_units)}) shutil.rmtree(tmp_dir_ts) #get rid of tmp dirs shutil.rmtree(tmp_dir_old) shutil.rmtree(tmp_dir_new) return l10n_stats def handle_wi(self, wid): """ actual job thread """ wid.result = True if not wid.fields.msg: wid.fields.msg = [] if not wid.fields.ev: raise RuntimeError("Missing mandatory field 'ev'") if not wid.fields.ev.namespace: raise RuntimeError("Missing mandatory field 'ev.namespace'") if not wid.fields.ev.actions: raise RuntimeError("Missing mandatory field 'ev.actions'") self.setup_obs(wid.fields.ev.namespace) tgt_pkg_list = self.obs.getPackageList(str(wid.fields.project)) all_ok = True for action in wid.fields.ev.actions: if action['type'] != 'submit': continue # do the check only for l10n packages if not "-l10n" in action['sourcepackage']: continue if action['sourcepackage'] not in tgt_pkg_list: #nothing to diff, pass through continue msg = "" package_ok = True l10n_stats = self.get_l10n_stats(str(action['sourceproject']), str(action['targetproject']), str(action['sourcepackage'])) #store stats for later use wid.fields.l10n = {"stats": l10n_stats} # check if there is '<pkg> bypass' message if wid.fields.ev.description: re1 = re.compile(r'%s bypass' % action['sourcepackage']) if re1.search(wid.fields.ev.description): continue for key, value in l10n_stats.items(): # removed_langs & added_langs if "_langs" in key: continue if "removed_strings" in key: continue if "Instructions" in key: continue old_translated = float(value["old_trans_count"]) new_translated = float(value["new_trans_count"]) old_units = value["old_units"] new_units = value["new_units"] added = len(value["added"]) # check that translation level does not go down. New strings can be added # without an effect if (old_translated / old_units) > ( (new_translated + added) / new_units): all_ok = package_ok = False msg += "%s level down from %.4f to %.4f" % ( key, old_translated / old_units, (new_translated + added) / new_units) # check that already present languages are not removed if len(l10n_stats["removed_langs"]) > 0: all_ok = package_ok = False msg += "%s langs removed" % (", ".join( l10n_stats['removed_langs'])) if not package_ok: wid.fields.msg.append( "%(sourcepackage)s has following l10n error(s):" % action) wid.fields.msg.append(msg) wid.result = all_ok
def setup_obs(self, namespace): """Setup the Buildservice instance :param namespace: Alias to the OBS apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace)
class ParticipantHandler(object): """Participant class as defined by the SkyNET API.""" def __init__(self): self.oscrc = None self.obs = None def handle_wi_control(self, ctrl): """Job control thread.""" pass def handle_lifecycle_control(self, ctrl): """Participant control thread.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance Using the namespace as an alias to the apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def revert_trial(self, wid): """Copy packages from trunk to testing. * The revert notes the intended request destination and copies the latest version from there (usually Trunk) back into the build_in project. * Later, if/when parallel build_in projects are used it may be important to correctly sync build_in from a potentially updated Trunk * If build_in is a project link we remove the packages and wipebinaries """ wid.result = False rid = wid.fields.ev.id actions = wid.fields.ev.actions build_in = wid.params.build_in for act in actions: if act['type'] != 'submit': continue if wid.params.linked : self.obs.deletePackage(build_in, act['targetpackage']) # wipeBinaries errors if there are no packages to wipe if self.obs.getPackageList(build_in): self.obs.wipeBinaries(build_in) else: try: self.obs.copyPackage(self.obs.apiurl, act['targetproject'], act['targetpackage'], self.obs.apiurl, build_in, act['targetpackage'], client_side_copy = False, keep_maintainers = False, keep_develproject = False, expand = False, comment = "Trial revert for \ request %s" % rid) except HTTPError, exp: # If the package is not found in target, reverting is # done by deleting it from build_in. if exp.code == 404: self.obs.deletePackage(build_in, act['targetpackage']) else: raise
def setup_obs(self, namespace): """Set up OBS instance.""" if not self.obs or self.namespace != namespace: self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) self.namespace = namespace
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def get_spec_file(self, prj, pkg, rev=None): """ Get a package's spec file """ spec = "" file_list = self.obs.getPackageFileList(prj, pkg, revision=rev) for fil in file_list: if fil.endswith(".spec"): spec = self.obs.getFile(prj, pkg, fil, revision=rev) return spec @CheckActionProcessor("check_spec") def spec_valid(self, action, _wid): """ Get spec file and check for various indications of spec file validity """ spec = self.get_spec_file(action['sourceproject'], action['sourcepackage'], action['sourcerevision']) if has_section_or_tag(spec, "%changelog"): return False, "Spec file should not contain the %%changelog tag, "\ "otherwise the changes file is ignored." return True, None def handle_wi(self, wid): """ actual job thread """ wid.result = False if not wid.fields.msg: wid.fields.msg = [] actions = wid.fields.ev.actions if not actions: raise RuntimeError("Mandatory field: actions does not exist.") self.setup_obs(wid.fields.ev.namespace) result = True for action in actions: # Assert validity of spec file valid, _ = self.spec_valid(action, wid) result = result and valid wid.result = result
class ParticipantHandler(object): """Participant class as defined by the SkyNET API""" def __init__(self): self.obs = None self.oscrc = None self.validator = Validator() def handle_wi_control(self, ctrl): """Job control thread""" pass def handle_lifecycle_control(self, ctrl): """Handle messages for the participant itself, like start and stop.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance :param namespace: Alias to the OBS apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def _get_spec_file(self, prj, pkg, rev): file_list = [] try: file_list = self.obs.getPackageFileList(prj, pkg, revision=rev) except: pass specs = [ fil for fil in file_list if fil.endswith(".spec")] if len(specs) > 1: specs = [ fil for fil in specs if (fil == "%s.spec" % pkg or fil.endswith(":%s.spec" % pkg))] if not specs: return "No specfile in %s" % pkg, None elif len(specs) > 1: return "Couldn't determine which spec file to use for %s " % pkg, None print specs fil = specs[0] spec = self.obs.getFile(prj, pkg, fil, revision=rev) specob = None print fil print spec with NamedTemporaryFile() as specf: specf.write(spec) specf.flush() try: specob = parse_spec(specf.name) except ValueError, exobj: return "Could not parse spec in %s: %s" % (pkg, exobj), None return None, specob
#!/usr/bin/python from pprint import pprint from buildservice import BuildService bs = BuildService(apiurl='http://api.meego.com', oscrc='/etc/boss/oscrc' ) print 'Bugowners of Trunk' pprint(bs.getProjectPersons('Trunk', 'bugowner')) print 'Maintainers of Trunk:' pprint(bs.getProjectPersons('Trunk', 'maintainers'))
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def get_l10n_stats(self, source_project, target_project, package): tmp_dir_old = mkdtemp() tmp_dir_new = mkdtemp() old_ts_dir = tmp_dir_old + "/ts" new_ts_dir = tmp_dir_new + "/ts" target = self.obs.getTargets(str(source_project))[0] #get src.rpm as it contains all .ts files src_rpm = [rpm for rpm in self.obs.getBinaryList( source_project, target, package) if "src.rpm" in rpm] target_rpm = [rpm for rpm in self.obs.getBinaryList( target_project, target, package) if "src.rpm" in rpm] #download source and target rpms self.obs.getBinary(target_project, target, package, target_rpm[0], tmp_dir_old + "/old.rpm") self.obs.getBinary(source_project, target, package, src_rpm[0], tmp_dir_new + "/new.rpm") # extract rpms old_file = extract_rpm(tmp_dir_old + "/old.rpm", tmp_dir_old) new_file = extract_rpm(tmp_dir_new + "/new.rpm", tmp_dir_new) #rpm contains tar.bz2 and .spec file. Open and extract tar.bz2 old_tar = tarfile.open(tmp_dir_old + '/' + old_file[0]) old_tar.extractall(old_ts_dir) new_tar = tarfile.open(tmp_dir_new + '/' + new_file[0]) new_tar.extractall(new_ts_dir) old_ts_files = {} for member in old_tar.members: # rpm directrory has .spec file if member.name.split('/')[1] == 'rpm': continue # "lang : path_to_ts_file" pair old_ts_files.update({member.name.split('/')[1] : member.name }) new_ts_files = {} for member in new_tar.members: # rpm directrory has .spec file if member.name.split('/')[1] == 'rpm': continue # "lang : path_to_ts_file" pair new_ts_files.update({member.name.split('/')[1] : member.name }) l10n_stats = {} for key in set(new_ts_files.keys()) & set(old_ts_files.keys()): _old_path = tmp_dir_old + "/ts/" + old_ts_files[key] _new_path = tmp_dir_new + "/ts/" + new_ts_files[key] unit_diff = _make_ts_diff(_old_path, _new_path) l10n_stats.update({ key : unit_diff }) l10n_stats.update({"removed_langs" : list(set(old_ts_files.keys()) - set(new_ts_files.keys())) }) l10n_stats.update({"added_langs" : list(set(new_ts_files.keys()) - set(old_ts_files.keys())) }) # possible removed strings l10n_stats.update({ "removed_strings" : [] }) #check that -ts-devel package is not going out of sync src_pkg = package.replace("-l10n", "") #is there a package that is using -l10n pakcage already src_pkg = [rpm for rpm in self.obs.getPackageList(target_project) if src_pkg == rpm] if len(src_pkg) > 0: #get -ts-devel rpm src_ts_devel_rpm = [rpm for rpm in self.obs.getBinaryList(target_project, target, src_pkg[0]) if "-ts-devel" in rpm] if len(src_ts_devel_rpm) > 0: tmp_dir_ts = mkdtemp() self.obs.getBinary(target_project, target, src_pkg[0], src_ts_devel_rpm[0], tmp_dir_ts + "/orig.rpm") orig_ts_file = extract_rpm(tmp_dir_ts + "/orig.rpm", tmp_dir_ts, patterns="*.ts") original_units = factory.getobject(tmp_dir_ts + "/" + orig_ts_file[0]) new_units = factory.getobject(tmp_dir_new + "/ts/" + new_ts_files['templates']) removed_units = set(original_units.getids()) - set(new_units.getids()) l10n_stats.update({"removed_strings" : list(removed_units)}) shutil.rmtree(tmp_dir_ts) #get rid of tmp dirs shutil.rmtree(tmp_dir_old) shutil.rmtree(tmp_dir_new) return l10n_stats def handle_wi(self, wid): """ actual job thread """ wid.result = True if not wid.fields.msg: wid.fields.msg = [] if not wid.fields.ev: raise RuntimeError("Missing mandatory field 'ev'") if not wid.fields.ev.namespace: raise RuntimeError("Missing mandatory field 'ev.namespace'") if not wid.fields.ev.actions: raise RuntimeError("Missing mandatory field 'ev.actions'") self.setup_obs(wid.fields.ev.namespace) tgt_pkg_list = self.obs.getPackageList(str(wid.fields.project)) all_ok = True for action in wid.fields.ev.actions: if action['type'] != 'submit': continue # do the check only for l10n packages if not "-l10n" in action['sourcepackage']: continue if action['sourcepackage'] not in tgt_pkg_list: #nothing to diff, pass through continue msg = "" package_ok = True l10n_stats = self.get_l10n_stats(str(action['sourceproject']), str(action['targetproject']), str(action['sourcepackage'])) #store stats for later use wid.fields.l10n = { "stats" : l10n_stats } # check if there is '<pkg> bypass' message if wid.fields.ev.description: re1 = re.compile(r'%s bypass' % action['sourcepackage']) if re1.search(wid.fields.ev.description): continue for key, value in l10n_stats.items(): # removed_langs & added_langs if "_langs" in key: continue if "removed_strings" in key: continue if "Instructions" in key: continue old_translated = float(value["old_trans_count"]) new_translated = float(value["new_trans_count"]) old_units = value["old_units"] new_units = value["new_units"] added = len(value["added"]) # check that translation level does not go down. New strings can be added # without an effect if (old_translated / old_units ) > ((new_translated + added) / new_units): all_ok = package_ok = False msg += "%s level down from %.4f to %.4f" % ( key, old_translated/ old_units, (new_translated + added) / new_units) # check that already present languages are not removed if len(l10n_stats["removed_langs"]) > 0: all_ok = package_ok = False msg += "%s langs removed" % (", ".join(l10n_stats['removed_langs'])) if not package_ok: wid.fields.msg.append("%(sourcepackage)s has following l10n error(s):" % action) wid.fields.msg.append(msg) wid.result = all_ok
class ParticipantHandler(object): """ Class implementation as required by the API""" def __init__(self): self.oscrc = None self.obs = None def handle_lifecycle_control(self, ctrl): """Participant lifecycle control.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def handle_wi(self, wid): """Participant workitem handler.""" wid.result = False if not wid.fields.msg: wid.fields.msg = [] if not wid.fields.ev: raise RuntimeError("Mandatory field 'ev' missing") if not wid.fields.ev.actions: raise RuntimeError("Mandatory field 'ev.actions' missing") if not wid.fields.ev.namespace: raise RuntimeError("Mandatory field 'ev.namespace' missing") wid.fields.package_conf = {} # We need direct access to the dictionary as DictAttrProxy does not # have __getitem__ package_conf = wid.fields.package_conf.as_dict() self._setup_obs(wid.fields.ev.namespace) for action in wid.fields.ev.actions: self._process_action(action, package_conf) wid.result = True def _setup_obs(self, namespace): """Initialize buildservice instance.""" self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def _get_boss_conf(self, project, package, revision=None): """Fetch boss.conf contents for package. :returns: boss.conf contents as string or None if package does not have boss.conf """ try: self.log.info("getting %s %s" % (project, package)) contents = self.obs.getFile( project, package, "boss.conf", revision) except HTTPError, exobj: if exobj.getcode() == 404: # Package does not have boss.conf contents = None else: # something else failed on OBS self.log.info("WTF!\n%s" % exobj) contents = None except Exception: # buildservice raises all kinds of weird exceptions self.log.info("Failed to get boss.conf for %s %s revision %s" % \ (project, package, revision)) raise
class ParticipantHandler(object): """Participant class as defined by the SkyNET API.""" def __init__(self): self.oscrc = None self.obs = None def handle_wi_control(self, ctrl): """Job control thread.""" pass def handle_lifecycle_control(self, ctrl): """Participant control thread.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance Using namespace as an alias to the apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def build_trial_results(self, wid): """Main function to get new failures related to a build trial.""" wid.result = False if not wid.fields.msg: wid.fields.msg = [] target_prj = wid.fields.project build_in_prj = wid.params.build_in exclude_repos = wid.fields.exclude_repos or [] exclude_archs = wid.fields.exclude_archs or [] new_failures = set() for target_repo in self.obs.getProjectRepositories(target_prj): if target_repo in exclude_repos: continue archs = self.obs.getRepositoryArchs(target_prj, target_repo) archs = [arch for arch in archs if arch not in exclude_archs] # Get the repository of the build trial which builds against the # required target repo in the target prj build_in_repo = self.obs.getTargetRepo(build_in_prj, target_prj, target_repo, archs) # Get trial build results trial_results = self.obs.getRepoResults(build_in_prj, build_in_repo) # Get destination results orig_results = self.obs.getRepoResults(target_prj, target_repo) # compare them and return new failures new_failures.update(get_new_failures(trial_results, orig_results, archs)) if len(new_failures): wid.fields.msg.append("During the trial build in %s, %s failed to"\ " build for one of the archs : %s" % (build_in_prj, " ".join(new_failures), " ".join(archs))) wid.fields.new_failures = list(new_failures) else: wid.fields.msg.append("Trial build of packages in %s successful" % build_in_prj) wid.result = True def handle_wi(self, wid): """Actual job thread.""" self.setup_obs(wid.fields.ev.namespace) self.build_trial_results(wid)
class ParticipantHandler(object): """ Participant class as defined by the SkyNET API """ def __init__(self): self.obs = None self.oscrc = None def handle_wi_control(self, ctrl): """ job control thread """ pass def handle_lifecycle_control(self, ctrl): """ participant control thread """ if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """ setup the Buildservice instance using the namespace as an alias to the apiurl """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) @CheckActionProcessor("check_package_is_complete") def is_complete(self, action, wid): """ Package file completeness check """ filelist = self.obs.getPackageFileList(action['sourceproject'], action['sourcepackage'], action['sourcerevision']) if "_service" in filelist: filelist.remove("_service") if "_ccache" in filelist: filelist.remove("_ccache") if "_src" in filelist: filelist.remove("_src") spec = self.has_spec_file(action, wid, filelist)[0] changes = self.has_changes_file(action, wid, filelist)[0] sources = spec and self.check_source_files(action, wid, filelist)[0] return (spec and changes and sources), "" def get_rpm_sources(self, action, filelist): """Extract source file list from package spec. :parma action: OBS request action dictionary :param filelist: List of package files :returns: List of source file names :raises SourceError: If something goes wrong """ try: specs = [name for name in filelist if name.endswith(".spec")] if len(specs) > 1: specs = [ name for name in filelist if name.endswith("%s.spec" % action['sourcepackage']) ] if len(specs) > 1: specs = [ name for name in filelist if name.endswith(":%s.spec" % action['sourcepackage']) ] spec_name = specs[0] except IndexError: # raise SourceError("No spec file found") return [] print spec_name try: spec = self.obs.getFile(action["sourceproject"], action["sourcepackage"], spec_name, action["sourcerevision"]) except Exception, exobj: raise SourceError( "Failed to fetch spec file %s/%s/%s rev %s: %s" % (action["sourceproject"], action["sourcepackage"], spec_name, action["sourcerevision"], exobj)) import hashlib print "Spec file retrieved from", action["sourceproject"], action[ "sourcepackage"], action["sourcerevision"], ": ", hashlib.md5( spec).hexdigest() try: tmp_spec = NamedTemporaryFile(mode="w", delete=False) tmp_spec.file.write(spec) tmp_spec.file.flush() print "Parsing spec file from", tmp_spec.name spec_obj = parse_spec(tmp_spec.name) sources = [ os.path.basename(name) for name, _, _ in spec_obj.sources ] tmp_spec.close() except ValueError, exobj: raise SourceError("Failed to parse spec file: %s" % exobj)
class ParticipantHandler(object): """Participant class as defined by the SkyNET API.""" def __init__(self): self.oscrc = None self.obs = None def handle_wi_control(self, ctrl): """Job control thread.""" pass def handle_lifecycle_control(self, ctrl): """Participant control thread.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance. Using the namespace as an alias to the apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def handle_request(self, wid): """Request handling implementation.""" wid.result = False if not wid.fields.msg: wid.fields.msg = [] rid = str(wid.fields.ev.id) # https://api.opensuse.org/apidocs/request.xsd obj_type = "request" newstate = None if wid.params.action == 'accept': newstate = "accepted" elif wid.params.action == 'reject' or wid.params.action == 'decline': newstate = "declined" elif wid.params.action == 'add review': obj_type = "review" newstate = "add" elif wid.params.action == 'accept review': obj_type = "review" newstate = "accepted" elif wid.params.action == 'decline review': obj_type = "review" newstate = "declined" Verify.assertNotNull("Request ID field must not be empty", rid) Verify.assertNotNull("Participant action should be one of accept, "\ "decline, add review, accept review, "\ "decline review", newstate) try: if obj_type == "review": reviewer = wid.params.reviewer extra_msg = "" if wid.params.comment: extra_msg = "%s\n" % wid.params.comment if not reviewer: reviewer = self.obs.getUserName() if newstate == "add": res = self.obs.addReview(rid, extra_msg, reviewer) else: res = self.obs.setReviewState(rid, newstate, extra_msg, reviewer) elif obj_type == "request": extra_msg = "" if wid.params.comment: extra_msg = "%s\n" % wid.params.comment msgstring = "%sBOSS %s this %s because:\n %s" % ( extra_msg, newstate, obj_type, "\n ".join(wid.fields.msg)) res = self.obs.setRequestState(rid, newstate, msgstring) if res: self.log.info("%s %s %s" % (newstate, obj_type, rid)) wid.result = True else: self.log.info("Failed to %s %s %s" % (wid.params.action, obj_type, rid)) except HTTPError, exc: if exc.code == 403: wid.fields.msg.append("Applying the actions required to "\ "automate this process has failed, "\ "because the %s user was not authorized "\ "to do so. "\ "Please add %s as a maintainer in the "\ "target projet %s" % (self.obs.getUserName(), self.obs.getUserName(), wid.fields.project)) self.log.info("Forbidden to %s %s %s" % (wid.params.action, obj_type, rid)) elif exc.code == 401: wid.fields.msg.append("Credentials for the '%s' user were "\ "refused. Please update the skynet "\ "configuration." % self.obs.getUserName()) self.log.info(exc) self.log.info("User is %s" % self.obs.getUserName()) else: import traceback self.log.info(exc) traceback.print_exc() raise
class ParticipantHandler(object): """Participant class as defined by the SkyNET API.""" def __init__(self): self.oscrc = None self.obs = None def handle_wi_control(self, ctrl): """Job control thread.""" pass def handle_lifecycle_control(self, ctrl): """Participant control thread.""" if ctrl.message == "start": if ctrl.config.has_option("obs", "oscrc"): self.oscrc = ctrl.config.get("obs", "oscrc") def setup_obs(self, namespace): """Setup the Buildservice instance Using namespace as an alias to the apiurl. """ self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace) def check_trial(self, prj, targets, acts, exc): exclude_repos, exclude_archs = exc new_failures = set() old_failures = set() for target_prj in targets: for target_repo in self.obs.getProjectRepositories(target_prj): if target_repo in exclude_repos: continue archs = self.obs.getRepositoryArchs(target_prj, target_repo) archs = [arch for arch in archs if arch not in exclude_archs] print archs # Get the repository of the build trial which builds against the # required target repo in the target prj #build_in_repo = self.obs.getTargetRepo(build_in_prj, target_prj, # target_repo, archs) # Get trial build results trial_results = self.obs.getRepoResults(prj, target_repo) # Get destination results orig_results = self.obs.getRepoResults(target_prj, target_repo) # compare them and return new failures comparison = get_new_failures(trial_results, orig_results, archs, acts) new_failures.update(comparison[0]) old_failures.update(comparison[1]) return new_failures - old_failures def build_trial_results(self, wid): """Main function to get new failures related to a build trial.""" wid.result = False if not wid.fields.msg: wid.fields.msg = [] exclude_repos = wid.fields.exclude_repos or [] exclude_archs = wid.fields.exclude_archs or [] trial_project = wid.fields.build_trial.project trial_subprjs = wid.fields.build_trial.as_dict().get('subprojects', {}) subtargets = list(itertools.chain.from_iterable(trial_subprjs.values())) actions = [act for act in wid.fields.ev.actions if act["targetproject"] not in subtargets] targets = [act["targetproject"] for act in actions] messages = [] all_fails = [] fails = self.check_trial(trial_project, targets, actions, exc=(exclude_repos, exclude_archs)) if fails: all_fails.extend(fails) wid.fields.msg.append("During the trial build in %s, %s failed to"\ " build" % (trial_project, " ".join(fails))) else: wid.fields.msg.append("Trial build of packages in %s successful" % trial_project) for trial_sub, subtargets in trial_subprjs.items(): subactions = [act for act in wid.fields.ev.actions if act["targetproject"] in subtargets] fails = self.check_trial(trial_sub, subtargets, subactions, exc=(exclude_repos, exclude_archs)) if fails: all_fails.extend(fails) wid.fields.msg.append("During the trial build in %s, %s failed to build" % (trial_sub, " ".join(fails))) else: wid.fields.msg.append("Trial build of packages in %s successful" % trial_sub) if all_fails: wid.fields.new_failures = all_fails else: wid.result = True def handle_wi(self, wid): """Actual job thread.""" self.setup_obs(wid.fields.ev.namespace) self.build_trial_results(wid)