class ParticipantHandler(object):
""" Participant class as defined by the SkyNET API """
def __init__(self):
self.obs = None
self.oscrc = None
def handle_wi_control(self, ctrl):
""" job control thread """
pass
def handle_lifecycle_control(self, ctrl):
""" participant control thread """
if ctrl.message == "start":
if ctrl.config.has_option("obs", "oscrc"):
self.oscrc = ctrl.config.get("obs", "oscrc")
def setup_obs(self, namespace):
""" setup the Buildservice instance using the namespace as an alias
to the apiurl """
self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace)
def get_l10n_stats(self, source_project, target_project, package):
tmp_dir_old = mkdtemp()
tmp_dir_new = mkdtemp()
old_ts_dir = tmp_dir_old + "/ts"
new_ts_dir = tmp_dir_new + "/ts"
target = self.obs.getTargets(str(source_project))[0]
#get src.rpm as it contains all .ts files
src_rpm = [
rpm
for rpm in self.obs.getBinaryList(source_project, target, package)
if "src.rpm" in rpm
]
target_rpm = [
rpm
for rpm in self.obs.getBinaryList(target_project, target, package)
if "src.rpm" in rpm
]
#download source and target rpms
self.obs.getBinary(target_project, target, package, target_rpm[0],
tmp_dir_old + "/old.rpm")
self.obs.getBinary(source_project, target, package, src_rpm[0],
tmp_dir_new + "/new.rpm")
# extract rpms
old_file = extract_rpm(tmp_dir_old + "/old.rpm", tmp_dir_old)
new_file = extract_rpm(tmp_dir_new + "/new.rpm", tmp_dir_new)
#rpm contains tar.bz2 and .spec file. Open and extract tar.bz2
old_tar = tarfile.open(tmp_dir_old + '/' + old_file[0])
old_tar.extractall(old_ts_dir)
new_tar = tarfile.open(tmp_dir_new + '/' + new_file[0])
new_tar.extractall(new_ts_dir)
old_ts_files = {}
for member in old_tar.members:
# rpm directrory has .spec file
if member.name.split('/')[1] == 'rpm':
continue
# "lang : path_to_ts_file" pair
old_ts_files.update({member.name.split('/')[1]: member.name})
new_ts_files = {}
for member in new_tar.members:
# rpm directrory has .spec file
if member.name.split('/')[1] == 'rpm':
continue
# "lang : path_to_ts_file" pair
new_ts_files.update({member.name.split('/')[1]: member.name})
l10n_stats = {}
for key in set(new_ts_files.keys()) & set(old_ts_files.keys()):
_old_path = tmp_dir_old + "/ts/" + old_ts_files[key]
_new_path = tmp_dir_new + "/ts/" + new_ts_files[key]
unit_diff = _make_ts_diff(_old_path, _new_path)
l10n_stats.update({key: unit_diff})
l10n_stats.update({
"removed_langs":
list(set(old_ts_files.keys()) - set(new_ts_files.keys()))
})
l10n_stats.update({
"added_langs":
list(set(new_ts_files.keys()) - set(old_ts_files.keys()))
})
# possible removed strings
l10n_stats.update({"removed_strings": []})
#check that -ts-devel package is not going out of sync
src_pkg = package.replace("-l10n", "")
#is there a package that is using -l10n pakcage already
src_pkg = [
rpm for rpm in self.obs.getPackageList(target_project)
if src_pkg == rpm
]
if len(src_pkg) > 0:
#get -ts-devel rpm
src_ts_devel_rpm = [
rpm for rpm in self.obs.getBinaryList(
target_project, target, src_pkg[0]) if "-ts-devel" in rpm
]
if len(src_ts_devel_rpm) > 0:
tmp_dir_ts = mkdtemp()
self.obs.getBinary(target_project, target, src_pkg[0],
src_ts_devel_rpm[0],
tmp_dir_ts + "/orig.rpm")
orig_ts_file = extract_rpm(tmp_dir_ts + "/orig.rpm",
tmp_dir_ts,
patterns="*.ts")
original_units = factory.getobject(tmp_dir_ts + "/" +
orig_ts_file[0])
new_units = factory.getobject(tmp_dir_new + "/ts/" +
new_ts_files['templates'])
removed_units = set(original_units.getids()) - set(
new_units.getids())
l10n_stats.update({"removed_strings": list(removed_units)})
shutil.rmtree(tmp_dir_ts)
#get rid of tmp dirs
shutil.rmtree(tmp_dir_old)
shutil.rmtree(tmp_dir_new)
return l10n_stats
def handle_wi(self, wid):
""" actual job thread """
wid.result = True
if not wid.fields.msg:
wid.fields.msg = []
if not wid.fields.ev:
raise RuntimeError("Missing mandatory field 'ev'")
if not wid.fields.ev.namespace:
raise RuntimeError("Missing mandatory field 'ev.namespace'")
if not wid.fields.ev.actions:
raise RuntimeError("Missing mandatory field 'ev.actions'")
self.setup_obs(wid.fields.ev.namespace)
tgt_pkg_list = self.obs.getPackageList(str(wid.fields.project))
all_ok = True
for action in wid.fields.ev.actions:
if action['type'] != 'submit':
continue
# do the check only for l10n packages
if not "-l10n" in action['sourcepackage']:
continue
if action['sourcepackage'] not in tgt_pkg_list:
#nothing to diff, pass through
continue
msg = ""
package_ok = True
l10n_stats = self.get_l10n_stats(str(action['sourceproject']),
str(action['targetproject']),
str(action['sourcepackage']))
#store stats for later use
wid.fields.l10n = {"stats": l10n_stats}
# check if there is '<pkg> bypass' message
if wid.fields.ev.description:
re1 = re.compile(r'%s bypass' % action['sourcepackage'])
if re1.search(wid.fields.ev.description):
continue
for key, value in l10n_stats.items():
# removed_langs & added_langs
if "_langs" in key:
continue
if "removed_strings" in key:
continue
if "Instructions" in key:
continue
old_translated = float(value["old_trans_count"])
new_translated = float(value["new_trans_count"])
old_units = value["old_units"]
new_units = value["new_units"]
added = len(value["added"])
# check that translation level does not go down. New strings can be added
# without an effect
if (old_translated / old_units) > (
(new_translated + added) / new_units):
all_ok = package_ok = False
msg += "%s level down from %.4f to %.4f" % (
key, old_translated / old_units,
(new_translated + added) / new_units)
# check that already present languages are not removed
if len(l10n_stats["removed_langs"]) > 0:
all_ok = package_ok = False
msg += "%s langs removed" % (", ".join(
l10n_stats['removed_langs']))
if not package_ok:
wid.fields.msg.append(
"%(sourcepackage)s has following l10n error(s):" % action)
wid.fields.msg.append(msg)
wid.result = all_ok
class ParticipantHandler(object):
""" Participant class as defined by the SkyNET API """
def __init__(self):
self.obs = None
self.oscrc = None
def handle_wi_control(self, ctrl):
""" job control thread """
pass
def handle_lifecycle_control(self, ctrl):
""" participant control thread """
if ctrl.message == "start":
if ctrl.config.has_option("obs", "oscrc"):
self.oscrc = ctrl.config.get("obs", "oscrc")
def setup_obs(self, namespace):
""" setup the Buildservice instance using the namespace as an alias
to the apiurl """
self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace)
def get_l10n_stats(self, source_project, target_project, package):
tmp_dir_old = mkdtemp()
tmp_dir_new = mkdtemp()
old_ts_dir = tmp_dir_old + "/ts"
new_ts_dir = tmp_dir_new + "/ts"
target = self.obs.getTargets(str(source_project))[0]
#get src.rpm as it contains all .ts files
src_rpm = [rpm for rpm in self.obs.getBinaryList(
source_project, target, package) if "src.rpm" in rpm]
target_rpm = [rpm for rpm in self.obs.getBinaryList(
target_project, target, package) if "src.rpm" in rpm]
#download source and target rpms
self.obs.getBinary(target_project, target, package, target_rpm[0],
tmp_dir_old + "/old.rpm")
self.obs.getBinary(source_project, target, package, src_rpm[0],
tmp_dir_new + "/new.rpm")
# extract rpms
old_file = extract_rpm(tmp_dir_old + "/old.rpm", tmp_dir_old)
new_file = extract_rpm(tmp_dir_new + "/new.rpm", tmp_dir_new)
#rpm contains tar.bz2 and .spec file. Open and extract tar.bz2
old_tar = tarfile.open(tmp_dir_old + '/' + old_file[0])
old_tar.extractall(old_ts_dir)
new_tar = tarfile.open(tmp_dir_new + '/' + new_file[0])
new_tar.extractall(new_ts_dir)
old_ts_files = {}
for member in old_tar.members:
# rpm directrory has .spec file
if member.name.split('/')[1] == 'rpm':
continue
# "lang : path_to_ts_file" pair
old_ts_files.update({member.name.split('/')[1] : member.name })
new_ts_files = {}
for member in new_tar.members:
# rpm directrory has .spec file
if member.name.split('/')[1] == 'rpm':
continue
# "lang : path_to_ts_file" pair
new_ts_files.update({member.name.split('/')[1] : member.name })
l10n_stats = {}
for key in set(new_ts_files.keys()) & set(old_ts_files.keys()):
_old_path = tmp_dir_old + "/ts/" + old_ts_files[key]
_new_path = tmp_dir_new + "/ts/" + new_ts_files[key]
unit_diff = _make_ts_diff(_old_path, _new_path)
l10n_stats.update({ key : unit_diff })
l10n_stats.update({"removed_langs" : list(set(old_ts_files.keys()) - set(new_ts_files.keys())) })
l10n_stats.update({"added_langs" : list(set(new_ts_files.keys()) - set(old_ts_files.keys())) })
# possible removed strings
l10n_stats.update({ "removed_strings" : [] })
#check that -ts-devel package is not going out of sync
src_pkg = package.replace("-l10n", "")
#is there a package that is using -l10n pakcage already
src_pkg = [rpm for rpm in self.obs.getPackageList(target_project) if src_pkg == rpm]
if len(src_pkg) > 0:
#get -ts-devel rpm
src_ts_devel_rpm = [rpm for rpm in self.obs.getBinaryList(target_project, target, src_pkg[0]) if "-ts-devel" in rpm]
if len(src_ts_devel_rpm) > 0:
tmp_dir_ts = mkdtemp()
self.obs.getBinary(target_project, target, src_pkg[0], src_ts_devel_rpm[0], tmp_dir_ts + "/orig.rpm")
orig_ts_file = extract_rpm(tmp_dir_ts + "/orig.rpm", tmp_dir_ts, patterns="*.ts")
original_units = factory.getobject(tmp_dir_ts + "/" + orig_ts_file[0])
new_units = factory.getobject(tmp_dir_new + "/ts/" + new_ts_files['templates'])
removed_units = set(original_units.getids()) - set(new_units.getids())
l10n_stats.update({"removed_strings" : list(removed_units)})
shutil.rmtree(tmp_dir_ts)
#get rid of tmp dirs
shutil.rmtree(tmp_dir_old)
shutil.rmtree(tmp_dir_new)
return l10n_stats
def handle_wi(self, wid):
""" actual job thread """
wid.result = True
if not wid.fields.msg:
wid.fields.msg = []
if not wid.fields.ev:
raise RuntimeError("Missing mandatory field 'ev'")
if not wid.fields.ev.namespace:
raise RuntimeError("Missing mandatory field 'ev.namespace'")
if not wid.fields.ev.actions:
raise RuntimeError("Missing mandatory field 'ev.actions'")
self.setup_obs(wid.fields.ev.namespace)
tgt_pkg_list = self.obs.getPackageList(str(wid.fields.project))
all_ok = True
for action in wid.fields.ev.actions:
if action['type'] != 'submit':
continue
# do the check only for l10n packages
if not "-l10n" in action['sourcepackage']:
continue
if action['sourcepackage'] not in tgt_pkg_list:
#nothing to diff, pass through
continue
msg = ""
package_ok = True
l10n_stats = self.get_l10n_stats(str(action['sourceproject']),
str(action['targetproject']),
str(action['sourcepackage']))
#store stats for later use
wid.fields.l10n = { "stats" : l10n_stats }
# check if there is '<pkg> bypass' message
if wid.fields.ev.description:
re1 = re.compile(r'%s bypass' % action['sourcepackage'])
if re1.search(wid.fields.ev.description):
continue
for key, value in l10n_stats.items():
# removed_langs & added_langs
if "_langs" in key:
continue
if "removed_strings" in key:
continue
if "Instructions" in key:
continue
old_translated = float(value["old_trans_count"])
new_translated = float(value["new_trans_count"])
old_units = value["old_units"]
new_units = value["new_units"]
added = len(value["added"])
# check that translation level does not go down. New strings can be added
# without an effect
if (old_translated / old_units ) > ((new_translated + added) / new_units):
all_ok = package_ok = False
msg += "%s level down from %.4f to %.4f" % (
key, old_translated/ old_units, (new_translated + added) / new_units)
# check that already present languages are not removed
if len(l10n_stats["removed_langs"]) > 0:
all_ok = package_ok = False
msg += "%s langs removed" % (", ".join(l10n_stats['removed_langs']))
if not package_ok:
wid.fields.msg.append("%(sourcepackage)s has following l10n error(s):" % action)
wid.fields.msg.append(msg)
wid.result = all_ok
class ParticipantHandler(object):
"""Participant class as defined by the SkyNET API."""
def __init__(self):
self.oscrc = None
self.obs = None
def handle_wi_control(self, ctrl):
"""Job control thread."""
pass
def handle_lifecycle_control(self, ctrl):
"""Participant control thread."""
if ctrl.message == "start":
if ctrl.config.has_option("obs", "oscrc"):
self.oscrc = ctrl.config.get("obs", "oscrc")
def setup_obs(self, namespace):
"""Setup the Buildservice instance
Using the namespace as an alias to the apiurl.
"""
self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace)
def revert_trial(self, wid):
"""Copy packages from trunk to testing.
* The revert notes the intended request destination and copies the
latest version from there (usually Trunk) back into the build_in
project.
* Later, if/when parallel build_in projects are used it may be
important to correctly sync build_in from a potentially updated
Trunk
* If build_in is a project link we remove the packages and
wipebinaries
"""
wid.result = False
rid = wid.fields.ev.id
actions = wid.fields.ev.actions
build_in = wid.params.build_in
for act in actions:
if act['type'] != 'submit':
continue
if wid.params.linked:
self.obs.deletePackage(build_in, act['targetpackage'])
# wipeBinaries errors if there are no packages to wipe
if self.obs.getPackageList(build_in):
self.obs.wipeBinaries(build_in)
else:
try:
self.obs.copyPackage(self.obs.apiurl,
act['targetproject'],
act['targetpackage'],
self.obs.apiurl,
build_in,
act['targetpackage'],
client_side_copy=False,
keep_maintainers=False,
keep_develproject=False,
expand=False,
comment="Trial revert for \
request %s" % rid)
except HTTPError, exp:
# If the package is not found in target, reverting is
# done by deleting it from build_in.
if exp.code == 404:
self.obs.deletePackage(build_in, act['targetpackage'])
else:
raise
#!/usr/bin/python
from buildservice import BuildService
from osc import conf, core
bs = BuildService(apiurl='https://api.meego.com', oscrc="/etc/oscrc")
#print bs.getRepoState('Trunk')
#print bs.getProjectDiff('Trunk:Testing', 'Trunk')
packages = bs.getPackageList('Trunk:Testing')
for src_package in packages:
print src_package
diff = core.server_diff(bs.apiurl, 'Trunk', src_package, None,
'Trunk:Testing', src_package, None, False)
p = open(src_package, 'w')
p.write(diff)
p.close()
class ParticipantHandler(object):
"""Participant class as defined by the SkyNET API."""
def __init__(self):
self.oscrc = None
self.obs = None
def handle_wi_control(self, ctrl):
"""Job control thread."""
pass
def handle_lifecycle_control(self, ctrl):
"""Participant control thread."""
if ctrl.message == "start":
if ctrl.config.has_option("obs", "oscrc"):
self.oscrc = ctrl.config.get("obs", "oscrc")
def setup_obs(self, namespace):
"""Setup the Buildservice instance
Using the namespace as an alias to the apiurl.
"""
self.obs = BuildService(oscrc=self.oscrc, apiurl=namespace)
def revert_trial(self, wid):
"""Copy packages from trunk to testing.
* The revert notes the intended request destination and copies the
latest version from there (usually Trunk) back into the build_in
project.
* Later, if/when parallel build_in projects are used it may be
important to correctly sync build_in from a potentially updated
Trunk
* If build_in is a project link we remove the packages and
wipebinaries
"""
wid.result = False
rid = wid.fields.ev.id
actions = wid.fields.ev.actions
build_in = wid.params.build_in
for act in actions:
if act['type'] != 'submit':
continue
if wid.params.linked :
self.obs.deletePackage(build_in, act['targetpackage'])
# wipeBinaries errors if there are no packages to wipe
if self.obs.getPackageList(build_in):
self.obs.wipeBinaries(build_in)
else:
try:
self.obs.copyPackage(self.obs.apiurl,
act['targetproject'],
act['targetpackage'],
self.obs.apiurl,
build_in,
act['targetpackage'],
client_side_copy = False,
keep_maintainers = False,
keep_develproject = False,
expand = False,
comment = "Trial revert for \
request %s" % rid)
except HTTPError, exp:
# If the package is not found in target, reverting is
# done by deleting it from build_in.
if exp.code == 404:
self.obs.deletePackage(build_in,
act['targetpackage'])
else:
raise
#!/usr/bin/python
from buildservice import BuildService
from osc import conf, core
bs = BuildService(apiurl = 'https://api.meego.com', oscrc = "/etc/oscrc")
#print bs.getRepoState('Trunk')
#print bs.getProjectDiff('Trunk:Testing', 'Trunk')
packages = bs.getPackageList('Trunk:Testing')
for src_package in packages:
print src_package
diff = core.server_diff(bs.apiurl,
'Trunk', src_package, None,
'Trunk:Testing', src_package, None, False)
p = open(src_package, 'w')
p.write(diff)
p.close()