def _download_kickstarts(self, project, configurations, ignore):
"""Downloads RPM and extrack .ks files."""
kickstarts = []
errors = []
rpms = set()
with Lab(prefix="get_kickstarts") as lab:
# Download binaries
for package in configurations:
for target in configurations[package]:
for binary in configurations[package][target]:
rpms.add(
self.download_binary(project, package, target,
binary, lab.path))
for rpm in rpms:
# Extract kickstart files
found = False
for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
# Read ks contents in images array
basedir, basename = os.path.split(fname)
if [True for pattern in ignore if pattern.match(basename)]:
continue
kickstarts.append({
"basedir": basedir,
"basename": basename,
"contents": lab.open(fname).read()
})
found = True
if not found:
errors.append("%s did not contain .ks files" %
os.path.basename(rpm))
return kickstarts, errors
def _download_kickstarts(self, project, configurations, ignore):
"""Downloads RPM and extrack .ks files."""
kickstarts = []
errors = []
rpms = set()
with Lab(prefix="get_kickstarts") as lab:
# Download binaries
for package in configurations:
for target in configurations[package]:
for binary in configurations[package][target]:
rpms.add(self.download_binary(project, package, target, binary, lab.path))
for rpm in rpms:
# Extract kickstart files
found = False
for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
# Read ks contents in images array
basedir, basename = os.path.split(fname)
if [True for pattern in ignore if pattern.match(basename)]:
continue
kickstarts.append({"basedir": basedir, "basename": basename, "contents": lab.open(fname).read()})
found = True
if not found:
errors.append("%s did not contain .ks files" % os.path.basename(rpm))
return kickstarts, errors
def _download_kickstarts(self, project, configurations):
"""Downloads RPM and extrack .ks files."""
rpms = set()
with Lab(prefix="get_kickstarts") as lab:
# Download binaries
for package in configurations:
for target in configurations[package]:
for binary in configurations[package][target]:
rpms.add(self.download_binary(project, package,
target, binary, lab.path))
for rpm in rpms:
# deploy dir is the name of the rpm without the versions
deploy_dir = os.path.join(self.deploy_prefix,
".%s" % os.path.basename(rpm).rsplit("-", 2)[0])
shutil.rmtree(deploy_dir, ignore_errors=True)
os.mkdir(deploy_dir)
# Extract kickstart files and copy to the deploy dir
for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
shutil.copy(os.path.join(lab.path, fname), deploy_dir)
symlink_src = os.path.join(deploy_dir, os.path.basename(fname))
symlink_dst = os.path.join(self.deploy_prefix, os.path.basename(fname))
if os.path.lexists(symlink_dst):
os.unlink(symlink_dst)
os.symlink(symlink_src, symlink_dst)
return
def update_ts(self, tsrpms, packagename, version, tmpdir):
"""Extract ts files from RPM and put them in GIT."""
workdir = os.path.join(tmpdir, packagename)
os.mkdir(workdir)
tsfiles = []
for tsbin in tsrpms:
tsfiles.extend(extract_rpm(tsbin, workdir, "*.ts"))
if len(tsfiles) == 0:
print "No ts files in '%s'. Continue..." % packagename
return
try:
projectdir = self.init_gitdir(packagename)
except CalledProcessError:
# invalidate cache and try once again
self.log.warning("Caught a git error. Removing local git repo and trying again...")
shutil.rmtree(os.path.join(self.gitconf["basedir"], packagename),
ignore_errors=True)
projectdir = self.init_gitdir(packagename)
tpldir = os.path.join(projectdir, "templates")
if not os.path.isdir(tpldir):
os.mkdir(tpldir)
for tsfile in tsfiles:
shutil.copy(os.path.join(workdir, tsfile), tpldir)
check_call(["git", "add", "*"], cwd=tpldir)
if len(check_output(["git", "diff", "--staged"],
cwd=projectdir)) == 0:
print "No updates. Exiting"
return
check_call(["git", "commit", "-m",
"%s translation templates update for %s" % ( self.gitconf['vcs_msg_prefix'], version)],
cwd=projectdir)
check_call(["git", "push", "origin", "master"], cwd=projectdir)
# auto-create/update Pootle translation projects
l10n_auth = (self.l10n_conf["username"], self.l10n_conf["password"])
data = json.dumps({"name": packagename})
resp = requests.post("%s/packages" % self.l10n_conf["apiurl"],
auth=l10n_auth,
headers={'content-type': 'application/json'},
data=data,
verify=False)
assert resp.status_code == 201
# This is a hack to make Pootle recalculate statistics
resp = requests.post("%s/packages" % self.l10n_conf["apiurl"],
auth=l10n_auth,
headers={'content-type': 'application/json'},
data=data,
verify=False)
assert resp.status_code == 201
def deploy_doc(self, docrpms, packagename, version, tmpdir, symlink=None):
"""Extract doc files from RPM and put them under docroot."""
deploydir = os.path.join(self.autodoc_conf['docroot'], packagename, version)
print deploydir
workdir = os.path.join(tmpdir, packagename)
os.mkdir(workdir)
deployed = False
for docbin in docrpms:
if len(docrpms) > 1:
deploydir = os.path.join(deploydir, os.path.basename(docbin))
print "extracting %s" % docbin
docfiles = extract_rpm(docbin, workdir)
toplevels = set()
print "walking %s" % workdir
for dirpath, dirnames, filenames in os.walk(workdir):
for fil in filenames:
if fil.endswith(".html"):
toplevels.add(dirpath)
# don't look further down
del dirnames[:]
# no need to look at other files
break
if len(toplevels) > 1:
deployed = True
for level in toplevels:
target = os.path.join(deploydir, os.path.basename(level))
shutil.rmtree(target, True)
shutil.copytree(level, target)
elif len(toplevels) == 1:
deployed = True
print deploydir
shutil.rmtree(deploydir, True)
shutil.copytree(toplevels.pop(), deploydir)
if deployed:
# fix permissions due to cpio not honoring umask
for root, dirs, files in os.walk(os.path.join(self.autodoc_conf['docroot'], packagename)):
for d in dirs:
os.chmod(os.path.join(root, d), 0755)
for f in files:
os.chmod(os.path.join(root, f), 0644)
if symlink:
symlink_name = os.path.join(self.autodoc_conf['docroot'], packagename, symlink)
print "creating symlink %s" % symlink_name
if os.path.lexists(symlink_name):
os.unlink(symlink_name)
os.symlink(version, symlink_name)
def __update_meta(self, project, providers, metatype):
"""Extracts a meta xml from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
for package, targets in providers.items():
for target, binaries in targets.items():
for binary in binaries:
with Lab(prefix=metatype) as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package,
binary, lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" % (binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path,
["*.xml"]):
meta = os.path.basename(xml)
submetatype = os.path.basename(os.path.dirname(xml))
print(meta, metatype, submetatype)
try:
with open(lab.real_path(xml), 'r') as fd:
metadata = [line.replace("@PROJECT@", project) for line in fd.readlines()]
# Update meta
if submetatype == "aggregates":
pkgname = os.path.splitext(meta)[0]
core.edit_meta(metatype='pkg', path_args=(project, pkgname), template_args=({'name': pkgname, 'user': '******'}), apiurl=self.obs.apiurl)
u = core.makeurl(self.obs.apiurl, ['source', project, pkgname, '_aggregate'])
print u
print metadata
core.http_PUT(u, data="\n".join(metadata))
else:
core.edit_meta(metatype, project, data=metadata)
uploaded.append(metatype + '/' + meta)
except HTTPError as exc:
errors.append("Failed to upload %s:\nHTTP %s %s\n%s" %
(meta, exc.code, exc.filename,
exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" %
(meta, exc))
return uploaded, errors
def __update_meta(self, project, providers, metatype):
"""Extracts a meta xml from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
for package, targets in providers.items():
for target, binaries in targets.items():
for binary in binaries:
with Lab(prefix=metatype) as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package,
binary, lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" %
(binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path,
["*.xml"]):
meta = os.path.basename(xml)
try:
with open(lab.real_path(xml), 'r') as fd:
metadata = [
line.replace("@PROJECT@", project)
for line in fd.readlines()
]
# Update meta
core.edit_meta(metatype,
project,
data=metadata)
uploaded.append(meta)
except HTTPError as exc:
errors.append(
"Failed to upload %s:\nHTTP %s %s\n%s" %
(meta, exc.code, exc.filename,
exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" %
(meta, exc))
return uploaded, errors
def __update_patterns(self, project, package, target, binary):
"""Extracts patterns from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
with Lab(prefix="update_patterns") as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package, binary,
lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" % (binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
print lab.real_path(binary)
for xml in extract_rpm(lab.real_path(binary), lab.path, ["*.xml"]):
pattern = os.path.basename(xml)
try:
# chop .xml from name
if pattern.endswith(".xml"):
pattern = pattern[:-4]
# Update pattern to project
self.obs.setProjectPattern(project,
lab.real_path(xml),
name=pattern)
uploaded.append(pattern)
except HTTPError as exc:
errors.append(
"Failed to upload %s:\nHTTP %s %s\n%s" %
(pattern, exc.code, exc.filename, exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" % (pattern, exc))
return uploaded, errors
def __update_patterns(self, project, package, target, binary):
"""Extracts patterns from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
with Lab(prefix="update_patterns") as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package, binary,
lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" % (binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path,
["*.xml"]):
pattern = os.path.basename(xml)
try:
# chop .xml from name
if pattern.endswith(".xml"):
pattern = pattern[:-4]
# Update pattern to project
self.obs.setProjectPattern(project, lab.real_path(xml), name=pattern)
uploaded.append(pattern)
except HTTPError as exc:
errors.append("Failed to upload %s:\nHTTP %s %s\n%s" %
(pattern, exc.code, exc.filename,
exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" %
(pattern, exc))
return uploaded, errors
def __update_meta(self, project, providers, metatype):
"""Extracts a meta xml from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
for package, targets in providers.items():
for target, binaries in targets.items():
for binary in binaries:
with Lab(prefix=metatype) as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package, binary, lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" % (binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" % (binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path, ["*.xml"]):
meta = os.path.basename(xml)
try:
with open(lab.real_path(xml), "r") as fd:
metadata = [line.replace("@PROJECT@", project) for line in fd.readlines()]
# Update meta
core.edit_meta(metatype, project, data=metadata)
uploaded.append(meta)
except HTTPError as exc:
errors.append(
"Failed to upload %s:\nHTTP %s %s\n%s"
% (meta, exc.code, exc.filename, exc.fp.read())
)
except Exception as exc:
errors.append("Failed to upload %s: %s" % (meta, exc))
return uploaded, errors
def _download_kickstarts(self, project, configurations):
"""Downloads RPM and extrack .ks files."""
rpms = set()
with Lab(prefix="get_kickstarts") as lab:
# Download binaries
if isinstance(project, unicode):
project = project.encode('utf8')
for package in configurations:
if isinstance(package, unicode):
package = package.encode('utf8')
for target in configurations[package]:
if isinstance(target, unicode):
target = target.encode('utf8')
for binary in configurations[package][target]:
if isinstance(binary, unicode):
binary = binary.encode('utf8')
rpms.add(self.download_binary(project, package,
target, binary, lab.path))
for rpm in rpms:
# deploy dir is the name of the rpm without the versions
deploy_dir = os.path.join(self.deploy_prefix,
".%s" % os.path.basename(rpm).rsplit("-", 2)[0])
# Both release and devel ks share the same directory
if not os.path.exists(deploy_dir):
os.mkdir(deploy_dir)
# Extract kickstart files and copy to the deploy dir
for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
shutil.copy(os.path.join(lab.path, fname), deploy_dir)
symlink_src = os.path.join(deploy_dir, os.path.basename(fname))
symlink_dst = os.path.join(self.deploy_prefix, os.path.basename(fname))
if os.path.lexists(symlink_dst):
os.unlink(symlink_dst)
os.symlink(symlink_src, symlink_dst)
return
def deploy_doc(self,
docrpms,
packagename,
version,
tmpdir,
symlink=None,
prefix=None):
"""Extract -doc- files from RPM and put them under docroot."""
deploydir = version
if prefix:
deploydir = os.path.join(prefix, version)
deploydir = os.path.join(self.autodoc_conf['docroot'], packagename,
deploydir)
print deploydir
workdir = os.path.join(tmpdir, packagename)
os.mkdir(workdir)
deployed = False
for docbin in docrpms:
if len(docrpms) > 1:
deploydir = os.path.join(deploydir, os.path.basename(docbin))
print "extracting %s" % docbin
docfiles = extract_rpm(docbin, workdir)
toplevels = set()
print "walking %s" % workdir
for dirpath, dirnames, filenames in os.walk(workdir):
for fil in filenames:
if fil.endswith(".html") or fil:
toplevels.add(dirpath)
# don't look further down
del dirnames[:]
# no need to look at other files
break
if len(toplevels) > 1:
deployed = True
for level in toplevels:
target = os.path.join(deploydir, os.path.basename(level))
shutil.rmtree(target, True)
shutil.copytree(level, target)
elif len(toplevels) == 1:
deployed = True
print deploydir
shutil.rmtree(deploydir, True)
shutil.copytree(toplevels.pop(), deploydir)
if deployed:
print "stuff was deployed"
# fix permissions due to cpio no honoring umask
for root, dirs, files in os.walk(
os.path.join(self.autodoc_conf['docroot'], packagename)):
for d in dirs:
print "fixing permission for %s" % d
os.chmod(os.path.join(root, d), 0755)
for f in files:
os.chmod(os.path.join(root, f), 0644)
if symlink:
symlink_name = symlink
if prefix:
symlink_name = os.path.join(prefix, symlink)
symlink_name = os.path.join(self.autodoc_conf['docroot'],
packagename, symlink_name)
print "creating symlink %s" % symlink_name
if os.path.lexists(symlink_name):
os.unlink(symlink_name)
os.symlink(version, symlink_name)
with open("%s.id" % symlink_name, 'w') as symid:
symid.write(version)
def __update_meta(self, project, providers, metatype):
"""Extracts a meta xml from rpm and uploads them to project.
:returns: uploaded pattern names and error messages
:rtype: tuple(list, list)
"""
uploaded = []
errors = []
for package, targets in providers.items():
for target, binaries in targets.items():
for binary in binaries:
with Lab(prefix=metatype) as lab:
# Download the rpm
try:
self.obs.getBinary(project, target, package,
binary, lab.real_path(binary))
except HTTPError as exc:
errors.append("Failed to download %s: HTTP %s %s" %
(binary, exc.code, exc.filename))
except Exception as exc:
errors.append("Failed to download %s: %s" %
(binary, exc))
if errors:
return uploaded, errors
# Extract pattern (xml) files from the rpm
for xml in extract_rpm(lab.real_path(binary), lab.path,
["*.xml"]):
meta = os.path.basename(xml)
submetatype = os.path.basename(
os.path.dirname(xml))
print(meta, metatype, submetatype)
try:
with open(lab.real_path(xml), 'r') as fd:
metadata = [
line.replace("@PROJECT@", project)
for line in fd.readlines()
]
# Update meta
if submetatype == "aggregates":
pkgname = os.path.splitext(meta)[0]
core.edit_meta(metatype='pkg',
path_args=(project,
pkgname),
template_args=({
'name':
pkgname,
'user':
'******'
}),
apiurl=self.obs.apiurl)
u = core.makeurl(self.obs.apiurl, [
'source', project, pkgname,
'_aggregate'
])
print u
print metadata
core.http_PUT(u, data="\n".join(metadata))
else:
core.edit_meta(metatype,
project,
data=metadata)
uploaded.append(metatype + '/' + meta)
except HTTPError as exc:
errors.append(
"Failed to upload %s:\nHTTP %s %s\n%s" %
(meta, exc.code, exc.filename,
exc.fp.read()))
except Exception as exc:
errors.append("Failed to upload %s: %s" %
(meta, exc))
return uploaded, errors
def get_l10n_stats(self, source_project, target_project, package):
tmp_dir_old = mkdtemp()
tmp_dir_new = mkdtemp()
old_ts_dir = tmp_dir_old + "/ts"
new_ts_dir = tmp_dir_new + "/ts"
target = self.obs.getTargets(str(source_project))[0]
#get src.rpm as it contains all .ts files
src_rpm = [
rpm
for rpm in self.obs.getBinaryList(source_project, target, package)
if "src.rpm" in rpm
]
target_rpm = [
rpm
for rpm in self.obs.getBinaryList(target_project, target, package)
if "src.rpm" in rpm
]
#download source and target rpms
self.obs.getBinary(target_project, target, package, target_rpm[0],
tmp_dir_old + "/old.rpm")
self.obs.getBinary(source_project, target, package, src_rpm[0],
tmp_dir_new + "/new.rpm")
# extract rpms
old_file = extract_rpm(tmp_dir_old + "/old.rpm", tmp_dir_old)
new_file = extract_rpm(tmp_dir_new + "/new.rpm", tmp_dir_new)
#rpm contains tar.bz2 and .spec file. Open and extract tar.bz2
old_tar = tarfile.open(tmp_dir_old + '/' + old_file[0])
old_tar.extractall(old_ts_dir)
new_tar = tarfile.open(tmp_dir_new + '/' + new_file[0])
new_tar.extractall(new_ts_dir)
old_ts_files = {}
for member in old_tar.members:
# rpm directrory has .spec file
if member.name.split('/')[1] == 'rpm':
continue
# "lang : path_to_ts_file" pair
old_ts_files.update({member.name.split('/')[1]: member.name})
new_ts_files = {}
for member in new_tar.members:
# rpm directrory has .spec file
if member.name.split('/')[1] == 'rpm':
continue
# "lang : path_to_ts_file" pair
new_ts_files.update({member.name.split('/')[1]: member.name})
l10n_stats = {}
for key in set(new_ts_files.keys()) & set(old_ts_files.keys()):
_old_path = tmp_dir_old + "/ts/" + old_ts_files[key]
_new_path = tmp_dir_new + "/ts/" + new_ts_files[key]
unit_diff = _make_ts_diff(_old_path, _new_path)
l10n_stats.update({key: unit_diff})
l10n_stats.update({
"removed_langs":
list(set(old_ts_files.keys()) - set(new_ts_files.keys()))
})
l10n_stats.update({
"added_langs":
list(set(new_ts_files.keys()) - set(old_ts_files.keys()))
})
# possible removed strings
l10n_stats.update({"removed_strings": []})
#check that -ts-devel package is not going out of sync
src_pkg = package.replace("-l10n", "")
#is there a package that is using -l10n pakcage already
src_pkg = [
rpm for rpm in self.obs.getPackageList(target_project)
if src_pkg == rpm
]
if len(src_pkg) > 0:
#get -ts-devel rpm
src_ts_devel_rpm = [
rpm for rpm in self.obs.getBinaryList(
target_project, target, src_pkg[0]) if "-ts-devel" in rpm
]
if len(src_ts_devel_rpm) > 0:
tmp_dir_ts = mkdtemp()
self.obs.getBinary(target_project, target, src_pkg[0],
src_ts_devel_rpm[0],
tmp_dir_ts + "/orig.rpm")
orig_ts_file = extract_rpm(tmp_dir_ts + "/orig.rpm",
tmp_dir_ts,
patterns="*.ts")
original_units = factory.getobject(tmp_dir_ts + "/" +
orig_ts_file[0])
new_units = factory.getobject(tmp_dir_new + "/ts/" +
new_ts_files['templates'])
removed_units = set(original_units.getids()) - set(
new_units.getids())
l10n_stats.update({"removed_strings": list(removed_units)})
shutil.rmtree(tmp_dir_ts)
#get rid of tmp dirs
shutil.rmtree(tmp_dir_old)
shutil.rmtree(tmp_dir_new)
return l10n_stats
project = project.encode('utf8')
for package in configurations:
if isinstance(package, unicode):
package = package.encode('utf8')
for target in configurations[package]:
if isinstance(target, unicode):
target = target.encode('utf8')
for binary in configurations[package][target]:
if isinstance(binary, unicode):
binary = binary.encode('utf8')
rpms.add(self.download_binary(project, package,
target, binary, lab.path))
for rpm in rpms:
# deploy dir is the name of the rpm without the versions
deploy_dir = os.path.join(self.deploy_prefix,
".%s" % os.path.basename(rpm).rsplit("-", 2)[0])
# Both release and devel ks share the same directory
if not os.path.exists(deploy_dir):
- os.mkdir(deploy_dir)
# Extract kickstart files and copy to the deploy dir
for fname in extract_rpm(rpm, lab.path, patterns=["*.ks"]):
shutil.copy(os.path.join(lab.path, fname), deploy_dir)
symlink_src = os.path.join(deploy_dir, os.path.basename(fname))
symlink_dst = os.path.join(self.deploy_prefix, os.path.basename(fname))
if os.path.lexists(symlink_dst):
os.unlink(symlink_dst)
os.symlink(symlink_src, symlink_dst)
return
def get_l10n_stats(self, source_project, target_project, package):
tmp_dir_old = mkdtemp()
tmp_dir_new = mkdtemp()
old_ts_dir = os.path.join(tmp_dir_old, "ts")
new_ts_dir = os.path.join(tmp_dir_new, "ts")
target = self.obs.getTargets(str(source_project))[0]
# Get src.rpm as it contains all .ts files
src_rpm = [rpm for rpm in self.obs.getBinaryList(
source_project, target, package) if "src.rpm" in rpm]
target_rpm = [rpm for rpm in self.obs.getBinaryList(
target_project, target, package) if "src.rpm" in rpm]
# Download source and target rpms
old_src_rpm = os.path.join(tmp_dir_old, target_rpm[0])
self.obs.getBinary(target_project, target, package, target_rpm[0],
old_src_rpm)
new_src_rpm = os.path.join(tmp_dir_new, src_rpm[0])
self.obs.getBinary(source_project, target, package, src_rpm[0],
new_src_rpm)
# Extract rpms and get the source tarball names
old_tar = next(
f for f in extract_rpm(old_src_rpm, tmp_dir_old)
if '.tar' in f
)
new_tar = next(
f for f in extract_rpm(new_src_rpm, tmp_dir_new)
if '.tar' in f
)
# Extract tarballs and get ts files per language
old_tar = os.path.join(tmp_dir_old, old_tar)
new_tar = os.path.join(tmp_dir_new, new_tar)
old_ts_files = _get_ts_files(_extract_tar(old_tar, old_ts_dir))
new_ts_files = _get_ts_files(_extract_tar(new_tar, new_ts_dir))
old_langs = set(old_ts_files.keys())
new_langs = set(new_ts_files.keys())
l10n_stats = {
"removed_langs": list(old_langs - new_langs),
"added_langs": list(new_langs - old_langs),
"removed_strings": [],
"languages": {},
}
for key in new_langs & old_langs:
_old_path = os.path.join(old_ts_dir, old_ts_files[key])
_new_path = os.path.join(new_ts_dir, new_ts_files[key])
unit_diff = _make_ts_diff(_old_path, _new_path)
l10n_stats['languages'][key] = unit_diff
# Check that -ts-devel package is not going out of sync
src_pkg = package.replace("-l10n", "")
# Is there a package that is using -l10n pakcage already
if src_pkg in self.obs.getPackageList(target_project):
# get -ts-devel rpm
src_ts_devel_rpm = next((
rpm for rpm in
self.obs.getBinaryList(target_project, target, src_pkg)
if "-ts-devel" in rpm),
None
)
if src_ts_devel_rpm:
tmp_dir_ts = mkdtemp()
tmp_src_ts_devel_rpm = os.path.join(
tmp_dir_ts, src_ts_devel_rpm)
self.obs.getBinary(
target_project, target, src_pkg, src_ts_devel_rpm,
tmp_src_ts_devel_rpm)
orig_ts_file = extract_rpm(
tmp_src_ts_devel_rpm, tmp_dir_ts, patterns="*.ts")
original_units = factory.getobject(
os.path.join(tmp_dir_ts, orig_ts_file[0]))
new_units = factory.getobject(
os.path.join(tmp_dir_new, "ts", new_ts_files['templates']))
l10n_stats["removed_strings"] = list(
set(original_units.getids()) - set(new_units.getids())
)
shutil.rmtree(tmp_dir_ts)
# get rid of tmp dirs
shutil.rmtree(tmp_dir_old)
shutil.rmtree(tmp_dir_new)
return l10n_stats
def get_l10n_stats(self, source_project, target_project, package):
tmp_dir_old = mkdtemp()
tmp_dir_new = mkdtemp()
old_ts_dir = tmp_dir_old + "/ts"
new_ts_dir = tmp_dir_new + "/ts"
target = self.obs.getTargets(str(source_project))[0]
#get src.rpm as it contains all .ts files
src_rpm = [rpm for rpm in self.obs.getBinaryList(
source_project, target, package) if "src.rpm" in rpm]
target_rpm = [rpm for rpm in self.obs.getBinaryList(
target_project, target, package) if "src.rpm" in rpm]
#download source and target rpms
self.obs.getBinary(target_project, target, package, target_rpm[0],
tmp_dir_old + "/old.rpm")
self.obs.getBinary(source_project, target, package, src_rpm[0],
tmp_dir_new + "/new.rpm")
# extract rpms
old_file = extract_rpm(tmp_dir_old + "/old.rpm", tmp_dir_old)
new_file = extract_rpm(tmp_dir_new + "/new.rpm", tmp_dir_new)
#rpm contains tar.bz2 and .spec file. Open and extract tar.bz2
old_tar = tarfile.open(tmp_dir_old + '/' + old_file[0])
old_tar.extractall(old_ts_dir)
new_tar = tarfile.open(tmp_dir_new + '/' + new_file[0])
new_tar.extractall(new_ts_dir)
old_ts_files = {}
for member in old_tar.members:
# rpm directrory has .spec file
if member.name.split('/')[1] == 'rpm':
continue
# "lang : path_to_ts_file" pair
old_ts_files.update({member.name.split('/')[1] : member.name })
new_ts_files = {}
for member in new_tar.members:
# rpm directrory has .spec file
if member.name.split('/')[1] == 'rpm':
continue
# "lang : path_to_ts_file" pair
new_ts_files.update({member.name.split('/')[1] : member.name })
l10n_stats = {}
for key in set(new_ts_files.keys()) & set(old_ts_files.keys()):
_old_path = tmp_dir_old + "/ts/" + old_ts_files[key]
_new_path = tmp_dir_new + "/ts/" + new_ts_files[key]
unit_diff = _make_ts_diff(_old_path, _new_path)
l10n_stats.update({ key : unit_diff })
l10n_stats.update({"removed_langs" : list(set(old_ts_files.keys()) - set(new_ts_files.keys())) })
l10n_stats.update({"added_langs" : list(set(new_ts_files.keys()) - set(old_ts_files.keys())) })
# possible removed strings
l10n_stats.update({ "removed_strings" : [] })
#check that -ts-devel package is not going out of sync
src_pkg = package.replace("-l10n", "")
#is there a package that is using -l10n pakcage already
src_pkg = [rpm for rpm in self.obs.getPackageList(target_project) if src_pkg == rpm]
if len(src_pkg) > 0:
#get -ts-devel rpm
src_ts_devel_rpm = [rpm for rpm in self.obs.getBinaryList(target_project, target, src_pkg[0]) if "-ts-devel" in rpm]
if len(src_ts_devel_rpm) > 0:
tmp_dir_ts = mkdtemp()
self.obs.getBinary(target_project, target, src_pkg[0], src_ts_devel_rpm[0], tmp_dir_ts + "/orig.rpm")
orig_ts_file = extract_rpm(tmp_dir_ts + "/orig.rpm", tmp_dir_ts, patterns="*.ts")
original_units = factory.getobject(tmp_dir_ts + "/" + orig_ts_file[0])
new_units = factory.getobject(tmp_dir_new + "/ts/" + new_ts_files['templates'])
removed_units = set(original_units.getids()) - set(new_units.getids())
l10n_stats.update({"removed_strings" : list(removed_units)})
shutil.rmtree(tmp_dir_ts)
#get rid of tmp dirs
shutil.rmtree(tmp_dir_old)
shutil.rmtree(tmp_dir_new)
return l10n_stats
