def tag_branch_mapping(package):
package_manager = PackagesManager()
release_manager = ReleaseBranchManager()
return_value = OrderedDict()
try:
package_details = package_manager.get_packages([package]).get()
except:
# log event, passing for now
pass
else:
branch_mapping = {}
if package_details.release_branch_mapping_json:
branch_mapping = package_details.release_branch_mapping_json.copy()
for k, v in package_details.release_branch_mapping_json.items():
branch_mapping[k]['product'] = \
release_manager.get_product_by_release(k).product_slug
return_value.update({
'package_name':
package_details.package_name,
'branch_mapping':
branch_mapping
if branch_mapping else package_details.release_branch_mapping_json,
'mapping_lastupdated':
package_details.release_branch_map_last_updated,
'mapping_keys':
BRANCH_MAPPING_KEYS
})
return return_value
def tag_job_form(template_type):
return_value = OrderedDict()
job_template_manager = JobTemplateManager()
filter_kwargs = {}
if template_type in TS_JOB_TYPES:
filter_kwargs['job_template_type'] = template_type
templates = job_template_manager.get_job_templates(**filter_kwargs)
if templates and len(templates) > 0:
if len(templates) == 1:
return_value['job_template'] = templates[0]
return_value['yml_file'] = yaml.dump(
templates[0].job_template_json,
default_flow_style=False).replace("\'", "")
return_value['job_params'] = templates[0].job_template_params
return_value['job_templates'] = templates.values()
package_manager = PackagesManager()
release_streams = \
package_manager.get_release_streams(
only_active=True, fields=('product_build_system',)
)
available_build_systems = {}
for relstream in release_streams:
available_build_systems.update(
{relstream.product_slug: relstream.product_build_system})
if available_build_systems:
return_value['build_systems'] = available_build_systems
packages = package_manager.get_package_name_tuple(check_mapping=True)
if packages:
return_value['packages'] = packages
relbranch_manager = ReleaseBranchManager()
release_branches = relbranch_manager.get_relbranch_name_slug_tuple()
if release_branches:
return_value['releases'] = release_branches
return return_value
def __init__(self):
"""
entry point
"""
super(ReleaseScheduleSyncManager, self).__init__()
self.job_manager = JobManager(TS_JOB_TYPES[1])
self.release_branch_manager = ReleaseBranchManager()
def __init__(self, *args, **kwargs):
"""
entry point
"""
super(BuildTagsSyncManager, self).__init__(self, *args, **kwargs)
self.job_manager = JobManager(TS_JOB_TYPES[4])
self.release_branch_manager = ReleaseBranchManager()
def tag_sync_from_coverage(stats, package, release, tag):
return_value = OrderedDict()
if not isinstance(stats, str):
return return_value
if isinstance(stats, str) and not stats.startswith('Not Synced with'):
return return_value
package_manager = PackagesManager()
release_manager = ReleaseBranchManager()
try:
package_details = package_manager.get_packages([package]).get()
except:
# log event, passing for now
pass
else:
branch_mapping = {}
if package_details.release_branch_mapping_json:
branch_mapping = package_details.release_branch_mapping_json.copy()
if release in branch_mapping:
branch_mapping = branch_mapping.get(release)
branch_mapping['product'] = \
release_manager.get_product_by_release(release).product_slug
return_value.update(
dict(
mapping=branch_mapping,
package=package,
tag=tag,
))
return return_value
def tag_job_form(template_type):
return_value = OrderedDict()
job_template_manager = JobTemplateManager()
filter_kwargs = {}
if template_type in TS_JOB_TYPES:
filter_kwargs['job_template_type'] = template_type
templates = job_template_manager.get_job_templates(**filter_kwargs)
if templates and len(templates) > 0:
if len(templates) == 1:
return_value['job_template'] = templates[0]
return_value['yml_file'] = yaml.dump(
templates[0].job_template_json, default_flow_style=False
).replace("\'", "")
return_value['job_params'] = templates[0].job_template_params
return_value['job_templates'] = templates.values()
package_manager = PackagesManager()
release_streams = \
package_manager.get_release_streams(
only_active=True, fields=('product_build_system',)
)
available_build_systems = []
for relstream in release_streams:
available_build_systems.append(relstream.product_build_system)
if available_build_systems:
return_value['build_systems'] = available_build_systems
packages = package_manager.get_package_name_tuple(check_mapping=True)
if packages:
return_value['packages'] = packages
relbranch_manager = ReleaseBranchManager()
release_branches = relbranch_manager.get_relbranch_name_slug_tuple()
if release_branches:
return_value['releases'] = release_branches
return return_value
def tag_release_map_view():
return_value = OrderedDict()
release_manager = ReleaseBranchManager()
latest_release = release_manager.get_latest_release()
geo_location_manager = GeoLocationManager()
territory_stats = \
geo_location_manager.get_territory_build_system_stats()
if territory_stats:
return_value["territory_stats"] = territory_stats
if territory_stats and latest_release:
return_value["latest_release"] = latest_release
return return_value
class ManagersMixin(object):
"""
Managers Mixin
"""
inventory_manager = InventoryManager()
packages_manager = PackagesManager()
jobs_log_manager = JobsLogManager()
release_branch_manager = ReleaseBranchManager()
graph_manager = GraphManager()
def get_summary(self):
"""
Application Inventory Stats
"""
locales_set = self.inventory_manager.get_locales_set()
summary = {}
summary['locales_len'] = len(locales_set[0]) \
if isinstance(locales_set, tuple) else 0
platforms = self.inventory_manager.get_transplatform_slug_url()
summary['platforms_len'] = len(platforms) if platforms else 0
relstreams = self.inventory_manager.get_relstream_slug_name()
summary['products_len'] = len(relstreams) if relstreams else 0
relbranches = self.release_branch_manager.get_release_branches()
summary['releases_len'] = relbranches.count() if relbranches else 0
summary['packages_len'] = self.packages_manager.count_packages()
jobs_count, last_ran_on, last_ran_type = \
self.jobs_log_manager.get_joblog_stats()
summary['jobs_len'] = jobs_count
graph_rules = self.graph_manager.get_graph_rules(only_active=True)
summary['graph_rules_len'] = graph_rules.count() if graph_rules else 0
return summary
def __init__(self, *args, **kwargs):
"""
entry point
"""
super(BuildTagsSyncManager, self).__init__(self, *args, **kwargs)
self.job_manager = JobManager(TS_JOB_TYPES[4])
self.release_branch_manager = ReleaseBranchManager()
class ManagersMixin(object):
"""
Managers Mixin
"""
inventory_manager = InventoryManager()
packages_manager = PackagesManager()
jobs_log_manager = JobsLogManager()
release_branch_manager = ReleaseBranchManager()
graph_manager = GraphManager()
def tag_trending_languages():
return_value = OrderedDict()
reports_manager = ReportsManager()
releases_summary = reports_manager.get_reports('releases')
release_manager = ReleaseBranchManager()
latest_release = release_manager.get_latest_release()
pkg_manager = PackagesManager()
lang_locale_dict = {
lang: locale
for locale, lang in pkg_manager.get_locale_lang_tuple()
}
if releases_summary and latest_release:
releases_summary = releases_summary.get()
trending_languages = reports_manager.get_trending_languages(
releases_summary.report_json, *latest_release)
if trending_languages and isinstance(trending_languages,
(list, tuple)):
return_value["trending_languages"] = trending_languages[:9]
return_value["lang_locale_dict"] = lang_locale_dict
return_value["latest_release"] = latest_release
return return_value
class PackageBranchMapping(object):
"""
Creates Branch Mapping
"""
package = None
package_name = None
original_versions = None
transplatform_versions = None
release_branches_dict = None
release_branches_list = None
release_streams_list = None
release_build_tags_dict = None
syncstats_manager = SyncStatsManager()
relbranch_manager = ReleaseBranchManager()
def __init__(self, package_name):
self.package_name = package_name
try:
self.package = Package.objects.filter(
package_name=self.package_name).first()
except Exception as e:
# log event, passing for now
pass
else:
pkg_stats = self.syncstats_manager.get_sync_stats(
pkgs=[package_name], fields=('project_version', ))
self.original_versions = [
pkg_stat.project_version for pkg_stat in pkg_stats
]
self.transplatform_versions = [
pkg_stat.project_version.lower()
for pkg_stat in self.syncstats_manager.get_sync_stats(
pkgs=[package_name],
fields=('project_version', ),
sources=TRANSPLATFORM_ENGINES)
if isinstance(pkg_stat.project_version, str)
]
self.release_branches_dict = self.relbranch_manager.get_branches_of_relstreams(
self.package.products, track_trans_flag=True)
self.release_branches_list = []
self.release_streams_list = []
self.release_build_tags_dict = {}
[
self.release_branches_list.extend(branches)
for _, branches in self.release_branches_dict.items()
]
[
self.release_streams_list.extend([stream])
for stream, _ in self.release_branches_dict.items()
if stream not in self.release_streams_list
]
for release_stream in self.release_streams_list:
build_tags = self.relbranch_manager.get_relstream_build_tags(
stream_slug=release_stream)
if build_tags:
self.release_build_tags_dict.update(build_tags)
def _get_stream_branch_belongs_to(self, branch):
"""
Get stream to which a branch belongs to
"""
for stream, branches in self.release_branches_dict.items():
if branch in branches:
return stream
return ''
def _sort_and_match_version_nm(self, release_branch, from_branches):
"""
Sort versions matching stream and then try to match version number
"""
belonging_stream = self._get_stream_branch_belongs_to(
release_branch).lower()
relevant_versions = [
version for version in from_branches if belonging_stream in version
]
version_numbers = re.findall(r'\d+', release_branch)
if len(version_numbers) >= 1:
expected_version = [
relevant_version for relevant_version in relevant_versions
if version_numbers[0] in relevant_version
]
if len(expected_version) >= 1:
return True, expected_version[0]
return False, ''
def _check_release_version(self, branch, versions, shortform=None):
"""
Match version numbers
"""
short_form = shortform if shortform else ''
version_numbers = re.findall(r'\d+', branch)
if len(version_numbers) >= 1:
short_form += str(version_numbers[0])
return [version for version in versions if short_form in version]
def _compare_with_short_names(self, release_branch, from_branches):
"""
Try short forms combined with version number
"""
belonging_stream = self._get_stream_branch_belongs_to(
release_branch).lower()
short_form = belonging_stream
if belonging_stream == RELSTREAM_SLUGS[1]:
short_form = 'f'
expected_version = self._check_release_version(release_branch,
from_branches,
shortform=short_form)
if len(expected_version) >= 1:
return True, expected_version[0]
return False, ''
def _return_original_version(self, matched_version):
required_version = [
version for version in self.original_versions
if version.lower() == matched_version
]
return required_version[0] if len(
required_version) == 1 else matched_version
def calculate_branch_mapping(self, branch, from_branches):
"""
Calculates branch mapping for a branch
Algorithm:
1. try with difflib, check version and return first found
2. sort versions matching stream and then try to match version number
3. try short forms combined with version number
4. todo: fetch release dates and try matching with that
5. try finding default version: 'master' may be
6. if nothing works, return blank
"""
match1 = difflib.get_close_matches(branch, from_branches)
if len(match1) >= 1:
match_found = self._check_release_version(branch, match1)
if len(match_found) >= 1:
return self._return_original_version(match_found[0])
status1, match2 = self._sort_and_match_version_nm(
branch, from_branches)
if status1:
return self._return_original_version(match2)
status2, match3 = self._compare_with_short_names(branch, from_branches)
if status2:
return self._return_original_version(match3)
probable_branches = [
'default', 'master', 'head', 'rawhide', 'devel', 'core',
'translations', 'programs', self.package_name
]
for version in probable_branches:
if version in from_branches:
return self._return_original_version(version)
for version in from_branches:
closest_ver = difflib.get_close_matches(version, probable_branches)
if isinstance(closest_ver, list) and len(closest_ver) > 0:
return self._return_original_version(version)
for version in from_branches:
if self.package_name in version:
return self._return_original_version(version)
return ''
@property
def versions(self):
return self.transplatform_versions
@property
def release_branches(self):
return self.release_branches_list
@property
def branch_mapping(self):
"""
Creates branch mapping based on rule
:return: dict
"""
branch_mapping_dict = {}
required_params = (self.transplatform_versions,
self.release_branches_dict,
self.release_streams_list,
self.release_build_tags_dict)
valid_params = [param for param in required_params if param]
if len(required_params) == len(valid_params):
for stream, branches in self.release_branches_dict.items():
for branch in branches:
branch_mapping_dict[branch] = {}
branch_mapping_dict[branch][BRANCH_MAPPING_KEYS[0]] = \
self.calculate_branch_mapping(branch, self.transplatform_versions)
branch_mapping_dict[branch][BRANCH_MAPPING_KEYS[1]] = \
self.relbranch_manager.get_relstream_buildsys(stream)
branch_mapping_dict[branch][BRANCH_MAPPING_KEYS[2]] = \
self.calculate_branch_mapping(branch, sorted(
self.release_build_tags_dict[stream]))
if not branch_mapping_dict[branch][BRANCH_MAPPING_KEYS[0]] \
and self.package.platform_slug_id in DAMNEDLIES_SLUGS:
release_stream = self.relbranch_manager.get_release_streams(
stream_slug=stream,
built=branch_mapping_dict[branch][
BRANCH_MAPPING_KEYS[1]],
fields=('product_server', ))
if release_stream:
release_stream_hub_url = release_stream.get(
).product_server or ''
build_info = self.relbranch_manager.api_resources.build_info(
hub_url=release_stream_hub_url,
tag=branch_mapping_dict[branch][
BRANCH_MAPPING_KEYS[2]],
pkg=self.package_name)
if build_info and isinstance(
build_info, list) and len(build_info) > 0:
version_from_latest_build = build_info[0].get(
'version')
seek_version = "-".join(
version_from_latest_build.split('.')[0:2])
for version in self.transplatform_versions:
if seek_version in version:
branch_mapping_dict[branch][
BRANCH_MAPPING_KEYS[0]] = version
# seek next (nearest) version
if not branch_mapping_dict[branch][
BRANCH_MAPPING_KEYS[0]]:
version_x, version_y = version_from_latest_build.split(
'.')[0:2]
first_place_matched_versions = \
[version for version in self.transplatform_versions
if version_x in version]
if first_place_matched_versions and len(
first_place_matched_versions) > 0:
probable_versions = [
int(version.split('-')[0:3][2])
for version in
first_place_matched_versions
]
version_y = int(version_y)
located_version = max(
probable_versions,
key=lambda x: abs(x + version_y))
if located_version:
required_version = [
ver for ver in
first_place_matched_versions
if str(located_version) in ver
]
if required_version and len(
required_version) > 0:
branch_mapping_dict[branch][
BRANCH_MAPPING_KEYS[
0]] = required_version[
0]
return branch_mapping_dict
class PackagesManager(InventoryManager):
"""
Packages Manager
"""
PROCESS_STATS = True
syncstats_manager = SyncStatsManager()
release_manager = ReleaseBranchManager()
def get_packages(self, pkgs=None, pkg_params=None):
"""
fetch packages from db
"""
packages = None
fields = pkg_params if isinstance(pkg_params, (list, tuple)) else []
kwargs = {}
if pkgs:
kwargs.update(dict(package_name__in=pkgs))
try:
packages = Package.objects.only(*fields).filter(**kwargs) \
.order_by('-platform_last_updated')
except Exception as e:
self.app_logger(
'ERROR', "Packages could not be fetched, details: " + str(e))
return packages
def is_package_exist(self, package_name):
"""
check package existence
"""
if self.get_packages(pkgs=[package_name]):
return True
return False
def update_package(self, package_name, fields):
"""
Update a package with certain fields
"""
try:
Package.objects.filter(package_name=package_name).update(**fields)
except Exception as e:
self.app_logger('ERROR',
"Package could not be updated, details: " + str(e))
def get_relbranch_specific_pkgs(self, release_branch, fields=None):
"""
fetch release branch specific packages from db
"""
packages = ()
fields_required = fields if fields else ()
try:
packages = Package.objects.only(*fields_required) \
.filter(release_branch_mapping__icontains=release_branch) \
.order_by('-platform_last_updated')
except Exception as e:
self.app_logger(
'ERROR',
("release branch specific package could not be fetched for " +
release_branch + " release branch, details: " + str(e)))
return packages
def get_release_specific_package_stats(self, release_branch):
"""
Fetch processed stats for all packages of a release
:param release_branch: str
:return: package stats: dict
"""
packages_stats = {}
release_packages = self.get_relbranch_specific_pkgs(
release_branch, fields=['package_name', 'release_branch_mapping'])
branch_locales = self.get_relbranch_locales(release_branch)
locales_count_match = True \
if len(branch_locales) == self.get_active_locales_count() else False
for package in release_packages:
version = package.release_branch_mapping_json[release_branch][
BRANCH_MAPPING_KEYS[0]]
package_stats = self.syncstats_manager.get_sync_stats(
pkgs=[package.package_name],
fields=['stats_raw_json_str', 'stats_processed_json_str'],
versions=[version]).first()
if package_stats:
processed_stats = {}
if not package_stats.stats_processed_json:
if package_stats.stats_raw_json.get('stats'):
processed_stats = self._process_response_stats_json(
package_stats.stats_raw_json['stats'])
if package.platform_slug.engine_name == TRANSPLATFORM_ENGINES[
1] and not processed_stats:
processed_stats = self._process_response_stats_json_tx(
package_stats.stats_raw_json)
else:
processed_stats = package_stats.stats_processed_json
package_processed_stats = processed_stats
packages_stats[package.package_name] = package_processed_stats \
if locales_count_match else {locale: package_processed_stats.get(locale, {})
for locale in branch_locales}
return packages_stats
def get_trans_stats_by_rule(self, coverage_rule):
"""
Get translation stats by rule args: release, packages, locales, tags
:param coverage_rule: coverage rule query object
:return: stats dict
"""
if not coverage_rule:
return
packages = self.get_packages(pkgs=coverage_rule.rule_packages)
locales = coverage_rule.rule_languages
tags = coverage_rule.rule_build_tags
release = coverage_rule.rule_release_slug_id
locale_lang_dict = dict(self.get_locale_lang_tuple(locales))
trans_stats_by_rule = {}
for package in packages:
versions = []
versions.extend(tags)
trans_stats_by_rule[package.package_name] = {
"translation_platform": {},
"build_system": {
tag: {
"Statistics":
"Not Synced with Build System for {0}".format(tag)
}
for tag in tags
}
}
if package.release_branch_mapping_json and package.release_branch_mapping_json.get(
release):
branch_map = package.release_branch_mapping_json
platform_version = branch_map[release].get(
BRANCH_MAPPING_KEYS[0])
build_system_version = branch_map[release].get(
BRANCH_MAPPING_KEYS[2])
package_stats = self.syncstats_manager.get_sync_stats(
pkgs=[package.package_name],
fields=[
'stats_raw_json_str', 'stats_processed_json_str',
'project_version', 'source'
])
for p_stats in package_stats:
processed_stats = {}
not_found = {
'Total': 'Not Found',
'Translated': 'Not Found',
'Untranslated': 'Not Found',
'Remaining': 'N/A'
}
if not p_stats.stats_processed_json:
if p_stats.stats_raw_json.get('stats'):
processed_stats = self._process_response_stats_json(
p_stats.stats_raw_json['stats'])
if p_stats.source == TRANSPLATFORM_ENGINES[
1] and not processed_stats:
processed_stats = self._process_response_stats_json_tx(
p_stats.stats_raw_json)
else:
processed_stats = p_stats.stats_processed_json
package_processed_stats = {
locale_lang_dict.get(locale, locale):
processed_stats.get(locale, not_found)
for locale in locales
}
if p_stats.project_version == platform_version:
trans_stats_by_rule[package.package_name][
"translation_platform"].update(
package_processed_stats)
elif p_stats.project_version == "{0} - {1}".format(p_stats.source, build_system_version) and \
build_system_version in tags:
trans_stats_by_rule[package.package_name]["build_system"][build_system_version] = \
package_processed_stats
ext_tags = [
tag for tag in tags if tag in p_stats.project_version
]
if ext_tags and len(ext_tags) == 1:
trans_stats_by_rule[package.package_name]["build_system"][ext_tags[0]] = \
package_processed_stats
if not trans_stats_by_rule[
package.package_name]["translation_platform"]:
tp_not_sync_msg = "Not Synced with Translation Platform for {0}".format(platform_version) \
if platform_version else "Not Synced with Translation Platform"
trans_stats_by_rule[
package.package_name]["translation_platform"].update(
{"Translation Stats": tp_not_sync_msg})
return trans_stats_by_rule
def count_packages(self):
"""
packages count
"""
packages = self.get_packages()
return packages.count() if packages else 0
def get_package_name_tuple(self, t_status=False, check_mapping=False):
"""
returns (package_name, upstream_name) tuple (only sync'd ones)
"""
packages = self.get_packages()
name_list = [(package.package_name, package.upstream_name)
for package in packages]
if t_status:
name_list = [(package.package_name, package.upstream_name)
for package in packages
if package.platform_last_updated
or package.upstream_last_updated]
elif check_mapping:
name_list = [(package.package_name, package.package_name)
for package in packages
if package.platform_last_updated
and package.release_branch_mapping]
return tuple(sorted(name_list))
def get_project_details(self, transplatform, package_name):
"""
Get platform-wise project details
"""
resp_dict = None
platform_url = None
if transplatform.engine_name == TRANSPLATFORM_ENGINES[0]:
platform_url = transplatform.api_url + "/module/" + package_name + "/"
resp_dict = self.api_resources.fetch_project_details(
transplatform.engine_name, transplatform.api_url, package_name)
elif transplatform.engine_name == TRANSPLATFORM_ENGINES[1]:
resp_dict = self.api_resources.fetch_project_details(
transplatform.engine_name, transplatform.api_url, package_name,
**dict(ext=True,
auth_user=transplatform.auth_login_id,
auth_token=transplatform.auth_token_key))
if resp_dict:
tx_org_slug = resp_dict['organization']['slug']
platform_url = transplatform.api_url + "/" + tx_org_slug + "/" + package_name
elif transplatform.engine_name == TRANSPLATFORM_ENGINES[2]:
platform_url = transplatform.api_url + "/project/view/" + package_name
resp_dict = self.api_resources.fetch_project_details(
transplatform.engine_name, transplatform.api_url, package_name,
**dict(auth_user=transplatform.auth_login_id,
auth_token=transplatform.auth_token_key))
elif transplatform.engine_name == TRANSPLATFORM_ENGINES[3]:
resp_dict = self.api_resources.fetch_project_details(
transplatform.engine_name, transplatform.api_url, package_name)
platform_url = transplatform.api_url + "/projects/" + package_name
return platform_url, resp_dict
def add_package(self, **kwargs):
"""
add package to db
:param kwargs: dict
:return: boolean
"""
required_params = ('package_name', 'upstream_url',
'transplatform_slug', 'release_streams')
if not set(required_params) <= set(kwargs.keys()):
return
if not (kwargs['package_name'] and kwargs['upstream_url']):
return
try:
# derive translation platform project URL
platform = Platform.objects.only('engine_name', 'api_url') \
.filter(platform_slug=kwargs.pop('transplatform_slug')).get()
kwargs['platform_url'], resp_dict = \
self.get_project_details(platform, kwargs['package_name'])
if resp_dict:
# save project details in db
kwargs['package_details_json_str'] = json.dumps(resp_dict)
kwargs['details_json_last_updated'] = timezone.now()
if 'update_stats' in kwargs:
del kwargs['update_stats']
kwargs['platform_slug'] = platform
kwargs['products'] = kwargs.pop('release_streams')
kwargs['platform_name'] = kwargs['package_name']
kwargs['upstream_name'] = kwargs['upstream_url'].split('/')[-1]
# save in db
new_package = Package(**kwargs)
new_package.save()
except:
# log event, pass for now
# todo - implement error msg handling
return False
else:
return True
def _get_project_ids_names(self, engine, projects):
ids = []
names = []
if isinstance(projects, list) and engine == TRANSPLATFORM_ENGINES[0]:
for project in projects:
ids.append(project['fields']['name'])
elif isinstance(projects, dict) and engine == TRANSPLATFORM_ENGINES[1]:
ids.append(projects.get('slug'))
names.append(projects.get('name'))
elif isinstance(projects, list) and engine == TRANSPLATFORM_ENGINES[2]:
for project in projects:
ids.append(project['id'])
names.append(project['name'])
elif isinstance(projects, list) and engine == TRANSPLATFORM_ENGINES[3]:
for project in projects:
ids.append(project.get('slug'))
names.append(project.get('name'))
return ids, names
def validate_package(self, **kwargs):
"""
Validates existence of a package at a transplatform
:param kwargs: dict
:return: package_name: str, Boolean
"""
if not (kwargs.get('package_name')):
return
package_name = kwargs['package_name']
transplatform_fields = ('engine_name', 'api_url', 'projects_json_str',
'auth_login_id', 'auth_token_key')
# get transplatform projects from db
platform = Platform.objects.only(*transplatform_fields) \
.filter(platform_slug=kwargs['transplatform_slug']).get()
projects_json = platform.projects_json
# if not found in db, fetch transplatform projects from API
if not projects_json:
response_dict = None
auth_dict = dict(auth_user=platform.auth_login_id,
auth_token=platform.auth_token_key)
if platform.engine_name == TRANSPLATFORM_ENGINES[1]:
response_dict = self.api_resources.fetch_project_details(
platform.engine_name, platform.api_url,
package_name.lower(), **auth_dict)
elif platform.engine_name in (TRANSPLATFORM_ENGINES[0],
TRANSPLATFORM_ENGINES[2],
TRANSPLATFORM_ENGINES[3]):
response_dict = self.api_resources.fetch_all_projects(
platform.engine_name, platform.api_url, **auth_dict) or {}
# save all_projects_json in db - faster validation next times
# except transifex, as there we have project level details
Platform.objects.filter(api_url=platform.api_url).update(
projects_json_str=json.dumps(response_dict),
projects_last_updated=timezone.now())
if response_dict:
projects_json = response_dict
ids, names = self._get_project_ids_names(platform.engine_name,
projects_json)
if package_name in ids:
return package_name
elif package_name in names:
return ids[names.index(package_name)]
else:
return False
def get_lang_id_name_dict(self, release_branch=None):
"""
Generates {(locale, alias): language_name} dict
"""
active_locales = self.get_locales(
pick_locales=self.get_relbranch_locales(release_branch)
) if release_branch else self.get_locales_set()[0]
lang_id_name = dict([((lang.locale_id, lang.locale_alias),
lang.lang_name) for lang in active_locales])
return OrderedDict(
sorted(lang_id_name.items(), key=operator.itemgetter(1)))
def get_trans_stats(self,
package_name,
apply_branch_mapping=False,
specify_branch=None):
"""
fetch stats of a package for all enabled languages
:param package_name: str
:param apply_branch_mapping: boolean
:param specify_branch: release branch name
:return: dict {project_version: stats_dict}
"""
trans_stats_dict = OrderedDict()
package_desc = ''
# 1st, get active locales for which stats are to be shown
# or, choose release branch specific locales
lang_id_name = self.get_lang_id_name_dict(specify_branch) \
if apply_branch_mapping and specify_branch else self.get_lang_id_name_dict()
# 2nd, filter stats json for required locales
if self.is_package_exist(package_name):
package_details = self.get_packages([package_name]).get()
if not (package_details.platform_last_updated
or package_details.upstream_last_updated):
return lang_id_name, trans_stats_dict, package_desc
if package_details.package_details_json and package_details.package_details_json.get(
'description'):
package_desc = package_details.package_details_json[
'description']
pkg_stats_versions = self.syncstats_manager.get_sync_stats(
pkgs=[package_name])
for pkg_stats_version in pkg_stats_versions:
trans_stats_list, missing_locales = \
self.syncstats_manager.filter_stats_for_required_locales(
package_details.platform_slug_id,
pkg_stats_version.stats_raw_json, list(lang_id_name),
pkg_stats_version.source
)
if 'test' not in pkg_stats_version.project_version \
and 'extras' not in pkg_stats_version.project_version:
trans_stats_dict[pkg_stats_version.project_version] = \
self.syncstats_manager.extract_locale_translated(package_details.platform_slug_id,
trans_stats_list,
pkg_stats_version.source)
if apply_branch_mapping and package_details.release_branch_mapping:
branch_mapping = package_details.release_branch_mapping_json
for relbranch, branch_mapping in branch_mapping.items():
trans_stats_dict[relbranch] = trans_stats_dict.get(
branch_mapping.get(BRANCH_MAPPING_KEYS[0]), [])
return lang_id_name, trans_stats_dict, package_desc
def _get_pkg_and_ext(self, package_name):
package = self.get_packages([package_name]).get()
# extension for Transifex should be true, otherwise false
extension = (True if package.platform_slug.engine_name
== TRANSPLATFORM_ENGINES[0] else False)
return package, extension
def sync_update_package_details(self, package_name):
"""
Sync with translation platform and update details in db for a package
:param package_name: str
:return: boolean
"""
update_pkg_status = False
package, ext = self._get_pkg_and_ext(package_name)
platform = package.platform_slug
project_details_response_dict = self.api_resources.fetch_project_details(
platform.engine_name, platform.api_url, package_name,
**(dict(auth_user=platform.auth_login_id,
auth_token=platform.auth_token_key)))
if project_details_response_dict:
try:
Package.objects.filter(
platform_url=package.platform_url).update(
package_details_json_str=json.dumps(
project_details_response_dict),
details_json_last_updated=timezone.now())
except Exception as e:
self.app_logger('ERROR',
"Package update failed, details: " + str(e))
else:
update_pkg_status = True
return update_pkg_status
def sync_update_package_stats(self, package_name):
"""
Sync with translation platform and update trans stats in db for a package
:param package_name: str
:return: boolean
"""
update_stats_status = False
package, ext = self._get_pkg_and_ext(package_name)
project, versions = parse_project_details_json(
package.platform_slug.engine_name, package.package_details_json)
for version in versions:
proj_trans_stats_response_dict = {}
if package.platform_slug.engine_name == TRANSPLATFORM_ENGINES[0]:
# this is a quick fix for chinese in DamnedLies modules
locales = [
locale.locale_alias
if 'zh' not in locale.locale_id else locale.locale_id
for locale in self.get_locales(only_active=True)
]
locales_stats_list = []
for locale in locales:
locale_stats = self.api_resources.fetch_translation_statistics(
package.platform_slug.engine_name,
package.platform_slug.api_url, locale, version,
**dict(package_name=package_name))
if locale_stats:
locales_stats_list.append(locale_stats)
proj_trans_stats_response_dict.update({
"id":
version,
"stats":
locales_stats_list
})
else:
proj_trans_stats_response_dict = self.api_resources.fetch_translation_statistics(
package.platform_slug.engine_name,
package.platform_slug.api_url, project, version,
**dict(auth_user=package.platform_slug.auth_login_id,
auth_token=package.platform_slug.auth_token_key))
if proj_trans_stats_response_dict:
processed_stats = {}
# Process and Update locale-wise stats
if self.PROCESS_STATS and proj_trans_stats_response_dict.get(
'stats'):
processed_stats = self._process_response_stats_json(
proj_trans_stats_response_dict['stats'],
package.platform_slug.engine_name)
if self.PROCESS_STATS and package.platform_slug.engine_name == TRANSPLATFORM_ENGINES[1] \
and not processed_stats:
processed_stats = self._process_response_stats_json_tx(
proj_trans_stats_response_dict)
if self.syncstats_manager.save_version_stats(
package,
version,
proj_trans_stats_response_dict,
package.platform_slug.engine_name,
p_stats=processed_stats):
Package.objects.filter(
platform_url=package.platform_url).update(
platform_last_updated=timezone.now())
update_stats_status = True
# this makes sense if we create branch-mapping just after package sync
self.build_branch_mapping(package_name)
return update_stats_status
@staticmethod
def get_pkg_branch_mapping(pkg):
return PackageBranchMapping(pkg).branch_mapping
def build_branch_mapping(self, package_name):
"""
Creates Branch mapping for the Package
:param package_name: str
:return: boolean
"""
kwargs = {}
branch_mapping_dict = self.get_pkg_branch_mapping(package_name)
if not branch_mapping_dict:
return False
kwargs['package_name_mapping_json_str'] = json.dumps(
{package_name: ''})
kwargs['release_branch_mapping'] = json.dumps(branch_mapping_dict)
kwargs['release_branch_map_last_updated'] = timezone.now()
try:
Package.objects.filter(package_name=package_name).update(**kwargs)
except Exception as e:
self.app_logger(
'ERROR',
"Package branch mapping could not be saved, details: " +
str(e))
else:
return True
return False
def filter_n_reduce_stats(self, locale_key, locale, locale_alias,
stats_json):
"""
Filter and reduce multiple statistics for single language
:param locale_key: str
:param locale: str
:param locale_alias: str
:param stats_json: dict
:return: list
"""
filter_n_reduced_stat = []
def _reduce_stats_for_a_locale(stats):
return [
reduce(
lambda x, y: x
if x.get('translated', 0) > y.get('translated', 0) else y,
stats)
]
filter_stat_locale = list(
filter(
lambda x: x[locale_key] == locale or x[locale_key].replace(
'-', '_') == locale, stats_json))
filter_stat_alias = list(
filter(
lambda x: x[locale_key] == locale_alias or x[locale_key].
replace('_', '-') == locale_alias, stats_json))
if filter_stat_locale and len(filter_stat_locale) > 1:
filter_stat_locale = _reduce_stats_for_a_locale(filter_stat_locale)
if filter_stat_alias and len(filter_stat_alias) > 1:
filter_stat_alias = _reduce_stats_for_a_locale(filter_stat_alias)
if filter_stat_locale and filter_stat_alias:
filter_n_reduced_stat = _reduce_stats_for_a_locale(
filter_stat_locale + filter_stat_alias)
elif filter_stat_locale and not filter_stat_alias:
filter_n_reduced_stat = filter_stat_locale
elif filter_stat_alias and not filter_stat_locale:
filter_n_reduced_stat = filter_stat_alias
return filter_n_reduced_stat
def _process_response_stats_json(self, stats_json, engine=None):
locale_key = 'locale'
if engine and engine == TRANSPLATFORM_ENGINES[3]:
locale_key = 'code'
lang_id_name = self.get_lang_id_name_dict() or []
processed_stats_json = {}
for locale, l_alias in list(lang_id_name.keys()):
filter_stat = {}
try:
filter_stat = self.filter_n_reduce_stats(
locale_key, locale, l_alias, stats_json)
except Exception as e:
self.app_logger(
'ERROR', "Error while filtering stats, details: " + str(e))
else:
filter_stat = filter_stat[0] \
if isinstance(filter_stat, list) and len(filter_stat) > 0 else {}
processed_stats_json[locale] = {}
processed_stats_json[locale]['Total'] = filter_stat.get('total', 0)
processed_stats_json[locale]['Translated'] = filter_stat.get(
'translated', 0)
processed_stats_json[locale]['Untranslated'] = \
filter_stat.get('untranslated', 0) or \
processed_stats_json[locale]['Total'] - processed_stats_json[locale]['Translated']
remaining = 0
try:
remaining = (filter_stat.get('untranslated', 0.0) /
filter_stat.get('total', 0.0)) * 100
except ZeroDivisionError:
# log error, pass for now
pass
processed_stats_json[locale]['Remaining'] = round(remaining, 2)
return processed_stats_json
def _process_response_stats_json_tx(self, stats_json):
lang_id_name = self.get_lang_id_name_dict() or []
processed_stats_json = {}
for lang_alias_tuple in lang_id_name.keys():
for locale, stats in stats_json.items():
if locale in lang_alias_tuple:
filter_stats = {}
filter_stats['Translated'] = stats.get(
'translated_entities', 0)
filter_stats['Untranslated'] = stats.get(
'untranslated_entities', 0)
total_entities = stats.get('translated_entities',
0) + stats.get(
'untranslated_entities', 0)
filter_stats['Total'] = total_entities
remaining = 0
try:
remaining = (stats.get('untranslated_entities', 0.0) /
total_entities) * 100
except ZeroDivisionError:
# log error, pass for now
pass
filter_stats['Remaining'] = round(remaining, 2)
processed_stats_json.update(
{lang_alias_tuple[0]: filter_stats})
return processed_stats_json
def refresh_package(self, package_name):
"""
Sync Stats and Rebuild Mapping
:param package_name: str
:return: boolean
"""
if not package_name:
return False
status = []
steps = (self.sync_update_package_details,
self.sync_update_package_stats)
for method in steps:
status.append(method(package_name))
return True if [i for i in status if i] else False
def _formats_stats_diff(self, lang_sequence, stats_diff_dict):
lang_dict = {}
[lang_dict.update({index: lang}) for index, lang in lang_sequence]
formatted_stats_dict = OrderedDict()
for branch, stats_diff in stats_diff_dict.items():
formatted_stats_dict[branch] = {}
buildsys_diff = stats_diff['buildsys']
platform_diff_dict = dict(stats_diff['platform'])
for lang_index, stat in buildsys_diff:
formatted_stats_dict[branch][lang_dict.get(lang_index)] = \
(platform_diff_dict.get(lang_index, 0) - stat
if platform_diff_dict.get(lang_index, 0) > stat else 0)
return formatted_stats_dict
@staticmethod
def _filter_pkg_branch_map(mapping):
if not mapping:
return {}
mapping_dict = mapping.copy()
for release, map_value in mapping.items():
for k, v in map_value.items():
if not map_value.get(k):
mapping_dict.pop(release)
return mapping_dict
def calculate_stats_diff(self, package, graph_ready_stats, pkg_branch_map):
"""
Calculates and stores translation stats differences
- first look for 100% in the build system
- if not, then look for anything to pull from translation platform
:param package: str
:param graph_ready_stats: dict
:param pkg_branch_map: dict
:return: languages list and stats diff dict
"""
pkg_branch_map = self._filter_pkg_branch_map(pkg_branch_map)
stats_diff_dict = OrderedDict()
stats_data = graph_ready_stats.get('graph_data', {})
languages = graph_ready_stats.get('ticks', [])
for branch, mapping in pkg_branch_map.items():
stats_diff_dict[branch] = {}
platform_stats = stats_data.get(mapping[BRANCH_MAPPING_KEYS[0]],
[])
buildsys_stats = stats_data.get(
mapping[BRANCH_MAPPING_KEYS[1]] + " - " +
mapping[BRANCH_MAPPING_KEYS[2]], [])
# Ignoring after-decimal differences for now
first_set = set(
map(tuple,
[(index, round(stat)) for index, stat in platform_stats]))
second_set = set(
map(tuple,
[(index, round(stat)) for index, stat in buildsys_stats]))
if first_set and second_set:
platform_diff = first_set.difference(second_set)
buildsys_diff = second_set.difference(first_set)
# filter 100% translated languages from buildsys_diff
buildsys_diff = set(
map(tuple, [(i, j) for i, j in buildsys_diff if j < 100]))
stats_diff_dict[branch]['platform'] = platform_diff
stats_diff_dict[branch]['buildsys'] = buildsys_diff
else:
raise Exception(
"Make sure all mapped versions/tags are sync'd for stats.")
polished_stats_diff = self._formats_stats_diff(languages,
stats_diff_dict)
if package and polished_stats_diff:
self.update_package(
package, {'stats_diff': json.dumps(polished_stats_diff)})
return polished_stats_diff
def is_package_build_latest(self, params):
"""
Determine if new build is available for the package
:param params: package_name, build_system, build_tag
:return: boolean - True or False
"""
if not isinstance(params, (list, tuple)) and not len(params) == 3:
return
package_name, build_system, build_tag = params
product_hub_url = ''
product = self.get_release_streams(built=build_system)
if product:
product_hub_url = product.first().product_server
try:
builds = self.api_resources.build_info(hub_url=product_hub_url,
tag=build_tag,
pkg=package_name)
latest_build = {}
if builds and len(builds) > 0:
latest_build = builds[0]
kwargs = {}
package_qs = self.get_packages(pkgs=[package_name])
kwargs.update(dict(package_name=package_qs.get()))
kwargs.update(dict(build_system=build_system))
kwargs.update(dict(build_tag=build_tag))
try:
cache_build_detail = CacheBuildDetails.objects.filter(**kwargs)
except Exception as e:
return False
else:
if cache_build_detail and \
cache_build_detail.first().build_details_json == latest_build:
return True
except Exception:
return False
return
def get_build_system_stats_by_release(self, release=None):
"""
Get Build System Stats by Release
:param release:
:return:
"""
releases = self.release_manager.get_release_branches(relbranch=release)
build_system_stats_query_set = self.syncstats_manager.get_build_system_stats(
)
stats_by_release = {release.release_slug: {} for release in releases}
for sync_stats in build_system_stats_query_set:
if isinstance(sync_stats.stats_raw_json,
dict) and sync_stats.stats_raw_json.get('stats'):
processed_stats = self._process_response_stats_json(
sync_stats.stats_raw_json['stats'])
pkg_branch_map = sync_stats.package_name.release_branch_mapping_json
respective_release = [
r for r, d in pkg_branch_map.items() if d and d.get(
BRANCH_MAPPING_KEYS[2]) in sync_stats.project_version
]
if respective_release and len(respective_release) > 0:
pkg_release = respective_release[0]
if pkg_release in stats_by_release and not stats_by_release.get(
pkg_release):
stats_by_release[pkg_release] = processed_stats
else:
for r_locale, r_stats in stats_by_release[
pkg_release].items():
for k, v in r_stats.items():
if not k == 'Remaining':
stats_by_release[pkg_release][r_locale][k] += \
processed_stats.get(r_locale, {}).get(k, 0)
if release and release in stats_by_release:
return stats_by_release.get(release, {})
return stats_by_release
class BuildTagsSyncManager(BaseManager):
"""
Build Tags Sync Manager
"""
def __init__(self, *args, **kwargs):
"""
entry point
"""
super(BuildTagsSyncManager, self).__init__(self, *args, **kwargs)
self.job_manager = JobManager(TS_JOB_TYPES[4])
self.release_branch_manager = ReleaseBranchManager()
def syncbuildtags_initiate_job(self):
"""
Creates a Sync Job
"""
if self.job_manager.create_job(user_email=self.active_user_email):
return self.job_manager.uuid
return None
def sync_build_tags(self):
"""
Run Sync process in sequential steps
"""
stages = (
self.update_build_system_tags,
self.job_manager.mark_job_finish,
)
[method() for method in stages]
def update_build_system_tags(self):
"""
Update build system tags
"""
SUBJECT = 'Build System Tags'
self.job_manager.log_json[SUBJECT] = OrderedDict()
active_release_streams = \
self.release_branch_manager.get_release_streams(only_active=True)
self.job_manager.log_json[SUBJECT].update(
{str(datetime.now()): str(len(active_release_streams)) + ' releases fetched from db.'}
)
for relstream in active_release_streams or []:
hub_server_url = relstream.product_server
# # for brew several configuration needs to be set
# # before we access its hub for info, #todo
# if relstream.product_build_system == 'koji':
try:
tags = self.api_resources.build_tags(hub_url=hub_server_url)
except Exception as e:
self.job_manager.log_json[SUBJECT].update(
{str(datetime.now()): 'Failed to fetch build tags of %s. Details: %s' %
(relstream.product_build_system, str(e))}
)
self.job_manager.job_result = False
else:
if tags:
relbranch_update_result = Product.objects.filter(
product_slug=relstream.product_slug
).update(product_build_tags=tags,
product_build_tags_last_updated=timezone.now())
if relbranch_update_result:
self.job_manager.log_json[SUBJECT].update(
{str(datetime.now()): '%s build tags of %s saved in db.' %
(str(len(tags)), relstream.product_build_system)}
)
self.job_manager.job_result = True
return self.job_manager.job_result
class ReleaseScheduleSyncManager(BaseManager):
"""
Release Schedule Sync Manager
"""
def __init__(self, *args, **kwargs):
"""
entry point
"""
super(ReleaseScheduleSyncManager, self).__init__(self, *args, **kwargs)
self.job_manager = JobManager(TS_JOB_TYPES[1])
self.release_branch_manager = ReleaseBranchManager()
def syncschedule_initiate_job(self):
"""
Creates a Sync Job
"""
if self.job_manager.create_job(user_email=self.active_user_email):
return self.job_manager.uuid
return None
def sync_release_schedule(self):
"""
Run Sync process in sequential steps
"""
stages = (
self.update_event_dates,
self.job_manager.mark_job_finish,
)
[method() for method in stages]
def update_event_dates(self):
"""
Update schedule_json for all release branches
"""
SUBJECT = 'Release Branches'
self.job_manager.log_json[SUBJECT] = OrderedDict()
try:
relbranches = Release.objects.only('release_slug', 'product_slug', 'calendar_url') \
.filter(sync_calendar=True).all()
except Exception as e:
self.job_manager.log_json[SUBJECT].update(
{str(datetime.now()): 'Fetch relbranches from db failed. Details: ' + str(e)}
)
self.job_result = False
else:
self.job_manager.log_json[SUBJECT].update(
{str(datetime.now()): str(len(relbranches)) + ' release branches fetched from db.'}
)
for relbranch in relbranches:
product_slug = relbranch.product_slug.product_slug
ical_events = self.release_branch_manager.get_calendar_events_dict(
relbranch.calendar_url, product_slug)
release_stream = self.release_branch_manager.get_release_streams(
stream_slug=product_slug).get()
required_events = release_stream.major_milestones
schedule_dict = \
self.release_branch_manager.parse_events_for_required_milestones(
product_slug, relbranch.release_slug, ical_events, required_events
)
if schedule_dict:
relbranch_update_result = Release.objects.filter(
release_slug=relbranch.release_slug
).update(schedule_json_str=json.dumps(schedule_dict))
if relbranch_update_result:
self.job_manager.log_json[SUBJECT].update(
{str(datetime.now()): 'Release schedule for ' + relbranch.release_slug +
' branch updated in db.'}
)
self.job_manager.job_result = True
else:
self.job_manager.log_json[SUBJECT].update(
{str(datetime.now()): 'Release schedule for ' + relbranch.release_slug +
' failed to get saved in db.'}
)
self.job_manager.job_result = False
return self.job_manager.job_result
class GraphManager(BaseManager):
"""
Manage graph representations
"""
package_manager = PackagesManager()
branch_manager = ReleaseBranchManager()
def get_graph_rules(self, graph_rule=None, only_active=None):
"""
Fetch graph rules from db
:return: resultset
"""
filter_kwargs = {}
if graph_rule:
filter_kwargs.update(dict(rule_name=graph_rule))
if only_active:
filter_kwargs.update(dict(rule_status=True))
rules = None
try:
rules = GraphRule.objects.filter(
**filter_kwargs).order_by('rule_name')
except:
# log event, passing for now
pass
return rules
def slugify_graph_rule_name(self, suggested_name):
try:
return slugify(suggested_name)
except:
# log even, passing for now
return False
def validate_package_branch_participation(self, relbranch, packages):
"""
This validates that packages belong to relbranch or not
:param relbranch: release branch
:param packages: list of packages
:return: list of packages that DO NOT belong
"""
if not relbranch and not packages:
return
pkg_not_participate = []
for package in packages:
relbranches = PackageBranchMapping(package).release_branches
if relbranch not in relbranches:
pkg_not_participate.append(package)
return pkg_not_participate
def validate_tags_product_participation(self, release_slug, tags):
"""
This validates that tag belongs to release
:param release_slug: Release Slug
:param tags: Build Tags
:return: List of tags that do not belong
"""
if not release_slug and not tags:
return
tags_not_participate = []
q_release = self.branch_manager.get_release_branches(
relbranch=release_slug)
if q_release:
release = q_release.get()
release_build_tags = release.product_slug.product_build_tags
for tag in tags:
if tag not in release_build_tags:
tags_not_participate.append(tag)
return tags_not_participate
def add_graph_rule(self, **kwargs):
"""
Save graph rule in db
:param kwargs: dict
:return: boolean
"""
if not kwargs.get('rule_name'):
return
if not kwargs.get('rule_relbranch'):
return
relbranch_slug = kwargs.get('rule_relbranch')
release_branch = \
self.branch_manager.get_release_branches(relbranch=relbranch_slug)
if kwargs.get('tags_selection'
) == "pick" and not kwargs.get('rule_build_tags'):
release_packages = \
self.package_manager.get_relbranch_specific_pkgs(
relbranch_slug, fields=['release_branch_mapping']
)
package_tags = [
package.release_branch_mapping_json[relbranch_slug][
BRANCH_MAPPING_KEYS[2]] for package in release_packages
if package.release_branch_mapping_json
and package.release_branch_mapping_json.get(
relbranch_slug, {}).get(BRANCH_MAPPING_KEYS[2])
]
if package_tags:
kwargs['rule_build_tags'] = list(set(package_tags))
else:
return False
if kwargs.get(
'lang_selection') == "pick" and not kwargs.get('rule_langs'):
relbranch_lang_set = self.branch_manager.get_release_branches(
relbranch=relbranch_slug,
fields=['language_set_slug']).first()
locales = relbranch_lang_set.language_set_slug.locale_ids
if locales:
kwargs['rule_languages'] = locales
else:
return False
elif kwargs.get('rule_langs'):
kwargs['rule_languages'] = kwargs.pop('rule_langs')
filters = [
'tags_selection', 'lang_selection', 'rule_langs', 'rule_relbranch'
]
[kwargs.pop(i) for i in filters if i in kwargs]
try:
kwargs['rule_release_slug'] = release_branch.get()
kwargs['created_on'] = timezone.now()
kwargs['rule_status'] = True
new_rule = GraphRule(**kwargs)
new_rule.save()
except:
# log event, pass for now
# todo implement error msg handling
return False
else:
return True
def _normalize_stats(self, stats_nested_list, index_list):
"""
Normalize stats for a locale index and picks higher value
"""
temp_index_list = []
temp_stat_list = []
for index, stat in stats_nested_list:
if index not in temp_index_list:
temp_index_list.append(index)
temp_stat_list.append(stat)
else:
last_stat = temp_stat_list.pop(len(temp_stat_list) - 1)
temp_stat_list.append(last_stat) \
if last_stat > stat else temp_stat_list.append(stat)
expected_stats_list = list(zip(temp_index_list, temp_stat_list))
if len(index_list) > len(expected_stats_list):
expected_stats_dict = dict(
(k[0], k[1:]) for k in expected_stats_list)
temp_patched_stats_list = []
for index in index_list:
temp_patched_stats_list.append(
[index, expected_stats_dict.get(index, [0.0])[0]])
expected_stats_list = temp_patched_stats_list
return expected_stats_list
def _format_stats_for_default_graphs(self,
locale_sequence,
stats_dict,
desc,
prepend_source=False):
"""
Formats stats dict for graph-ready material
- sorting and normalization
"""
stats_for_graphs_dict = OrderedDict()
stats_for_graphs_dict['pkg_desc'] = desc
stats_for_graphs_dict['ticks'] = \
[[i, lang] for i, lang in enumerate(locale_sequence.values(), 0)]
indexes = [index for index, lang in stats_for_graphs_dict['ticks']]
graph_data_dict = {}
for version, stats_lists in stats_dict.items():
new_stats_list = []
for stats_tuple in stats_lists:
index = [
i
for i, locale_tuple in enumerate(list(locale_sequence), 0)
if (stats_tuple[0] in locale_tuple) or (
stats_tuple[0].replace('-', '_') in locale_tuple) or (
stats_tuple[0].replace('_', '-') in locale_tuple)
]
if index:
index.append(stats_tuple[1] or 0.0)
new_stats_list.append(index)
if prepend_source:
if stats_tuple[0] == 'source' and stats_tuple[
1] not in version.lower():
version = "{0} - {1}".format(stats_tuple[1], version)
normalized_stats = self._normalize_stats(sorted(new_stats_list),
indexes)
if len(list(filter(lambda x: x[1] > 0.0, normalized_stats))) > 0:
graph_data_dict[version] = normalized_stats
stats_for_graphs_dict['graph_data'] = OrderedDict(
sorted(graph_data_dict.items()))
return stats_for_graphs_dict
def get_trans_stats_by_package(self, package, prepend_source=False):
"""
formats stats of a package for all enabled languages
:param package: str
:param prepend_source: boolean
:return: Graph data for "Package-wise" view: dict
"""
if not package:
return {}
lang_id_name, stats_dict, pkg_desc = self.package_manager.get_trans_stats(
package)
# format trans_stats_list for graphs
return self._format_stats_for_default_graphs(lang_id_name, stats_dict,
pkg_desc, prepend_source)
def _format_stats_for_lang_wise_graphs(self, input_locale, locale_sequence,
stats_dict, desc):
"""
Formats stats dict for bar graph-ready material
"""
stats_for_graphs_dict = OrderedDict()
stats_branches = list(stats_dict.keys())
stats_for_graphs_dict['pkg_desc'] = desc
stats_for_graphs_dict['ticks'] = \
[[i, version] for i, version in enumerate(stats_branches, 0)]
stats_for_graphs_dict['graph_data'] = []
for version, stats_lists in stats_dict.items():
index = stats_branches.index(version)
required_stat = 0.0
for locale_tuple in list(locale_sequence.keys()):
if input_locale in locale_tuple:
locale, alias = locale_tuple
expected_stat = [
stat for lang, stat in stats_lists
if lang.replace('-', '_') == locale
or lang.replace('-', '_') == alias
]
required_stat = expected_stat[0] if expected_stat and len(
expected_stat) > 0 else 0.0
stats_for_graphs_dict['graph_data'].append([required_stat, index])
return stats_for_graphs_dict
def get_stats_by_pkg_per_lang(self, package, locale):
"""
formats stats of a package for given locale
:param package: str
:param locale: str
:return: Graph data for "Language-wise" view: dict
"""
if not package and not locale:
return {}
lang_id_name, stats_dict, pkg_desc = self.package_manager.get_trans_stats(
package)
# format stats for lang-wise graph
return self._format_stats_for_lang_wise_graphs(locale, lang_id_name,
stats_dict, pkg_desc)
def get_trans_stats_by_rule(self, graph_rule):
"""
formats stats of a graph rule
:param: graph_rule: str
:return: Graph data for "Rule-wise" view: dict
"""
rule_data = {}
rule = self.get_graph_rules(graph_rule=graph_rule).get()
packages = rule.rule_packages
locales = rule.rule_languages
tags = rule.rule_build_tags
release = rule.rule_release_slug_id
if rule:
rule_data = self.package_manager.get_trans_stats_by_rule(rule)
return rule_data, len(packages), len(locales), len(tags), release
def _consolidate_branch_specific_stats(self, packages_stats_dict):
"""
Sum up stats per language
"""
temp_stats_dict = {}
pkgs_stats_list = list(packages_stats_dict.values())
pkgs_length = len(pkgs_stats_list)
for pkg_stats in pkgs_stats_list:
for pkg_stat in pkg_stats:
if pkg_stat[0] not in temp_stats_dict:
temp_stats_dict[pkg_stat[0]] = pkg_stat[1]
else:
temp_stats_dict[pkg_stat[0]] += pkg_stat[1]
# Reverse stats to depict how much is left
return sorted([(i, 100 - int(j / pkgs_length))
for i, j in temp_stats_dict.items()])
def _format_data_for_pie_chart(self, consolidated_stats, lang_options):
"""
Takes consolidated stats and formats for pie chart
"""
formatted_stats = []
formatted_langs = []
for lang, stat in consolidated_stats:
formatted_stats.append({'label': lang, 'data': stat})
for locale, language in lang_options:
formatted_langs.append({'value': locale, 'text': language})
return {
'graph_data': formatted_stats,
'select_options': sorted(formatted_langs, key=itemgetter('text'))
}
def _get_branch_specific_pkgs_stats(self, relbranch):
"""
Generates translation stats of all packages in all langs of attached lang-set
"""
specific_pkgs = [
package.package_name
for package in self.package_manager.get_relbranch_specific_pkgs(
relbranch, ['package_name'])
]
all_pkgs_stats_dict = OrderedDict()
for pkg in specific_pkgs:
pkg_lang_stats = []
locale_seq, trans_stats_dict, pkg_desc = \
self.package_manager.get_trans_stats(pkg, apply_branch_mapping=True, specify_branch=relbranch)
t_stats = self._format_stats_for_default_graphs(
locale_seq, trans_stats_dict, pkg_desc)
branch_stats = t_stats.get('graph_data').get(relbranch)
langs = t_stats.get('ticks')
if branch_stats and langs:
pkg_lang_stats.extend(
list(
zip([lang[1] for lang in langs],
[stat[1] for stat in branch_stats])))
all_pkgs_stats_dict[pkg] = pkg_lang_stats
return all_pkgs_stats_dict
def get_workload_graph_data(self, release_branch):
"""
Build or generates workload graph data
"""
consolidated_stats = self._consolidate_branch_specific_stats(
self._get_branch_specific_pkgs_stats(release_branch))
# get branch specific languages for select option
locale_lang_tuple = self.package_manager.get_locale_lang_tuple(
locales=self.package_manager.get_relbranch_locales(release_branch))
return self._format_data_for_pie_chart(consolidated_stats,
locale_lang_tuple)
def _process_workload_combined_view(self, packages_stats_dict, headers):
stats_summary_dict = OrderedDict()
for package, locale_stats in packages_stats_dict.items():
reduced_stats = {}
try:
reduced_stats = functools.reduce(
lambda x, y: dict(Counter(x) + Counter(y)),
list(locale_stats.values()))
except Exception as e:
# log error, pass for now
pass
if not reduced_stats:
for field in headers:
reduced_stats[field] = 0
try:
if not reduced_stats.get('Untranslated'):
reduced_stats['Untranslated'] = 0
reduced_stats[headers[3]] = \
(reduced_stats['Untranslated'] /
reduced_stats.get('Total', 0) * 100)
except ZeroDivisionError:
# log error, pass for now
pass
stats_summary_dict[package] = reduced_stats
return stats_summary_dict
def get_workload_estimate(self, release_branch, locale=None):
"""
Build list of packages with translation workload for a given branch
"""
headers = WORKLOAD_HEADERS
pkg_stats = self.package_manager.get_release_specific_package_stats(
release_branch=release_branch)
required_stats_dict = {}
if not locale:
required_stats_dict = self._process_workload_combined_view(
pkg_stats, headers)
elif isinstance(locale, str):
for pkg, locale_stat in pkg_stats.items():
required_stats_dict[pkg] = locale_stat.get(locale) or {
header: 0
for header in headers
}
return headers, OrderedDict(
sorted(required_stats_dict.items(),
key=lambda x: x[1]['Remaining'],
reverse=True))
def get_workload_combined(self, release_branch):
"""
Build list of packages with translation workload for a given branch in all languages
"""
return self.get_workload_estimate(release_branch)
def get_workload_detailed(self, release_branch):
"""
Build translation workload percentage for a given branch in all languages
"""
relbranch_packages_stats = self._get_branch_specific_pkgs_stats(
release_branch)
locale_lang_tuple = self.package_manager.get_locale_lang_tuple(
locales=self.package_manager.get_relbranch_locales(release_branch))
headers = sorted([lang for locale, lang in locale_lang_tuple])
# Format data to fill table
workload_combined = OrderedDict()
for package, lang_stats in relbranch_packages_stats.items():
temp_stat_list = []
for lang_stat_tuple in lang_stats:
temp_stat_list.insert(headers.index(lang_stat_tuple[0]),
lang_stat_tuple[1])
# flag incorrect branch mapping
if len([i for i in temp_stat_list
if i == 0]) == len(temp_stat_list):
package += "*"
if temp_stat_list:
workload_combined[package] = temp_stat_list
return headers, OrderedDict(sorted(workload_combined.items()))
def get_workload_combined_detailed(self, release_branch):
"""
Build translation workload for a given branch in all langs for all pkgs
:param release_branch: str
:return: dict
"""
if not isinstance(release_branch, str):
return {}
locale_lang_tuple = self.package_manager.get_locale_lang_tuple(
locales=self.package_manager.get_relbranch_locales(release_branch))
workload_combined_detailed = {}
for locale, lang in locale_lang_tuple:
workload_combined_detailed[lang] = \
self.get_workload_estimate(release_branch, locale=locale)[1]
return workload_combined_detailed
def get_threshold_based(self, release_branch, threshold=70):
"""
Build language list those have fulfilled given threshold
:param release_branch: str
:param threshold: translation %age margin: int
:return: dict
"""
consolidated_stats = self._consolidate_branch_specific_stats(
self._get_branch_specific_pkgs_stats(release_branch))
# Reverse the stats to have - what has been covered
consolidated_stats_reversed = [(lang, 100 - stat)
for lang, stat in consolidated_stats]
filtered_stats = list(
filter(lambda elem: elem[1] > threshold,
consolidated_stats_reversed))
headers = ['Languages', 'Translation Complete %age']
locale_lang_tuple = self.package_manager.get_locale_lang_tuple(
locales=self.package_manager.get_relbranch_locales(release_branch))
lang_locale_dict = {v: k for k, v in dict(locale_lang_tuple).items()}
return headers, filtered_stats, lang_locale_dict
class ReleaseScheduleSyncManager(BaseManager):
"""
Release Schedule Sync Manager
"""
def __init__(self):
"""
entry point
"""
super(ReleaseScheduleSyncManager, self).__init__()
self.job_manager = JobManager(TS_JOB_TYPES[1])
self.release_branch_manager = ReleaseBranchManager()
def syncschedule_initiate_job(self):
"""
Creates a Sync Job
"""
if self.job_manager.create_job():
return self.job_manager.uuid
return None
def sync_release_schedule(self):
"""
Run Sync process in sequential steps
"""
stages = (
self.update_event_dates,
self.job_manager.mark_job_finish,
)
[method() for method in stages]
def update_event_dates(self):
"""
Update schedule_json for all release branches
"""
SUBJECT = 'Release Branches'
self.job_manager.log_json[SUBJECT] = OrderedDict()
try:
relbranches = StreamBranches.objects.only('relbranch_slug', 'relstream_slug', 'calendar_url') \
.filter(sync_calendar=True).all()
except Exception as e:
self.job_manager.log_json[SUBJECT].update({
str(datetime.now()):
'Fetch relbranches from db failed. Details: ' + str(e)
})
self.job_result = False
else:
self.job_manager.log_json[SUBJECT].update({
str(datetime.now()):
str(len(relbranches)) + ' release branches fetched from db.'
})
for relbranch in relbranches:
ical_events = self.release_branch_manager.get_calender_events_dict(
relbranch.calendar_url, relbranch.relstream_slug)
release_stream = self.release_branch_manager.get_release_streams(
stream_slug=relbranch.relstream_slug).get()
required_events = release_stream.major_milestones
schedule_dict = \
self.release_branch_manager.parse_events_for_required_milestones(
relbranch.relstream_slug, relbranch.relbranch_slug, ical_events, required_events
)
if schedule_dict:
relbranch_update_result = StreamBranches.objects.filter(
relbranch_slug=relbranch.relbranch_slug).update(
schedule_json=schedule_dict)
if relbranch_update_result:
self.job_manager.log_json[SUBJECT].update({
str(datetime.now()):
'Release schedule for ' +
relbranch.relbranch_slug + ' branch updated in db.'
})
self.job_manager.job_result = True
else:
self.job_manager.log_json[SUBJECT].update({
str(datetime.now()):
'Release schedule for ' + relbranch.relbranch_slug +
' failed to get saved in db.'
})
self.job_manager.job_result = False
return self.job_manager.job_result
class InventoryManagerMixin(object):
"""
Required Manager
"""
inventory_manager = ReleaseBranchManager()
class BuildTagsSyncManager(BaseManager):
"""
Build Tags Sync Manager
"""
def __init__(self, *args, **kwargs):
"""
entry point
"""
super(BuildTagsSyncManager, self).__init__(self, *args, **kwargs)
self.job_manager = JobManager(TS_JOB_TYPES[4])
self.release_branch_manager = ReleaseBranchManager()
def syncbuildtags_initiate_job(self):
"""
Creates a Sync Job
"""
if self.job_manager.create_job(user_email=self.active_user_email):
return self.job_manager.uuid
return None
def sync_build_tags(self):
"""
Run Sync process in sequential steps
"""
stages = (
self.update_build_system_tags,
self.job_manager.mark_job_finish,
)
[method() for method in stages]
def update_build_system_tags(self):
"""
Update build system tags
"""
SUBJECT = 'Build System Tags'
self.job_manager.log_json[SUBJECT] = OrderedDict()
active_release_streams = \
self.release_branch_manager.get_release_streams(only_active=True)
self.job_manager.log_json[SUBJECT].update({
str(datetime.now()):
str(len(active_release_streams)) + ' releases fetched from db.'
})
for relstream in active_release_streams or []:
hub_server_url = relstream.product_server
# # for brew several configuration needs to be set
# # before we access its hub for info, #todo
# if relstream.product_build_system == 'koji':
try:
tags = self.api_resources.build_tags(hub_url=hub_server_url,
product=relstream)
except Exception as e:
self.job_manager.log_json[SUBJECT].update({
str(datetime.now()):
'Failed to fetch build tags of %s. Details: %s' %
(relstream.product_build_system, str(e))
})
self.job_manager.job_result = False
else:
if tags:
relbranch_update_result = Product.objects.filter(
product_slug=relstream.product_slug).update(
product_build_tags=tags,
product_build_tags_last_updated=timezone.now())
if relbranch_update_result:
self.job_manager.log_json[SUBJECT].update({
str(datetime.now()):
'%s build tags of %s saved in db.' %
(str(len(tags)), relstream.product_build_system)
})
self.job_manager.job_result = True
return self.job_manager.job_result
class GraphManager(BaseManager):
"""
Manage graph representations
"""
package_manager = PackagesManager()
branch_manager = ReleaseBranchManager()
def get_graph_rules(self, graph_rule=None, only_active=None):
"""
Fetch graph rules from db
:return: resultset
"""
filter_kwargs = {}
if graph_rule:
filter_kwargs.update(dict(rule_name=graph_rule))
if only_active:
filter_kwargs.update(dict(rule_status=True))
rules = None
try:
rules = GraphRules.objects.filter(
**filter_kwargs).order_by('rule_name')
except:
# log event, passing for now
pass
return rules
def slugify_graph_rule_name(self, suggested_name):
try:
return slugify(suggested_name)
except:
# log even, passing for now
return False
def validate_package_branch_participation(self, relbranch, packages):
"""
This validates that packages belong to relbranch or not
:param relbranch: release branch
:param packages: list of packages
:return: list of packages that DO NOT belong
"""
if not relbranch and not packages:
return
pkg_not_participate = []
for package in packages:
relbranches = PackageBranchMapping(package).release_branches
if relbranch not in relbranches:
pkg_not_participate.append(package)
return pkg_not_participate
def add_graph_rule(self, **kwargs):
"""
Save graph rule in db
:param kwargs: dict
:return: boolean
"""
if kwargs.get(
'lang_selection') == "pick" and not kwargs.get('rule_langs'):
relbranch_slug = kwargs.get('rule_relbranch')
relbranch_lang_set = self.branch_manager.get_release_branches(
relbranch=relbranch_slug, fields=['lang_set']).first()
locales = self.branch_manager.get_langset(
relbranch_lang_set.lang_set, fields=['locale_ids']).locale_ids
if locales:
kwargs['rule_langs'] = locales
else:
return False
if not (kwargs['rule_name']):
return
try:
kwargs.pop('lang_selection')
kwargs['created_on'] = timezone.now()
kwargs['rule_status'] = True
new_rule = GraphRules(**kwargs)
new_rule.save()
except:
# log event, pass for now
# todo implement error msg handling
return False
else:
return True
def _normalize_stats(self, stats_nested_list, index_list):
"""
Normalize stats for a locale index and picks higher value
"""
temp_index_list = []
temp_stat_list = []
for index, stat in stats_nested_list:
if index not in temp_index_list:
temp_index_list.append(index)
temp_stat_list.append(stat)
else:
last_stat = temp_stat_list.pop(len(temp_stat_list) - 1)
temp_stat_list.append(last_stat) \
if last_stat > stat else temp_stat_list.append(stat)
expected_stats_list = list(zip(temp_index_list, temp_stat_list))
if len(index_list) > len(expected_stats_list):
expected_stats_dict = dict(
(k[0], k[1:]) for k in expected_stats_list)
temp_patched_stats_list = []
for index in index_list:
temp_patched_stats_list.append(
[index, expected_stats_dict.get(index, [0.0])[0]])
expected_stats_list = temp_patched_stats_list
return expected_stats_list
def _format_stats_for_default_graphs(self, locale_sequence, stats_dict,
desc):
"""
Formats stats dict for graph-ready material
- sorting and normalization
"""
stats_for_graphs_dict = OrderedDict()
stats_for_graphs_dict['pkg_desc'] = desc
stats_for_graphs_dict['ticks'] = \
[[i, lang] for i, lang in enumerate(locale_sequence.values(), 0)]
indexes = [index for index, lang in stats_for_graphs_dict['ticks']]
graph_data_dict = {}
for version, stats_lists in stats_dict.items():
new_stats_list = []
for stats_tuple in stats_lists:
index = [
i
for i, locale_tuple in enumerate(list(locale_sequence), 0)
if (stats_tuple[0] in locale_tuple) or (
stats_tuple[0].replace('-', '_') in locale_tuple)
]
if index:
index.append(stats_tuple[1] or 0.0)
new_stats_list.append(index)
graph_data_dict[version] = self._normalize_stats(
sorted(new_stats_list), indexes)
stats_for_graphs_dict['graph_data'] = OrderedDict(
sorted(graph_data_dict.items()))
return stats_for_graphs_dict
def get_trans_stats_by_package(self, package):
"""
formats stats of a package for all enabled languages
:param package: str
:return: Graph data for "Package-wise" view: dict
"""
if not package:
return {}
lang_id_name, stats_dict, pkg_desc = self.package_manager.get_trans_stats(
package)
upstream_stats = self.package_manager.get_upstream_stats(package)
if upstream_stats:
stats_dict['Upstream'] = upstream_stats
# format trans_stats_list for graphs
return self._format_stats_for_default_graphs(lang_id_name, stats_dict,
pkg_desc)
def _format_stats_for_lang_wise_graphs(self, input_locale, locale_sequence,
stats_dict, desc):
"""
Formats stats dict for bar graph-ready material
"""
stats_for_graphs_dict = OrderedDict()
stats_branches = list(stats_dict.keys())
stats_for_graphs_dict['pkg_desc'] = desc
stats_for_graphs_dict['ticks'] = \
[[i, version] for i, version in enumerate(stats_branches, 0)]
stats_for_graphs_dict['graph_data'] = []
for version, stats_lists in stats_dict.items():
index = stats_branches.index(version)
required_stat = 0.0
for locale_tuple in list(locale_sequence.keys()):
if input_locale in locale_tuple:
locale, alias = locale_tuple
expected_stat = [
stat for lang, stat in stats_lists
if lang.replace('-', '_') == locale
or lang.replace('-', '_') == alias
]
required_stat = expected_stat[0] if expected_stat and len(
expected_stat) > 0 else 0.0
stats_for_graphs_dict['graph_data'].append([required_stat, index])
return stats_for_graphs_dict
def get_stats_by_pkg_per_lang(self, package, locale):
"""
formats stats of a package for given locale
:param package: str
:param locale: str
:return: Graph data for "Language-wise" view: dict
"""
if not package and not locale:
return {}
lang_id_name, stats_dict, pkg_desc = self.package_manager.get_trans_stats(
package)
# format stats for lang-wise graph
return self._format_stats_for_lang_wise_graphs(locale, lang_id_name,
stats_dict, pkg_desc)
def _format_stats_for_custom_graphs(self, rel_branch, languages,
stats_dict):
"""
Formats stats dict for graph-ready material
"""
stats_for_graphs_dict = OrderedDict()
stats_for_graphs_dict['branch'] = rel_branch
stats_for_graphs_dict['ticks'] = \
[[i, package] for i, package in enumerate(stats_dict.keys(), 0)]
stats_for_graphs_dict['graph_data'] = []
for language in languages:
stats = []
graph_lang_stats_dict = OrderedDict()
graph_lang_stats_dict['label'] = language
for lang_stat in stats_dict.values():
stats.append(lang_stat.get(language))
graph_lang_stats_dict['data'] = \
[[i, stat] for i, stat in enumerate(stats, 0)]
stats_for_graphs_dict['graph_data'].append(graph_lang_stats_dict)
return stats_for_graphs_dict
def get_trans_stats_by_rule(self, graph_rule):
"""
formats stats of a graph rule
:param: graph_rule: str
:return: Graph data for "Rule-wise" view: dict
"""
if not graph_rule:
return {}
rule = self.get_graph_rules(graph_rule=graph_rule).get()
graph_rule_branch = rule.rule_relbranch
release_branch = self.branch_manager.get_release_branches(
relbranch=graph_rule_branch,
fields=['relbranch_name']).get().relbranch_name
packages = rule.rule_packages
exclude_packages = []
rule_locales = rule.rule_langs
trans_stats_dict_set = OrderedDict()
languages_list = []
for package in packages:
lang_id_name, package_stats = self.package_manager.get_trans_stats(
package, apply_branch_mapping=True)[0:2]
relbranch_stats = {}
if graph_rule_branch in package_stats:
relbranch_stats.update(package_stats.get(graph_rule_branch))
elif 'master' in package_stats and package_stats.get('master'):
relbranch_stats.update(package_stats.get('master'))
elif 'default' in package_stats and package_stats.get('default'):
relbranch_stats.update(package_stats.get('default'))
else:
exclude_packages.append(package)
# filter locale_tuple for required locales
required_locales = []
for locale_tuple, lang in lang_id_name.items():
rule_locale = [
rule_locale for rule_locale in rule_locales
if rule_locale in locale_tuple
]
if rule_locale:
required_locales.append((locale_tuple, lang))
# set stat for filtered_locale_tuple checking with both locale and alias
lang_stats = OrderedDict()
for locale, stat in relbranch_stats.items():
locale = locale.replace('-', '_') if '-' in locale else locale
for locale_tuple, lang in required_locales:
if locale in locale_tuple:
lang_stats[lang] = stat
languages_list = lang_stats.keys()
trans_stats_dict_set[package] = lang_stats
# this is to prevent any breaking in graphs
for ex_package in exclude_packages:
trans_stats_dict_set.pop(ex_package)
# here trans_stats_dict_set would contain {'package': {'language': stat}}
# now, lets format trans_stats_list for graphs
return self._format_stats_for_custom_graphs(release_branch,
languages_list,
trans_stats_dict_set)
def _consolidate_branch_specific_stats(self, packages_stats_dict):
"""
Sum up stats per language
"""
temp_stats_dict = {}
pkgs_stats_list = list(packages_stats_dict.values())
pkgs_length = len(pkgs_stats_list)
for pkg_stats in pkgs_stats_list:
for pkg_stat in pkg_stats:
if pkg_stat[0] not in temp_stats_dict:
temp_stats_dict[pkg_stat[0]] = pkg_stat[1]
else:
temp_stats_dict[pkg_stat[0]] += pkg_stat[1]
# Reverse stats to depict how much is left
return sorted([(i, 100 - int(j / pkgs_length))
for i, j in temp_stats_dict.items()])
def _format_data_for_pie_chart(self, consolidated_stats, lang_options):
"""
Takes consolidated stats and formats for pie chart
"""
formatted_stats = []
formatted_langs = []
for lang, stat in consolidated_stats:
formatted_stats.append({'label': lang, 'data': stat})
for locale, language in lang_options:
formatted_langs.append({'value': locale, 'text': language})
return {
'graph_data': formatted_stats,
'select_options': sorted(formatted_langs, key=itemgetter('text'))
}
def _get_branch_specific_pkgs_stats(self, relbranch):
"""
Generates translation stats of all packages in all langs of attached lang-set
"""
specific_pkgs = [
package.package_name
for package in self.package_manager.get_relbranch_specific_pkgs(
relbranch, ['package_name'])
]
all_pkgs_stats_dict = OrderedDict()
for pkg in specific_pkgs:
pkg_lang_stats = []
locale_seq, trans_stats_dict, pkg_desc = \
self.package_manager.get_trans_stats(pkg, apply_branch_mapping=True, specify_branch=relbranch)
t_stats = self._format_stats_for_default_graphs(
locale_seq, trans_stats_dict, pkg_desc)
branch_stats = t_stats.get('graph_data').get(relbranch)
langs = t_stats.get('ticks')
if branch_stats and langs:
pkg_lang_stats.extend(
list(
zip([lang[1] for lang in langs],
[stat[1] for stat in branch_stats])))
all_pkgs_stats_dict[pkg] = pkg_lang_stats
return all_pkgs_stats_dict
def get_workload_graph_data(self, release_branch):
"""
Build or generates workload graph data
"""
consolidated_stats = self._consolidate_branch_specific_stats(
self._get_branch_specific_pkgs_stats(release_branch))
# get branch specific languages for select option
locale_lang_tuple = self.package_manager.get_locale_lang_tuple(
locales=self.package_manager.get_relbranch_locales(release_branch))
return self._format_data_for_pie_chart(consolidated_stats,
locale_lang_tuple)
def _process_workload_combined_view(self, packages, stats_dict, fields):
"""
Process packages stats to sum-up all locales and find avg
"""
stats_summary_dict = OrderedDict()
fields_dict = OrderedDict([(field, 0) for field in fields])
[
stats_summary_dict.update({package: fields_dict.copy()})
for package in packages
]
if not isinstance(stats_dict, dict):
return {}
forloop_counter = 0
for __, stat_dict in stats_dict.items():
forloop_counter += 1
for pkg, detail_stat in stat_dict.items():
for field in fields:
stats_summary_dict[pkg][field] += detail_stat.get(field, 0)
for package, detail_stats_dict in stats_summary_dict.items():
try:
stats_summary_dict[package][fields[3]] = \
(detail_stats_dict.get('Untranslated', 0) /
detail_stats_dict.get('Total', 0) * 100)
except ZeroDivisionError:
# log error, pass for now
pass
return stats_summary_dict
def get_workload_estimate(self, release_branch, locale=None):
"""
Build list of packages with translation workload for a given branch
"""
specific_locale_present = True if locale else False
relbranch_pkgs_stats_dict = self._get_branch_specific_pkgs_stats(
release_branch)
relbranch_pkgs = sorted(list(relbranch_pkgs_stats_dict.keys()))
locales = self.package_manager.get_relbranch_locales(release_branch) \
if not specific_locale_present else [locale]
required_stats = OrderedDict()
HEADERS = ('Total', 'Translated', 'Untranslated', 'Remaining')
for locale in locales:
required_stats[locale] = OrderedDict()
locale_alias = self.package_manager.get_locale_alias(locale)
for package in relbranch_pkgs:
pkg_branch_stats = PackageBranchMapping(package).branch_stats(
release_branch)
for stat in pkg_branch_stats.get('stats', []):
locale_found = stat.get('locale')
if (locale_found in locale or locale_found in locale_alias
or locale_found.replace('-', '_') in locale
or locale_found.replace('-', '_') in locale_alias):
temp_stat_field = OrderedDict()
temp_stat_field['Total'] = stat.get('total', 0)
temp_stat_field['Translated'] = stat.get(
'translated', 0)
temp_stat_field['Untranslated'] = stat.get(
'untranslated', 0)
remaining = 0
try:
remaining = (stat.get('untranslated') /
stat.get('total')) * 100
except ZeroDivisionError:
# log error, pass for now
pass
temp_stat_field['Remaining'] = remaining
required_stats[locale][package] = temp_stat_field
required_stats_dict = required_stats.get(locale, {}) \
if specific_locale_present else \
self._process_workload_combined_view(relbranch_pkgs, required_stats, HEADERS)
return HEADERS, OrderedDict(
sorted(required_stats_dict.items(),
key=lambda x: x[1]['Remaining'],
reverse=True))
def get_workload_combined(self, release_branch):
"""
Build list of packages with translation workload for a given branch in all languages
"""
return self.get_workload_estimate(release_branch)
def get_workload_detailed(self, release_branch):
"""
Build translation workload percentage for a given branch in all languages
"""
relbranch_packages_stats = self._get_branch_specific_pkgs_stats(
release_branch)
locale_lang_tuple = self.package_manager.get_locale_lang_tuple(
locales=self.package_manager.get_relbranch_locales(release_branch))
headers = sorted([lang for locale, lang in locale_lang_tuple])
# Format data to fill table
workload_combined = OrderedDict()
for package, lang_stats in relbranch_packages_stats.items():
temp_stat_list = []
for lang_stat_tuple in lang_stats:
temp_stat_list.insert(headers.index(lang_stat_tuple[0]),
lang_stat_tuple[1])
# flag incorrect branch mapping
if len([i for i in temp_stat_list
if i == 0]) == len(temp_stat_list):
package += "*"
workload_combined[package] = temp_stat_list
return headers, OrderedDict(sorted(workload_combined.items()))
def get_workload_combined_detailed(self, release_branch):
"""
Build translation workload for a given branch in all langs for all pkgs
:param release_branch: str
:return: dict
"""
if not isinstance(release_branch, str):
return {}
locale_lang_tuple = self.package_manager.get_locale_lang_tuple(
locales=self.package_manager.get_relbranch_locales(release_branch))
workload_combined_detailed = {}
for locale, lang in locale_lang_tuple:
workload_combined_detailed[lang] = \
self.get_workload_estimate(release_branch, locale=locale)[1]
return workload_combined_detailed
