def tag_build_tags(buildsys, product): return_value = OrderedDict() package_manager = PackagesManager() tags = package_manager.get_build_tags(buildsys=buildsys, product_slug=product) return_value.update(dict(build_tags=tags)) return return_value
def task_sync_packages_with_platform(): """ sync all packages with translation platform """ package_manager = PackagesManager() reports_manager = ReportsManager() def _sync_package(pkg): package_manager.sync_update_package_stats(pkg) all_packages = package_manager.get_packages().filter( platform_last_updated__lte=timezone.now() - timedelta(hours=6)).order_by('platform_url') for package in all_packages: th = threading.Thread(target=_sync_package, args=(package.package_name, )) th.start() th.join() time.sleep(4) logger.info("%s Packages sync'd with Translation Platform" % len(all_packages)) if reports_manager.analyse_releases_status(): logger.info("Releases Summary Updated") if reports_manager.analyse_packages_status(): logger.info("Packages Summary Updated")
def tag_job_form(template_type): return_value = OrderedDict() job_template_manager = JobTemplateManager() filter_kwargs = {} if template_type in TS_JOB_TYPES: filter_kwargs['job_template_type'] = template_type templates = job_template_manager.get_job_templates(**filter_kwargs) if templates and len(templates) > 0: if len(templates) == 1: return_value['job_template'] = templates[0] return_value['yml_file'] = yaml.dump( templates[0].job_template_json, default_flow_style=False).replace("\'", "") return_value['job_params'] = templates[0].job_template_params return_value['job_templates'] = templates.values() package_manager = PackagesManager() release_streams = \ package_manager.get_release_streams( only_active=True, fields=('product_build_system',) ) available_build_systems = {} for relstream in release_streams: available_build_systems.update( {relstream.product_slug: relstream.product_build_system}) if available_build_systems: return_value['build_systems'] = available_build_systems packages = package_manager.get_package_name_tuple(check_mapping=True) if packages: return_value['packages'] = packages relbranch_manager = ReleaseBranchManager() release_branches = relbranch_manager.get_relbranch_name_slug_tuple() if release_branches: return_value['releases'] = release_branches return return_value
def tag_branch_mapping(package): package_manager = PackagesManager() release_manager = ReleaseBranchManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: branch_mapping = {} if package_details.release_branch_mapping_json: branch_mapping = package_details.release_branch_mapping_json.copy() for k, v in package_details.release_branch_mapping_json.items(): branch_mapping[k]['product'] = \ release_manager.get_product_by_release(k).product_slug return_value.update({ 'package_name': package_details.package_name, 'branch_mapping': branch_mapping if branch_mapping else package_details.release_branch_mapping_json, 'mapping_lastupdated': package_details.release_branch_map_last_updated, 'mapping_keys': BRANCH_MAPPING_KEYS }) return return_value
def clean(self): """ Check if the package name exist on the selected translation platform, if not add error message for package_name """ cleaned_data = super().clean() package_name = cleaned_data['package_name'] platform_slug = getattr(cleaned_data['platform_slug'], 'platform_slug', None) platform_url = cleaned_data['platform_url'] packages_manager = PackagesManager() pkg_platform_url, _ = packages_manager.get_project_details( cleaned_data['platform_slug'], package_name) if not platform_url == pkg_platform_url: cleaned_data['platform_url'] = pkg_platform_url validate_package = packages_manager.validate_package( package_name=package_name, transplatform_slug=platform_slug) if not validate_package: self.add_error('package_name', "Not found at selected translation platform") else: old_platform_engine = getattr( getattr(getattr(self, 'instance', None), 'platform_slug', None), 'engine_name', None) if old_platform_engine: packages_manager.syncstats_manager.toggle_visibility( package_name, stats_source=old_platform_engine) new_platform_engine = packages_manager.get_engine_from_slug( platform_slug) if new_platform_engine: packages_manager.syncstats_manager.toggle_visibility( package_name, visibility=True, stats_source=new_platform_engine)
def tag_job_form(template_type): return_value = OrderedDict() job_template_manager = JobTemplateManager() filter_kwargs = {} if template_type in TS_JOB_TYPES: filter_kwargs['job_template_type'] = template_type templates = job_template_manager.get_job_templates(**filter_kwargs) if templates and len(templates) > 0: if len(templates) == 1: return_value['job_template'] = templates[0] return_value['yml_file'] = yaml.dump( templates[0].job_template_json, default_flow_style=False ).replace("\'", "") return_value['job_params'] = templates[0].job_template_params return_value['job_templates'] = templates.values() package_manager = PackagesManager() release_streams = \ package_manager.get_release_streams( only_active=True, fields=('product_build_system',) ) available_build_systems = [] for relstream in release_streams: available_build_systems.append(relstream.product_build_system) if available_build_systems: return_value['build_systems'] = available_build_systems packages = package_manager.get_package_name_tuple(check_mapping=True) if packages: return_value['packages'] = packages relbranch_manager = ReleaseBranchManager() release_branches = relbranch_manager.get_relbranch_name_slug_tuple() if release_branches: return_value['releases'] = release_branches return return_value
def tag_sync_from_coverage(stats, package, release, tag): return_value = OrderedDict() if not isinstance(stats, str): return return_value if isinstance(stats, str) and not stats.startswith('Not Synced with'): return return_value package_manager = PackagesManager() release_manager = ReleaseBranchManager() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: branch_mapping = {} if package_details.release_branch_mapping_json: branch_mapping = package_details.release_branch_mapping_json.copy() if release in branch_mapping: branch_mapping = branch_mapping.get(release) branch_mapping['product'] = \ release_manager.get_product_by_release(release).product_slug return_value.update( dict( mapping=branch_mapping, package=package, tag=tag, )) return return_value
def tag_build_tags(buildsys): return_value = OrderedDict() package_manager = PackagesManager() tags = package_manager.get_build_tags(buildsys=buildsys) return_value.update(dict( build_tags=tags )) return return_value
def tag_outofsync_packages(): return_value = OrderedDict() package_manager = PackagesManager() all_packages = package_manager.get_packages() outofsync_packages = \ [i.package_name for i in all_packages if not i.stats_diff_health] if all_packages and outofsync_packages: return_value["insync_packages"] = \ (all_packages.count() - len(outofsync_packages)) or 0 return_value["outofsync_packages"] = len(outofsync_packages) or 0 return_value["total_packages"] = all_packages.count() or 0 return return_value
def tag_packages_summary(): return_value = OrderedDict() package_manager = PackagesManager() reports_manager = ReportsManager() packages_summary = reports_manager.get_reports('packages') if packages_summary: return_value.update( dict( pkgsummary=json.loads(packages_summary.get().report_json_str), last_updated=packages_summary.get().report_updated, package_count=package_manager.count_packages(), )) return return_value
def tag_package_details(package_name, user): package_manager = PackagesManager() return_value = OrderedDict() try: package = package_manager.get_packages([package_name]).get() pkg_details = package.package_details_json if pkg_details and pkg_details.get('description'): return_value.update({'package_desc': pkg_details['description']}) except: # log event, passing for now pass else: return_value.update({'package': package, 'user': user}) return return_value
def tag_branch_mapping(package): package_manager = PackagesManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: return_value.update( {'package_name': package_details.package_name, 'branch_mapping': package_details.release_branch_mapping_json, 'mapping_lastupdated': package_details.release_branch_map_last_updated, 'mapping_keys': BRANCH_MAPPING_KEYS} ) return return_value
def tag_latest_builds(package): package_manager = PackagesManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: return_value.update({ 'package_name': package, 'latest_builds': package_details.package_latest_builds_json.copy(), 'builds_lastupdated': package_details.package_latest_builds_last_updated }) return return_value
def tag_releases_summary(): return_value = OrderedDict() reports_manager = ReportsManager() releases_summary = reports_manager.get_reports('releases') if releases_summary: report = releases_summary.get().report_json_str release_report_json = json.loads(report) if isinstance(report, str) else {} pkg_manager = PackagesManager() lang_locale_dict = {lang: locale for locale, lang in pkg_manager.get_locale_lang_tuple()} for release, summary in release_report_json.items(): if summary.get('languages'): release_report_json[release]['languages'] = \ OrderedDict(sorted(summary['languages'].items())) return_value.update(dict( relsummary=release_report_json, last_updated=releases_summary.get().report_updated, lang_locale=lang_locale_dict )) return return_value
def tag_package_details(package_name, user): package_manager = PackagesManager() return_value = OrderedDict() try: package = package_manager.get_packages([package_name]).get() pkg_details = package.package_details_json if pkg_details and pkg_details.get('description'): return_value.update( {'package_desc': pkg_details['description']} ) except: # log event, passing for now pass else: return_value.update( {'package': package, 'user': user} ) return return_value
def tag_stats_diff(package): package_manager = PackagesManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: stats_diff = package_details.stats_diff_json or {} langs_out_of_sync = {} for branch, diff in stats_diff.items(): langs_out_of_sync[branch] = {} for lang, diff_percent in diff.items(): langs_out_of_sync[branch][lang] = diff_percent return_value.update({ 'package_name': package_details.package_name, 'stats_diff': langs_out_of_sync }) return return_value
def tag_stats_diff(package): package_manager = PackagesManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: stats_diff = package_details.stats_diff_json or {} langs_out_of_sync = {} for branch, diff in stats_diff.items(): langs_out_of_sync[branch] = {} for lang, diff_percent in diff.items(): langs_out_of_sync[branch][lang] = diff_percent return_value.update( {'package_name': package_details.package_name, 'stats_diff': langs_out_of_sync} ) return return_value
def task_sync_packages_with_platform(): """ sync all packages with translation platform """ package_manager = PackagesManager() reports_manager = ReportsManager() def _sync_package(pkg): package_manager.sync_update_package_stats(pkg) package_manager.fetch_latest_builds(pkg) all_packages = [] packages_except_weblate_fedora = package_manager.get_packages().filter( platform_last_updated__lte=timezone.now() - timedelta(hours=12)).order_by('platform_url').exclude( platform_slug_id=WEBLATE_SLUGS[1]) weblate_fedora_packages = package_manager.get_packages().filter( platform_last_updated__lte=timezone.now() - timedelta(hours=60), platform_slug_id=WEBLATE_SLUGS[1]).order_by('platform_url') if packages_except_weblate_fedora: all_packages.extend(packages_except_weblate_fedora) if weblate_fedora_packages: all_packages.extend(weblate_fedora_packages) for package in all_packages: th = threading.Thread(target=_sync_package, args=(package.package_name, )) th.start() th.join() time.sleep(randrange(5, 10)) logger.info("%s Packages sync'd with Translation Platform" % len(all_packages)) if reports_manager.analyse_releases_status(): logger.info("Releases Summary Updated") if reports_manager.analyse_packages_status(): logger.info("Packages Summary Updated")
def tag_releases_summary(): return_value = OrderedDict() reports_manager = ReportsManager() releases_summary = reports_manager.get_reports('releases') if releases_summary: report = releases_summary.get().report_json_str release_report_json = json.loads(report) if isinstance(report, str) else {} pkg_manager = PackagesManager() lang_locale_dict = { lang: locale for locale, lang in pkg_manager.get_locale_lang_tuple() } for release, summary in release_report_json.items(): if summary.get('languages'): release_report_json[release]['languages'] = \ OrderedDict(sorted(summary['languages'].items())) return_value.update( dict(relsummary=release_report_json, last_updated=releases_summary.get().report_updated, lang_locale=lang_locale_dict)) return return_value
def tag_trending_languages(): return_value = OrderedDict() reports_manager = ReportsManager() releases_summary = reports_manager.get_reports('releases') release_manager = ReleaseBranchManager() latest_release = release_manager.get_latest_release() pkg_manager = PackagesManager() lang_locale_dict = { lang: locale for locale, lang in pkg_manager.get_locale_lang_tuple() } if releases_summary and latest_release: releases_summary = releases_summary.get() trending_languages = reports_manager.get_trending_languages( releases_summary.report_json, *latest_release) if trending_languages and isinstance(trending_languages, (list, tuple)): return_value["trending_languages"] = trending_languages[:9] return_value["lang_locale_dict"] = lang_locale_dict return_value["latest_release"] = latest_release return return_value
def task_sync_packages_with_build_system(): """ sync all packages with build system """ package_manager = PackagesManager() graph_manager = GraphManager() reports_manager = ReportsManager() job_template_manager = JobTemplateManager() location_manager = GeoLocationManager() def _update_diff(package): try: package_stats = graph_manager.get_trans_stats_by_package( package.package_name) graph_manager.package_manager.calculate_stats_diff( package.package_name, package_stats, package.release_branch_mapping_json) except Exception: # pass for now pass def _sync_build_system(template, params): if package_manager.is_package_build_latest(params): return t_params = template.job_template_params if len(t_params) == len(params): job_data = { field.upper(): param for field, param in zip(t_params, params) } job_data.update({ 'YML_FILE': yaml.dump(template.job_template_json, default_flow_style=False).replace("\'", "") }) job_data.update({'SCRATCH': True}) temp_path = 'false/{0}/'.format('-'.join(params)) job_manager = YMLBasedJobManager( **job_data, **{ 'params': [p.upper() for p in t_params], 'type': TS_JOB_TYPES[3] }, **{'active_user_email': '*****@*****.**'}, **{'sandbox_path': temp_path}, **{'job_log_file': temp_path + '.log'}) try: if os.path.isdir(temp_path): shutil.rmtree(temp_path) os.mkdir(temp_path) job_manager.execute_job() except Exception as e: # pass for now pass finally: shutil.rmtree(temp_path) job_template = None all_packages = package_manager.get_packages().filter( release_branch_mapping__isnull=False) job_templates = job_template_manager.get_job_templates( job_template_type=TS_JOB_TYPES[3]) if job_templates: job_template = job_templates.first() if all_packages and job_template: for package in all_packages: candidates = [] mapping = package.release_branch_mapping_json or {} for release, map_dict in mapping.items(): candidates.append((package.package_name, map_dict.get(BRANCH_MAPPING_KEYS[1]), map_dict.get(BRANCH_MAPPING_KEYS[2]))) for candidate in candidates: th = threading.Thread(target=_sync_build_system, args=( job_template, candidate, )) th.start() th.join() time.sleep(6) _update_diff(package) logger.info("%s Packages sync'd with Build System" % len(all_packages)) if reports_manager.analyse_packages_status(): logger.info("Packages Summary Updated") time.sleep(2) if reports_manager.refresh_stats_required_by_territory(): logger.info("Location Summary Updated") time.sleep(2) if location_manager.save_territory_build_system_stats(): logger.info("Territory Summary Updated")
class Command(BaseCommand): help = 'Sync packages with their respective translation platform.' graph_manager = GraphManager() package_manager = PackagesManager() reports_manager = ReportsManager() job_template_manager = JobTemplateManager() location_manager = GeoLocationManager() def _sync_package(self, pkg): self.package_manager.sync_update_package_stats(pkg) def sync_with_platform(self): all_packages = self.package_manager.get_packages().filter( platform_last_updated__lte=timezone.now() - timedelta(hours=6)).order_by('platform_url') for package in all_packages: th = threading.Thread(target=self._sync_package, args=(package.package_name, )) th.start() th.join() time.sleep(2) self.reports_manager.analyse_releases_status() self.reports_manager.analyse_packages_status() def _update_diff(self, package): try: package_stats = self.graph_manager.get_trans_stats_by_package( package.package_name) self.graph_manager.package_manager.calculate_stats_diff( package.package_name, package_stats, package.release_branch_mapping_json) except Exception: # pass for now pass def _sync_build_system(self, template, params): if self.package_manager.is_package_build_latest(params): return t_params = template.job_template_params if len(t_params) == len(params): job_data = { field.upper(): param for field, param in zip(t_params, params) } job_data.update({ 'YML_FILE': yaml.dump(template.job_template_json, default_flow_style=False).replace("\'", "") }) job_data.update({'SCRATCH': True}) temp_path = 'false/{0}/'.format('-'.join(params)) job_manager = YMLBasedJobManager( **job_data, **{ 'params': [p.upper() for p in t_params], 'type': TS_JOB_TYPES[3] }, **{'active_user_email': '*****@*****.**'}, **{'sandbox_path': temp_path}, **{'job_log_file': temp_path + '.log'}) try: if os.path.isdir(temp_path): shutil.rmtree(temp_path) os.mkdir(temp_path) job_manager.execute_job() except Exception as e: # pass for now pass finally: os.rmdir(temp_path) def sync_with_build_system(self): job_template = None all_packages = self.package_manager.get_packages().filter( release_branch_mapping__isnull=False) job_templates = self.job_template_manager.get_job_templates( job_template_type=TS_JOB_TYPES[3]) if job_templates: job_template = job_templates.first() if all_packages and job_template: for package in all_packages: candidates = [] mapping = package.release_branch_mapping_json or {} for release, map_dict in mapping.items(): candidates.append((package.package_name, map_dict.get(BRANCH_MAPPING_KEYS[1]), map_dict.get(BRANCH_MAPPING_KEYS[2]))) for candidate in candidates: th = threading.Thread(target=self._sync_build_system, args=( job_template, candidate, )) th.start() th.join() time.sleep(5) self._update_diff(package) self.reports_manager.analyse_packages_status() self.reports_manager.refresh_stats_required_by_territory() self.location_manager.save_territory_build_system_stats() def add_arguments(self, parser): # Named (optional) arguments parser.add_argument( '--platform', action='store_true', help='Sync packages with translation platform only.', ) parser.add_argument( '--build-system', action='store_true', help='Sync packages with build system only.', ) def handle(self, *args, **options): cmd_combinations_options = ['platform', 'build_system', 'default_both'] cmd_combinations = { cmd_combinations_options[0]: self.sync_with_platform, cmd_combinations_options[1]: self.sync_with_build_system, cmd_combinations_options[2]: [self.sync_with_platform, self.sync_with_build_system] } if options.get(cmd_combinations_options[0]): cmd_combinations.get(cmd_combinations_options[0])() elif options.get(cmd_combinations_options[1]): cmd_combinations.get(cmd_combinations_options[1])() else: [m() for m in cmd_combinations.get(cmd_combinations_options[2])]
class JobsLogManager(BaseManager): """ Maintains Job Logs """ package_manager = PackagesManager() def get_job_logs(self, remarks=None, result=None): """ Fetch all job logs from the db """ job_logs = None filters = {} if remarks: filters.update(dict(job_remarks=remarks)) if result: filters.update(dict(job_result=True)) try: job_logs = Job.objects.filter( **filters).order_by('-job_start_time') except: # log event, passing for now pass return job_logs def get_job_detail(self, job_id): """ Fetch just one job :param job_id: Job ID: uuid :return: Job object """ job_log = None if not job_id: return job_log try: job_log = Job.objects.filter(job_uuid=job_id).first() except: # log event, passing for now pass return job_log def get_joblog_stats(self): """ Stats about jobs log """ last_ran_on = None last_ran_type = None jobs_logs = self.get_job_logs() jobs_count = jobs_logs.count() successful_jobs = jobs_logs.filter(**{'job_result': True}) if successful_jobs.count() > 0: last_ran_on = successful_jobs[0].job_end_time last_ran_type = successful_jobs[0].job_type return jobs_count, last_ran_on, last_ran_type def analyse_job_data(self, job_data): """ Analyses job output (of a package) to emit meaningful data :param job_data: dict :return: dict or None """ if not job_data: return analysable_fields = [ 'Latest Build Details', 'Calculate Translation Stats' ] fields_to_analyse = [ field for field in job_data.keys() if field in analysable_fields ] field_heading = [ 'Language', 'Total', 'Translated', 'Fuzzy', 'Untranslated', 'Complete' ] if not fields_to_analyse and not len(fields_to_analyse) > 0: return else: try: analysed_data = {} total_no_of_messages = [] for field in fields_to_analyse: if field == analysable_fields[0]: build_details = list( job_data.get(analysable_fields[0], {}).values()) if build_details and len(build_details) > 0: build_details = load(build_details[0], Loader=FullLoader) if isinstance(build_details, dict): analysed_data.update({ 'meta_data': { 'package': build_details.get('package_name', ''), 'nvr': build_details.get('nvr', ''), 'built_on': build_details.get('completion_time', '') } }) elif field == analysable_fields[1]: stats_json_data = list( job_data.get(analysable_fields[1], {}).values()) if stats_json_data and len(stats_json_data) > 0: data = {} for json_data in stats_json_data: if '{' in json_data: data = load( json_data[json_data.find('{'):], Loader=FullLoader) stats_json = data.get('stats', {}) if stats_json and isinstance(stats_json, list): lang_id_name = self.package_manager.get_lang_id_name_dict( ) or [] locale_lang_dict = dict( self.package_manager.get_locale_lang_tuple( )) locale_key = 'locale' processed_stats = [] analysed_data['stats'] = [] for locale, l_alias in list( lang_id_name.keys()): filter_stat = [] stats_chunk = [] try: filter_stat = self.package_manager.filter_n_reduce_stats( locale_key, locale, l_alias, stats_json) except Exception as e: self.app_logger( 'ERROR', "Error while filtering stats, details: " + str(e)) else: filter_stat = filter_stat[0] \ if isinstance(filter_stat, list) and len(filter_stat) > 0 else {} stats_chunk.append( locale_lang_dict.get(locale, locale)) if filter_stat.get( 'total' ) and filter_stat.get('total') > 0: total_no_of_messages.append( filter_stat.get('total')) stats_chunk.append( filter_stat.get('total', 0)) stats_chunk.append( filter_stat.get('translated', 0)) stats_chunk.append( filter_stat.get('fuzzy', 0)) stats_chunk.append( filter_stat.get('untranslated', 0)) completion_percentage = 0 try: completion_percentage = int( (filter_stat.get('translated', 0) * 100 / filter_stat.get('total', 0))) except ZeroDivisionError: pass stats_chunk.append(completion_percentage) if stats_chunk: non_zero_stats = [ i for i in stats_chunk[1:] if i > 0 ] if non_zero_stats and len( non_zero_stats) > 0: processed_stats.append(stats_chunk) analysed_data['stats'].extend(processed_stats) analysed_data.update(dict(headings=field_heading)) except Exception as e: self.app_logger( 'ERROR', "Error while analysing job data, details: " + str(e)) return if len(set(total_no_of_messages)) > 1: analysed_data[ 'pot_differ'] = "Total number of messages differ across languages." return analysed_data
class YMLBasedJobManager(BaseManager): """ Single Entry Point for - all YML based Jobs: syncupstream, syncdownstream, stringchange - this should be the base class to handle YML Jobs """ sandbox_path = 'dashboard/sandbox/' job_log_file = sandbox_path + '.log' package_manager = PackagesManager() @staticmethod def job_suffix(args): return "-".join(args) def __init__(self, *args, **kwargs): """ Set Job Environment here """ super(YMLBasedJobManager, self).__init__(**kwargs) self.suffix = self.job_suffix( [getattr(self, param, '') for param in self.params]) self.job_log_file = self.sandbox_path + '.log' def _get_package(self): package_details = \ self.package_manager.get_packages([self.package]) return package_details.get() def _bootstrap(self, package=None, build_system=None): if build_system: try: release_streams = \ self.package_manager.get_release_streams(built=build_system) release_stream = release_streams.first() except Exception as e: self.app_logger( 'ERROR', "Release stream could not be found, details: " + str(e)) raise Exception( 'Build Server URL could NOT be located for %s.' % build_system) else: self.hub_url = release_stream.product_server if package: try: package_detail = self._get_package() except Exception as e: self.app_logger( 'ERROR', "Package could not be found, details: " + str(e)) raise Exception( 'Upstream URL could NOT be located for %s package.' % package) else: self.upstream_repo_url = package_detail.upstream_url \ if package_detail.upstream_url.endswith('.git') \ else package_detail.upstream_url + ".git" t_ext = package_detail.translation_file_ext file_ext = t_ext if t_ext.startswith('.') else '.' + t_ext self.trans_file_ext = file_ext.lower() self.pkg_upstream_name = package_detail.upstream_name self.pkg_tp_engine = package_detail.platform_slug.engine_name self.pkg_tp_url = package_detail.platform_slug.api_url self.pkg_tp_auth_usr = package_detail.platform_slug.auth_login_id self.pkg_tp_auth_token = package_detail.platform_slug.auth_token_key self.pkg_branch_map = package_detail.release_branch_mapping_json def _save_result_in_db(self, stats_dict, build_details): """ Save derived stats in db from YML Job :param stats_dict: translation stats calculated """ if not self.package_manager.is_package_exist( package_name=self.package): raise Exception("Stats NOT saved. Package does not exist.") stats_version, stats_source = '', '' if self.type == TS_JOB_TYPES[2]: stats_version, stats_source = 'Upstream', 'upstream' elif self.type == TS_JOB_TYPES[3]: stats_version, stats_source = self.buildsys + ' - ' + self.tag, self.buildsys try: self.package_manager.syncstats_manager.save_version_stats( self._get_package(), stats_version, stats_dict, stats_source) if stats_source == 'upstream': self.package_manager.update_package( self.package, {'upstream_last_updated': timezone.now()}) # If its for rawhide, update downstream sync time for the package if getattr(self, 'tag', ''): self.package_manager.update_package( self.package, {'downstream_last_updated': timezone.now()}) # If invoked by system user, cache build details if self.active_user_email == '*****@*****.**' and \ self.type == TS_JOB_TYPES[3]: cache_params = {} match_params = { 'package_name': self._get_package(), 'build_system': self.buildsys, 'build_tag': self.tag } cache_params.update(match_params) latest_build = {} if isinstance( build_details, list) and build_details and len(build_details) > 0: latest_build = build_details[0] cache_params['build_details_json_str'] = json.dumps( latest_build) try: CacheBuildDetails.objects.update_or_create( package_name=self._get_package(), build_system=self.buildsys, build_tag=self.tag, defaults=cache_params) except Exception as e: # log error pass except Exception as e: self.app_logger('ERROR', "Package could not be updated, details: " + str(e)) raise Exception('Stats could NOT be saved in db.') def _wipe_workspace(self): """ This makes sandbox clean for a new job to run """ # remove log file if exists if os.path.exists(self.job_log_file): os.remove(self.job_log_file) for file in os.listdir(self.sandbox_path): file_path = os.path.join(self.sandbox_path, file) try: if os.path.isdir(file_path): shutil.rmtree(file_path) elif os.path.isfile(file_path) and not file_path.endswith('.py') \ and '.log.' not in file_path: os.unlink(file_path) except Exception as e: pass def execute_job(self): """ 1. PreProcess YML and replace variables with input_values - Example: %PACKAGE_NAME% 2. Parse processed YML and build Job object 3. Select data structure for tasks and instantiate one - Example: Linked list should be used for sequential tasks 3. Discover namespace and method for each task and fill in TaskNode 4. Perform actions (execute tasks) and return responses """ self._wipe_workspace() yml_preprocessed = YMLPreProcessor( self.YML_FILE, **{param: getattr(self, param, '') for param in self.params}).output self.job_base_dir = os.path.dirname(settings.BASE_DIR) yml_job = YMLJobParser(yml_stream=io.StringIO(yml_preprocessed)) try: self.yml_job_name = yml_job.job_name except AttributeError: raise Exception('Input YML could not be parsed.') self.package = yml_job.package self.buildsys = yml_job.buildsys if isinstance(yml_job.tags, list) and len(yml_job.tags) > 0: self.tag = yml_job.tags[0] elif isinstance(yml_job.tags, str): self.tag = yml_job.tags if self.type != yml_job.job_type: raise Exception('Selected job type differs to that of YML.') if (self.type == TS_JOB_TYPES[2] or self.type == TS_JOB_TYPES[5]) and self.package: self._bootstrap(package=self.package) self.release = yml_job.release elif self.type == TS_JOB_TYPES[3] and self.buildsys: self._bootstrap(build_system=self.buildsys) # for sequential jobs, tasks should be pushed to linked list # and output of previous task should be input for next task if JOB_EXEC_TYPES[0] in yml_job.execution: self.tasks_ds = TaskList() else: raise Exception('%s exec type is NOT supported yet.' % yml_job.execution) # lets create a job job_manager = JobManager(self.type) if job_manager.create_job(user_email=self.active_user_email): self.job_id = job_manager.uuid job_manager.job_remarks = self.package # and set tasks tasks = yml_job.tasks for task in tasks: self.tasks_ds.add_task(task) log_file = self.job_log_file + ".%s.%s" % (self.suffix, self.type) if os.path.exists(log_file): os.unlink(log_file) action_mapper = ActionMapper(self.tasks_ds, self.job_base_dir, getattr(self, 'tag', ''), getattr(self, 'package', ''), getattr(self, 'hub_url', ''), getattr(self, 'buildsys', ''), getattr(self, 'release', ''), getattr(self, 'upstream_repo_url', ''), getattr(self, 'trans_file_ext', ''), getattr(self, 'pkg_upstream_name', ''), getattr(self, 'pkg_branch_map', {}), getattr(self, 'pkg_tp_engine', ''), getattr(self, 'pkg_tp_auth_usr', ''), getattr(self, 'pkg_tp_auth_token', ''), getattr(self, 'pkg_tp_url', ''), log_file) action_mapper.set_actions() # lets execute collected tasks try: action_mapper.execute_tasks() except Exception as e: job_manager.job_result = False raise Exception(e) else: job_manager.output_json = action_mapper.result job_manager.job_params.update( {param: getattr(self, param, '') for param in self.params}) job_template_manager = JobTemplateManager() templates = job_template_manager.get_job_templates( **{'job_template_type': self.type}) if templates and len(templates) > 0: template_obj = templates.first() job_manager.job_template = template_obj job_manager.visible_on_url = True job_manager.job_result = True finally: job_manager.job_yml = yml_preprocessed job_manager.log_json = action_mapper.log action_mapper.clean_workspace() if not getattr(self, 'SCRATCH', None): job_manager.mark_job_finish() else: job_manager.mark_job_finish(remove=True) time.sleep(4) # if not a dry run, save results is db if action_mapper.result and not getattr(self, 'DRY_RUN', None): self._save_result_in_db(action_mapper.result, action_mapper.build) if os.path.exists(log_file): os.unlink(log_file) return self.job_id
class GraphManager(BaseManager): """ Manage graph representations """ package_manager = PackagesManager() branch_manager = ReleaseBranchManager() def get_graph_rules(self, graph_rule=None, only_active=None): """ Fetch graph rules from db :return: resultset """ filter_kwargs = {} if graph_rule: filter_kwargs.update(dict(rule_name=graph_rule)) if only_active: filter_kwargs.update(dict(rule_status=True)) rules = None try: rules = GraphRule.objects.filter( **filter_kwargs).order_by('rule_name') except: # log event, passing for now pass return rules def slugify_graph_rule_name(self, suggested_name): try: return slugify(suggested_name) except: # log even, passing for now return False def validate_package_branch_participation(self, relbranch, packages): """ This validates that packages belong to relbranch or not :param relbranch: release branch :param packages: list of packages :return: list of packages that DO NOT belong """ if not relbranch and not packages: return pkg_not_participate = [] for package in packages: relbranches = PackageBranchMapping(package).release_branches if relbranch not in relbranches: pkg_not_participate.append(package) return pkg_not_participate def validate_tags_product_participation(self, release_slug, tags): """ This validates that tag belongs to release :param release_slug: Release Slug :param tags: Build Tags :return: List of tags that do not belong """ if not release_slug and not tags: return tags_not_participate = [] q_release = self.branch_manager.get_release_branches( relbranch=release_slug) if q_release: release = q_release.get() release_build_tags = release.product_slug.product_build_tags for tag in tags: if tag not in release_build_tags: tags_not_participate.append(tag) return tags_not_participate def add_graph_rule(self, **kwargs): """ Save graph rule in db :param kwargs: dict :return: boolean """ if not kwargs.get('rule_name'): return if not kwargs.get('rule_relbranch'): return relbranch_slug = kwargs.get('rule_relbranch') release_branch = \ self.branch_manager.get_release_branches(relbranch=relbranch_slug) if kwargs.get('tags_selection' ) == "pick" and not kwargs.get('rule_build_tags'): release_packages = \ self.package_manager.get_relbranch_specific_pkgs( relbranch_slug, fields=['release_branch_mapping'] ) package_tags = [ package.release_branch_mapping_json[relbranch_slug][ BRANCH_MAPPING_KEYS[2]] for package in release_packages if package.release_branch_mapping_json and package.release_branch_mapping_json.get( relbranch_slug, {}).get(BRANCH_MAPPING_KEYS[2]) ] if package_tags: kwargs['rule_build_tags'] = list(set(package_tags)) else: return False if kwargs.get( 'lang_selection') == "pick" and not kwargs.get('rule_langs'): relbranch_lang_set = self.branch_manager.get_release_branches( relbranch=relbranch_slug, fields=['language_set_slug']).first() locales = relbranch_lang_set.language_set_slug.locale_ids if locales: kwargs['rule_languages'] = locales else: return False elif kwargs.get('rule_langs'): kwargs['rule_languages'] = kwargs.pop('rule_langs') filters = [ 'tags_selection', 'lang_selection', 'rule_langs', 'rule_relbranch' ] [kwargs.pop(i) for i in filters if i in kwargs] try: kwargs['rule_release_slug'] = release_branch.get() kwargs['created_on'] = timezone.now() kwargs['rule_status'] = True new_rule = GraphRule(**kwargs) new_rule.save() except: # log event, pass for now # todo implement error msg handling return False else: return True def _normalize_stats(self, stats_nested_list, index_list): """ Normalize stats for a locale index and picks higher value """ temp_index_list = [] temp_stat_list = [] for index, stat in stats_nested_list: if index not in temp_index_list: temp_index_list.append(index) temp_stat_list.append(stat) else: last_stat = temp_stat_list.pop(len(temp_stat_list) - 1) temp_stat_list.append(last_stat) \ if last_stat > stat else temp_stat_list.append(stat) expected_stats_list = list(zip(temp_index_list, temp_stat_list)) if len(index_list) > len(expected_stats_list): expected_stats_dict = dict( (k[0], k[1:]) for k in expected_stats_list) temp_patched_stats_list = [] for index in index_list: temp_patched_stats_list.append( [index, expected_stats_dict.get(index, [0.0])[0]]) expected_stats_list = temp_patched_stats_list return expected_stats_list def _format_stats_for_default_graphs(self, locale_sequence, stats_dict, desc, prepend_source=False): """ Formats stats dict for graph-ready material - sorting and normalization """ stats_for_graphs_dict = OrderedDict() stats_for_graphs_dict['pkg_desc'] = desc stats_for_graphs_dict['ticks'] = \ [[i, lang] for i, lang in enumerate(locale_sequence.values(), 0)] indexes = [index for index, lang in stats_for_graphs_dict['ticks']] graph_data_dict = {} for version, stats_lists in stats_dict.items(): new_stats_list = [] for stats_tuple in stats_lists: index = [ i for i, locale_tuple in enumerate(list(locale_sequence), 0) if (stats_tuple[0] in locale_tuple) or ( stats_tuple[0].replace('-', '_') in locale_tuple) or ( stats_tuple[0].replace('_', '-') in locale_tuple) ] if index: index.append(stats_tuple[1] or 0.0) new_stats_list.append(index) if prepend_source: if stats_tuple[0] == 'source' and stats_tuple[ 1] not in version.lower(): version = "{0} - {1}".format(stats_tuple[1], version) normalized_stats = self._normalize_stats(sorted(new_stats_list), indexes) if len(list(filter(lambda x: x[1] > 0.0, normalized_stats))) > 0: graph_data_dict[version] = normalized_stats stats_for_graphs_dict['graph_data'] = OrderedDict( sorted(graph_data_dict.items())) return stats_for_graphs_dict def get_trans_stats_by_package(self, package, prepend_source=False): """ formats stats of a package for all enabled languages :param package: str :param prepend_source: boolean :return: Graph data for "Package-wise" view: dict """ if not package: return {} lang_id_name, stats_dict, pkg_desc = self.package_manager.get_trans_stats( package) # format trans_stats_list for graphs return self._format_stats_for_default_graphs(lang_id_name, stats_dict, pkg_desc, prepend_source) def _format_stats_for_lang_wise_graphs(self, input_locale, locale_sequence, stats_dict, desc): """ Formats stats dict for bar graph-ready material """ stats_for_graphs_dict = OrderedDict() stats_branches = list(stats_dict.keys()) stats_for_graphs_dict['pkg_desc'] = desc stats_for_graphs_dict['ticks'] = \ [[i, version] for i, version in enumerate(stats_branches, 0)] stats_for_graphs_dict['graph_data'] = [] for version, stats_lists in stats_dict.items(): index = stats_branches.index(version) required_stat = 0.0 for locale_tuple in list(locale_sequence.keys()): if input_locale in locale_tuple: locale, alias = locale_tuple expected_stat = [ stat for lang, stat in stats_lists if lang.replace('-', '_') == locale or lang.replace('-', '_') == alias ] required_stat = expected_stat[0] if expected_stat and len( expected_stat) > 0 else 0.0 stats_for_graphs_dict['graph_data'].append([required_stat, index]) return stats_for_graphs_dict def get_stats_by_pkg_per_lang(self, package, locale): """ formats stats of a package for given locale :param package: str :param locale: str :return: Graph data for "Language-wise" view: dict """ if not package and not locale: return {} lang_id_name, stats_dict, pkg_desc = self.package_manager.get_trans_stats( package) # format stats for lang-wise graph return self._format_stats_for_lang_wise_graphs(locale, lang_id_name, stats_dict, pkg_desc) def get_trans_stats_by_rule(self, graph_rule): """ formats stats of a graph rule :param: graph_rule: str :return: Graph data for "Rule-wise" view: dict """ rule_data = {} rule = self.get_graph_rules(graph_rule=graph_rule).get() packages = rule.rule_packages locales = rule.rule_languages tags = rule.rule_build_tags release = rule.rule_release_slug_id if rule: rule_data = self.package_manager.get_trans_stats_by_rule(rule) return rule_data, len(packages), len(locales), len(tags), release def _consolidate_branch_specific_stats(self, packages_stats_dict): """ Sum up stats per language """ temp_stats_dict = {} pkgs_stats_list = list(packages_stats_dict.values()) pkgs_length = len(pkgs_stats_list) for pkg_stats in pkgs_stats_list: for pkg_stat in pkg_stats: if pkg_stat[0] not in temp_stats_dict: temp_stats_dict[pkg_stat[0]] = pkg_stat[1] else: temp_stats_dict[pkg_stat[0]] += pkg_stat[1] # Reverse stats to depict how much is left return sorted([(i, 100 - int(j / pkgs_length)) for i, j in temp_stats_dict.items()]) def _format_data_for_pie_chart(self, consolidated_stats, lang_options): """ Takes consolidated stats and formats for pie chart """ formatted_stats = [] formatted_langs = [] for lang, stat in consolidated_stats: formatted_stats.append({'label': lang, 'data': stat}) for locale, language in lang_options: formatted_langs.append({'value': locale, 'text': language}) return { 'graph_data': formatted_stats, 'select_options': sorted(formatted_langs, key=itemgetter('text')) } def _get_branch_specific_pkgs_stats(self, relbranch): """ Generates translation stats of all packages in all langs of attached lang-set """ specific_pkgs = [ package.package_name for package in self.package_manager.get_relbranch_specific_pkgs( relbranch, ['package_name']) ] all_pkgs_stats_dict = OrderedDict() for pkg in specific_pkgs: pkg_lang_stats = [] locale_seq, trans_stats_dict, pkg_desc = \ self.package_manager.get_trans_stats(pkg, apply_branch_mapping=True, specify_branch=relbranch) t_stats = self._format_stats_for_default_graphs( locale_seq, trans_stats_dict, pkg_desc) branch_stats = t_stats.get('graph_data').get(relbranch) langs = t_stats.get('ticks') if branch_stats and langs: pkg_lang_stats.extend( list( zip([lang[1] for lang in langs], [stat[1] for stat in branch_stats]))) all_pkgs_stats_dict[pkg] = pkg_lang_stats return all_pkgs_stats_dict def get_workload_graph_data(self, release_branch): """ Build or generates workload graph data """ consolidated_stats = self._consolidate_branch_specific_stats( self._get_branch_specific_pkgs_stats(release_branch)) # get branch specific languages for select option locale_lang_tuple = self.package_manager.get_locale_lang_tuple( locales=self.package_manager.get_relbranch_locales(release_branch)) return self._format_data_for_pie_chart(consolidated_stats, locale_lang_tuple) def _process_workload_combined_view(self, packages_stats_dict, headers): stats_summary_dict = OrderedDict() for package, locale_stats in packages_stats_dict.items(): reduced_stats = {} try: reduced_stats = functools.reduce( lambda x, y: dict(Counter(x) + Counter(y)), list(locale_stats.values())) except Exception as e: # log error, pass for now pass if not reduced_stats: for field in headers: reduced_stats[field] = 0 try: if not reduced_stats.get('Untranslated'): reduced_stats['Untranslated'] = 0 reduced_stats[headers[3]] = \ (reduced_stats['Untranslated'] / reduced_stats.get('Total', 0) * 100) except ZeroDivisionError: # log error, pass for now pass stats_summary_dict[package] = reduced_stats return stats_summary_dict def get_workload_estimate(self, release_branch, locale=None): """ Build list of packages with translation workload for a given branch """ headers = WORKLOAD_HEADERS pkg_stats = self.package_manager.get_release_specific_package_stats( release_branch=release_branch) required_stats_dict = {} if not locale: required_stats_dict = self._process_workload_combined_view( pkg_stats, headers) elif isinstance(locale, str): for pkg, locale_stat in pkg_stats.items(): required_stats_dict[pkg] = locale_stat.get(locale) or { header: 0 for header in headers } return headers, OrderedDict( sorted(required_stats_dict.items(), key=lambda x: x[1]['Remaining'], reverse=True)) def get_workload_combined(self, release_branch): """ Build list of packages with translation workload for a given branch in all languages """ return self.get_workload_estimate(release_branch) def get_workload_detailed(self, release_branch): """ Build translation workload percentage for a given branch in all languages """ relbranch_packages_stats = self._get_branch_specific_pkgs_stats( release_branch) locale_lang_tuple = self.package_manager.get_locale_lang_tuple( locales=self.package_manager.get_relbranch_locales(release_branch)) headers = sorted([lang for locale, lang in locale_lang_tuple]) # Format data to fill table workload_combined = OrderedDict() for package, lang_stats in relbranch_packages_stats.items(): temp_stat_list = [] for lang_stat_tuple in lang_stats: temp_stat_list.insert(headers.index(lang_stat_tuple[0]), lang_stat_tuple[1]) # flag incorrect branch mapping if len([i for i in temp_stat_list if i == 0]) == len(temp_stat_list): package += "*" if temp_stat_list: workload_combined[package] = temp_stat_list return headers, OrderedDict(sorted(workload_combined.items())) def get_workload_combined_detailed(self, release_branch): """ Build translation workload for a given branch in all langs for all pkgs :param release_branch: str :return: dict """ if not isinstance(release_branch, str): return {} locale_lang_tuple = self.package_manager.get_locale_lang_tuple( locales=self.package_manager.get_relbranch_locales(release_branch)) workload_combined_detailed = {} for locale, lang in locale_lang_tuple: workload_combined_detailed[lang] = \ self.get_workload_estimate(release_branch, locale=locale)[1] return workload_combined_detailed def get_threshold_based(self, release_branch, threshold=70): """ Build language list those have fulfilled given threshold :param release_branch: str :param threshold: translation %age margin: int :return: dict """ consolidated_stats = self._consolidate_branch_specific_stats( self._get_branch_specific_pkgs_stats(release_branch)) # Reverse the stats to have - what has been covered consolidated_stats_reversed = [(lang, 100 - stat) for lang, stat in consolidated_stats] filtered_stats = list( filter(lambda elem: elem[1] > threshold, consolidated_stats_reversed)) headers = ['Languages', 'Translation Complete %age'] locale_lang_tuple = self.package_manager.get_locale_lang_tuple( locales=self.package_manager.get_relbranch_locales(release_branch)) lang_locale_dict = {v: k for k, v in dict(locale_lang_tuple).items()} return headers, filtered_stats, lang_locale_dict
class ReportsManager(GraphManager): """ Manage Reports Generations """ package_manager = PackagesManager() def get_reports(self, report_subject): """ Fetch reports from db :return: Queryset """ filter_kwargs = {} if report_subject: filter_kwargs.update(dict(report_subject=report_subject)) reports = None try: reports = Report.objects.filter(**filter_kwargs) except Exception as e: self.app_logger('ERROR', "Reports could not be fetched, details: " + str(e)) return reports def create_or_update_report(self, **kwargs): """ Creates or Updates a report :param kwargs: dict :return: boolean """ if not kwargs.get('subject') and not kwargs.get('report_json'): return default_params = {} match_params = {'report_subject': kwargs['subject']} default_params.update(match_params) default_params['report_json_str'] = json.dumps(kwargs['report_json']) default_params['report_updated'] = timezone.now() try: Report.objects.update_or_create(report_subject=kwargs['subject'], defaults=default_params) except Exception as e: # log error return False else: return True def _filter_disabled_languages(self, lang_stats_dict): active_locales = self.package_manager.get_locales(only_active=True) active_languages = [locale.lang_name for locale in active_locales] return { k: v for k, v in lang_stats_dict.items() if k in active_languages } def analyse_releases_status(self): """ Summarize Releases Status """ relbranches = self.branch_manager.get_relbranch_name_slug_tuple() relbranch_report = {} for branch_slug, branch_name in relbranches: relbranch_report[branch_name] = {} relbranch_report[branch_name]['slug'] = branch_slug stats_estimate = self.get_workload_estimate(branch_slug) untranslated_messages = \ [stats.get('Untranslated') for pkg, stats in stats_estimate[1].items()] if untranslated_messages: packages_need_attention = (len(untranslated_messages) - untranslated_messages.count(0)) or 0 relbranch_report[branch_name][ 'packages_need_attention'] = packages_need_attention total_untranslated_msgs = (functools.reduce( (lambda x, y: x + y), untranslated_messages)) or 0 relbranch_report[branch_name][ 'total_untranslated_msgs'] = total_untranslated_msgs lang_stats_report = self._filter_disabled_languages( self.get_workload_combined_detailed(branch_slug)) relbranch_report[branch_name]['languages'] = {} for lang, pkg_stats in lang_stats_report.items(): untranslated_msgs = [] untranslated_msgs.extend([ stats.get('Untranslated') for pkg, stats in pkg_stats.items() ]) total_untranslated_msgs = (functools.reduce( (lambda x, y: x + y), untranslated_msgs)) or 0 translated_msgs = [] translated_msgs.extend([ stats.get('Translated') for pkg, stats in pkg_stats.items() ]) total_translated_msgs = (functools.reduce( (lambda x, y: x + y), translated_msgs)) or 0 msgs = [] msgs.extend([ stats.get('Total') for pkg, stats in pkg_stats.items() ]) total_msgs = (functools.reduce( (lambda x, y: x + y), msgs)) or 0 # 0: untranslated, 1: translated, 2: total relbranch_report[branch_name]['languages'][lang] = ( total_untranslated_msgs, total_translated_msgs, total_msgs) if self.create_or_update_report(**{ 'subject': 'releases', 'report_json': relbranch_report }): return OrderedDict(sorted(relbranch_report.items(), reverse=True)) return False def analyse_packages_status(self): """ Summarize Packages Status """ all_packages = self.package_manager.get_packages(pkg_params=[ 'package_name', 'products', 'details_json_last_updated', 'stats_diff', 'release_branch_mapping', 'platform_last_updated', 'upstream_last_updated' ]) pkg_tracking_for_RHEL = all_packages.filter( products__icontains=RELSTREAM_SLUGS[0]).count() pkg_tracking_for_fedora = all_packages.filter( products__icontains=RELSTREAM_SLUGS[1]).count() pkg_details_week_old = all_packages.filter( details_json_last_updated__lte=timezone.now() - timezone.timedelta(days=7)).count() pkg_transtats_week_old = all_packages.filter( platform_last_updated__lte=timezone.now() - timezone.timedelta(days=7)).count() pkg_upstream_week_old = all_packages.filter( upstream_last_updated__lte=timezone.now() - timezone.timedelta(days=7)).count() relbranches = self.branch_manager.get_relbranch_name_slug_tuple() pkg_improper_branch_mapping = 0 pkg_with_stats_diff = [] if relbranches and len(relbranches) > 0: relbranch_slugs = sorted([slug for slug, name in relbranches], reverse=True) pkgs_improper_branch_mapping = [ i.package_name for i in all_packages if not i.release_branch_mapping_health ] pkg_improper_branch_mapping = len(pkgs_improper_branch_mapping) pkg_with_stats_diff = [ i.package_name for i in all_packages if not i.stats_diff_health ] package_report = { RELSTREAM_SLUGS[0]: pkg_tracking_for_RHEL or 0, RELSTREAM_SLUGS[1]: pkg_tracking_for_fedora or 0, 'pkg_details_week_old': pkg_details_week_old or 0, 'pkg_transtats_week_old': pkg_transtats_week_old or 0, 'pkg_upstream_week_old': pkg_upstream_week_old or 0, 'pkg_improper_branch_mapping': pkg_improper_branch_mapping or 0, 'pkg_with_stats_diff': pkg_with_stats_diff, 'pkg_having_stats_diff': len(pkg_with_stats_diff) or 0 } if self.create_or_update_report(**{ 'subject': 'packages', 'report_json': package_report }): return package_report return False def refresh_stats_required_by_territory(self): """ This refreshes statistics which is required by territory - this includes: - all languages (both disabled and enabled) - build system stats where sync_visibility is True - both for translation platform and build system :return: master_statistics or False """ all_locales = self.package_manager.get_locales() all_releases = self.branch_manager.get_release_branches() platform_release_stats_report = self.get_reports( report_subject='releases') if not platform_release_stats_report: return platform_release_stats_report = platform_release_stats_report.get() # Create basic skeleton master_statistics = {} for locale in all_locales: master_statistics[locale.locale_id] = {} master_statistics[locale.locale_id].update( {'language': locale.lang_name}) for release in all_releases: if release.track_trans_flag: master_statistics[locale.locale_id].update({ release.release_slug: { 'Release Name': release.release_name, 'Translation Platform': [], 'Build System': [] } }) lang_locale_dict = { locale.lang_name: locale.locale_id for locale in all_locales } # Let's fill master_statistics with platform_release_stats_report for release, data in platform_release_stats_report.report_json.items(): release_slug = data.get('slug') language_stats = data.get('languages') if release_slug and language_stats: for language, stats in language_stats.items(): locale = lang_locale_dict.get(language) if 'Translation Platform' in master_statistics.get( locale, {}).get(release_slug, {}): master_statistics[locale][release_slug][ 'Translation Platform'] = stats # Now, fill the build system stats build_system_stats = self.package_manager.get_build_system_stats_by_release( ) for b_release, locale_stats in build_system_stats.items(): for b_locale, b_stats in locale_stats.items(): if 'Build System' in master_statistics.get(b_locale, {}).get( b_release, {}): master_statistics[b_locale][b_release]['Build System'] = [ b_stats.get('Untranslated') or 0, b_stats.get('Translated') or 0, b_stats.get('Total') or 0 ] if self.create_or_update_report(**{ 'subject': 'location', 'report_json': master_statistics }): return master_statistics return False @staticmethod def get_trending_languages(release_summary_data, *release_tuple): """ Get Trending Languages Based on statistics of the latest releases. :param release_summary_data: dict :param release_tuple: tuple :return: dict: {language: average percentage} """ if not release_summary_data: return {} trending_languages = [] lang_stats_data = release_summary_data.get(release_tuple[1], {}).get('languages') if not lang_stats_data: return {} try: for lang, stats in lang_stats_data.items(): try: percent = round((stats[1] * 100) / stats[2], 2) except (ZeroDivisionError, FloatingPointError): percent = 0 trending_languages.append((lang, percent, stats)) except Exception as e: # log for now return {} if trending_languages: return sorted(trending_languages, key=lambda x: x[1], reverse=True)
class PackagesManagerTest(FixtureTestCase): packages_manager = PackagesManager() fixture = db_fixture datasets = [PackageData] def test_get_packages(self): """ Test get_packages """ packages = self.packages_manager.get_packages() self.assertEqual(len(packages), 4) package_names = [ PackageData.package_anaconda.package_name, PackageData.package_ibus.package_name ] packages = self.packages_manager.get_packages( pkgs=package_names).values() self.assertEqual(len(packages), 2) self.assertEqual(packages[0]['package_name'], PackageData.package_anaconda.package_name) self.assertEqual(packages[1]['package_name'], PackageData.package_ibus.package_name) # todo: test the filtering according to params # params = ['package_name', 'upstream_url'] # packages = self.packages_manager.get_packages(pkgs=['ibus'], pkg_params=params) # self.assertTrue(set(params).issubset(vars(packages.get()).keys())) def test_is_package_exist(self): """ Test is_package_exist """ self.assertTrue( self.packages_manager.is_package_exist( PackageData.package_anaconda.package_name)) self.assertFalse( self.packages_manager.is_package_exist('otherpackage')) @patch('requests.get', new=mock_requests_get_add_package) def test_add_package(self): """ Test add_package """ transplatform = PlatformData.platform_zanata_fedora.platform_slug kwargs = { 'package_name': 'authconfig', 'upstream_url': 'https://github.com/jcam/authconfig', 'transplatform_slug': transplatform, 'release_streams': ['fedora'] } package_added = self.packages_manager.add_package(**kwargs) self.assertTrue(package_added) self.assertTrue(self.packages_manager.is_package_exist('authconfig')) package_added = self.packages_manager.add_package(**kwargs) self.assertFalse(package_added) @patch('requests.get', new=mock_requests_get_validate_package) def test_validate_package(self): """ Test validate_package """ transplatform = PlatformData.platform_zanata_public.platform_slug package_candlepin_name = PackageData.package_candlepin.package_name package_validated = self.packages_manager.validate_package( package_name=package_candlepin_name, transplatform_slug=transplatform) self.assertEqual(package_validated, package_candlepin_name) package_validated = self.packages_manager.validate_package( package_name='otherpackage', transplatform_slug=transplatform) self.assertFalse(package_validated)