def tag_sync_from_coverage(stats, package, release, tag): return_value = OrderedDict() if not isinstance(stats, str): return return_value if isinstance(stats, str) and not stats.startswith('Not Synced with'): return return_value package_manager = PackagesManager() release_manager = ReleaseBranchManager() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: branch_mapping = {} if package_details.release_branch_mapping_json: branch_mapping = package_details.release_branch_mapping_json.copy() if release in branch_mapping: branch_mapping = branch_mapping.get(release) branch_mapping['product'] = \ release_manager.get_product_by_release(release).product_slug return_value.update( dict( mapping=branch_mapping, package=package, tag=tag, )) return return_value
def tag_branch_mapping(package): package_manager = PackagesManager() release_manager = ReleaseBranchManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: branch_mapping = {} if package_details.release_branch_mapping_json: branch_mapping = package_details.release_branch_mapping_json.copy() for k, v in package_details.release_branch_mapping_json.items(): branch_mapping[k]['product'] = \ release_manager.get_product_by_release(k).product_slug return_value.update({ 'package_name': package_details.package_name, 'branch_mapping': branch_mapping if branch_mapping else package_details.release_branch_mapping_json, 'mapping_lastupdated': package_details.release_branch_map_last_updated, 'mapping_keys': BRANCH_MAPPING_KEYS }) return return_value
def task_sync_packages_with_platform(): """ sync all packages with translation platform """ package_manager = PackagesManager() reports_manager = ReportsManager() def _sync_package(pkg): package_manager.sync_update_package_stats(pkg) all_packages = package_manager.get_packages().filter( platform_last_updated__lte=timezone.now() - timedelta(hours=6)).order_by('platform_url') for package in all_packages: th = threading.Thread(target=_sync_package, args=(package.package_name, )) th.start() th.join() time.sleep(4) logger.info("%s Packages sync'd with Translation Platform" % len(all_packages)) if reports_manager.analyse_releases_status(): logger.info("Releases Summary Updated") if reports_manager.analyse_packages_status(): logger.info("Packages Summary Updated")
def task_sync_packages_with_platform(): """ sync all packages with translation platform """ package_manager = PackagesManager() reports_manager = ReportsManager() def _sync_package(pkg): package_manager.sync_update_package_stats(pkg) package_manager.fetch_latest_builds(pkg) all_packages = [] packages_except_weblate_fedora = package_manager.get_packages().filter( platform_last_updated__lte=timezone.now() - timedelta(hours=12)).order_by('platform_url').exclude( platform_slug_id=WEBLATE_SLUGS[1]) weblate_fedora_packages = package_manager.get_packages().filter( platform_last_updated__lte=timezone.now() - timedelta(hours=60), platform_slug_id=WEBLATE_SLUGS[1]).order_by('platform_url') if packages_except_weblate_fedora: all_packages.extend(packages_except_weblate_fedora) if weblate_fedora_packages: all_packages.extend(weblate_fedora_packages) for package in all_packages: th = threading.Thread(target=_sync_package, args=(package.package_name, )) th.start() th.join() time.sleep(randrange(5, 10)) logger.info("%s Packages sync'd with Translation Platform" % len(all_packages)) if reports_manager.analyse_releases_status(): logger.info("Releases Summary Updated") if reports_manager.analyse_packages_status(): logger.info("Packages Summary Updated")
def tag_outofsync_packages(): return_value = OrderedDict() package_manager = PackagesManager() all_packages = package_manager.get_packages() outofsync_packages = \ [i.package_name for i in all_packages if not i.stats_diff_health] if all_packages and outofsync_packages: return_value["insync_packages"] = \ (all_packages.count() - len(outofsync_packages)) or 0 return_value["outofsync_packages"] = len(outofsync_packages) or 0 return_value["total_packages"] = all_packages.count() or 0 return return_value
def tag_package_details(package_name, user): package_manager = PackagesManager() return_value = OrderedDict() try: package = package_manager.get_packages([package_name]).get() pkg_details = package.package_details_json if pkg_details and pkg_details.get('description'): return_value.update({'package_desc': pkg_details['description']}) except: # log event, passing for now pass else: return_value.update({'package': package, 'user': user}) return return_value
def tag_branch_mapping(package): package_manager = PackagesManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: return_value.update( {'package_name': package_details.package_name, 'branch_mapping': package_details.release_branch_mapping_json, 'mapping_lastupdated': package_details.release_branch_map_last_updated, 'mapping_keys': BRANCH_MAPPING_KEYS} ) return return_value
def tag_latest_builds(package): package_manager = PackagesManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: return_value.update({ 'package_name': package, 'latest_builds': package_details.package_latest_builds_json.copy(), 'builds_lastupdated': package_details.package_latest_builds_last_updated }) return return_value
def tag_package_details(package_name, user): package_manager = PackagesManager() return_value = OrderedDict() try: package = package_manager.get_packages([package_name]).get() pkg_details = package.package_details_json if pkg_details and pkg_details.get('description'): return_value.update( {'package_desc': pkg_details['description']} ) except: # log event, passing for now pass else: return_value.update( {'package': package, 'user': user} ) return return_value
def tag_stats_diff(package): package_manager = PackagesManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: stats_diff = package_details.stats_diff_json or {} langs_out_of_sync = {} for branch, diff in stats_diff.items(): langs_out_of_sync[branch] = {} for lang, diff_percent in diff.items(): langs_out_of_sync[branch][lang] = diff_percent return_value.update( {'package_name': package_details.package_name, 'stats_diff': langs_out_of_sync} ) return return_value
def tag_stats_diff(package): package_manager = PackagesManager() return_value = OrderedDict() try: package_details = package_manager.get_packages([package]).get() except: # log event, passing for now pass else: stats_diff = package_details.stats_diff_json or {} langs_out_of_sync = {} for branch, diff in stats_diff.items(): langs_out_of_sync[branch] = {} for lang, diff_percent in diff.items(): langs_out_of_sync[branch][lang] = diff_percent return_value.update({ 'package_name': package_details.package_name, 'stats_diff': langs_out_of_sync }) return return_value
def task_sync_packages_with_build_system(): """ sync all packages with build system """ package_manager = PackagesManager() graph_manager = GraphManager() reports_manager = ReportsManager() job_template_manager = JobTemplateManager() location_manager = GeoLocationManager() def _update_diff(package): try: package_stats = graph_manager.get_trans_stats_by_package( package.package_name) graph_manager.package_manager.calculate_stats_diff( package.package_name, package_stats, package.release_branch_mapping_json) except Exception: # pass for now pass def _sync_build_system(template, params): if package_manager.is_package_build_latest(params): return t_params = template.job_template_params if len(t_params) == len(params): job_data = { field.upper(): param for field, param in zip(t_params, params) } job_data.update({ 'YML_FILE': yaml.dump(template.job_template_json, default_flow_style=False).replace("\'", "") }) job_data.update({'SCRATCH': True}) temp_path = 'false/{0}/'.format('-'.join(params)) job_manager = YMLBasedJobManager( **job_data, **{ 'params': [p.upper() for p in t_params], 'type': TS_JOB_TYPES[3] }, **{'active_user_email': '*****@*****.**'}, **{'sandbox_path': temp_path}, **{'job_log_file': temp_path + '.log'}) try: if os.path.isdir(temp_path): shutil.rmtree(temp_path) os.mkdir(temp_path) job_manager.execute_job() except Exception as e: # pass for now pass finally: shutil.rmtree(temp_path) job_template = None all_packages = package_manager.get_packages().filter( release_branch_mapping__isnull=False) job_templates = job_template_manager.get_job_templates( job_template_type=TS_JOB_TYPES[3]) if job_templates: job_template = job_templates.first() if all_packages and job_template: for package in all_packages: candidates = [] mapping = package.release_branch_mapping_json or {} for release, map_dict in mapping.items(): candidates.append((package.package_name, map_dict.get(BRANCH_MAPPING_KEYS[1]), map_dict.get(BRANCH_MAPPING_KEYS[2]))) for candidate in candidates: th = threading.Thread(target=_sync_build_system, args=( job_template, candidate, )) th.start() th.join() time.sleep(6) _update_diff(package) logger.info("%s Packages sync'd with Build System" % len(all_packages)) if reports_manager.analyse_packages_status(): logger.info("Packages Summary Updated") time.sleep(2) if reports_manager.refresh_stats_required_by_territory(): logger.info("Location Summary Updated") time.sleep(2) if location_manager.save_territory_build_system_stats(): logger.info("Territory Summary Updated")