def tag_sync_from_coverage(stats, package, release, tag):
return_value = OrderedDict()
if not isinstance(stats, str):
return return_value
if isinstance(stats, str) and not stats.startswith('Not Synced with'):
return return_value
package_manager = PackagesManager()
release_manager = ReleaseBranchManager()
try:
package_details = package_manager.get_packages([package]).get()
except:
# log event, passing for now
pass
else:
branch_mapping = {}
if package_details.release_branch_mapping_json:
branch_mapping = package_details.release_branch_mapping_json.copy()
if release in branch_mapping:
branch_mapping = branch_mapping.get(release)
branch_mapping['product'] = \
release_manager.get_product_by_release(release).product_slug
return_value.update(
dict(
mapping=branch_mapping,
package=package,
tag=tag,
))
return return_value
def tag_branch_mapping(package):
package_manager = PackagesManager()
release_manager = ReleaseBranchManager()
return_value = OrderedDict()
try:
package_details = package_manager.get_packages([package]).get()
except:
# log event, passing for now
pass
else:
branch_mapping = {}
if package_details.release_branch_mapping_json:
branch_mapping = package_details.release_branch_mapping_json.copy()
for k, v in package_details.release_branch_mapping_json.items():
branch_mapping[k]['product'] = \
release_manager.get_product_by_release(k).product_slug
return_value.update({
'package_name':
package_details.package_name,
'branch_mapping':
branch_mapping
if branch_mapping else package_details.release_branch_mapping_json,
'mapping_lastupdated':
package_details.release_branch_map_last_updated,
'mapping_keys':
BRANCH_MAPPING_KEYS
})
return return_value
def task_sync_packages_with_platform():
"""
sync all packages with translation platform
"""
package_manager = PackagesManager()
reports_manager = ReportsManager()
def _sync_package(pkg):
package_manager.sync_update_package_stats(pkg)
all_packages = package_manager.get_packages().filter(
platform_last_updated__lte=timezone.now() -
timedelta(hours=6)).order_by('platform_url')
for package in all_packages:
th = threading.Thread(target=_sync_package,
args=(package.package_name, ))
th.start()
th.join()
time.sleep(4)
logger.info("%s Packages sync'd with Translation Platform" %
len(all_packages))
if reports_manager.analyse_releases_status():
logger.info("Releases Summary Updated")
if reports_manager.analyse_packages_status():
logger.info("Packages Summary Updated")
def task_sync_packages_with_platform():
"""
sync all packages with translation platform
"""
package_manager = PackagesManager()
reports_manager = ReportsManager()
def _sync_package(pkg):
package_manager.sync_update_package_stats(pkg)
package_manager.fetch_latest_builds(pkg)
all_packages = []
packages_except_weblate_fedora = package_manager.get_packages().filter(
platform_last_updated__lte=timezone.now() -
timedelta(hours=12)).order_by('platform_url').exclude(
platform_slug_id=WEBLATE_SLUGS[1])
weblate_fedora_packages = package_manager.get_packages().filter(
platform_last_updated__lte=timezone.now() - timedelta(hours=60),
platform_slug_id=WEBLATE_SLUGS[1]).order_by('platform_url')
if packages_except_weblate_fedora:
all_packages.extend(packages_except_weblate_fedora)
if weblate_fedora_packages:
all_packages.extend(weblate_fedora_packages)
for package in all_packages:
th = threading.Thread(target=_sync_package,
args=(package.package_name, ))
th.start()
th.join()
time.sleep(randrange(5, 10))
logger.info("%s Packages sync'd with Translation Platform" %
len(all_packages))
if reports_manager.analyse_releases_status():
logger.info("Releases Summary Updated")
if reports_manager.analyse_packages_status():
logger.info("Packages Summary Updated")
def tag_outofsync_packages():
return_value = OrderedDict()
package_manager = PackagesManager()
all_packages = package_manager.get_packages()
outofsync_packages = \
[i.package_name for i in all_packages if not i.stats_diff_health]
if all_packages and outofsync_packages:
return_value["insync_packages"] = \
(all_packages.count() - len(outofsync_packages)) or 0
return_value["outofsync_packages"] = len(outofsync_packages) or 0
return_value["total_packages"] = all_packages.count() or 0
return return_value
def tag_package_details(package_name, user):
package_manager = PackagesManager()
return_value = OrderedDict()
try:
package = package_manager.get_packages([package_name]).get()
pkg_details = package.package_details_json
if pkg_details and pkg_details.get('description'):
return_value.update({'package_desc': pkg_details['description']})
except:
# log event, passing for now
pass
else:
return_value.update({'package': package, 'user': user})
return return_value
def tag_branch_mapping(package):
package_manager = PackagesManager()
return_value = OrderedDict()
try:
package_details = package_manager.get_packages([package]).get()
except:
# log event, passing for now
pass
else:
return_value.update(
{'package_name': package_details.package_name,
'branch_mapping': package_details.release_branch_mapping_json,
'mapping_lastupdated': package_details.release_branch_map_last_updated,
'mapping_keys': BRANCH_MAPPING_KEYS}
)
return return_value
def tag_latest_builds(package):
package_manager = PackagesManager()
return_value = OrderedDict()
try:
package_details = package_manager.get_packages([package]).get()
except:
# log event, passing for now
pass
else:
return_value.update({
'package_name':
package,
'latest_builds':
package_details.package_latest_builds_json.copy(),
'builds_lastupdated':
package_details.package_latest_builds_last_updated
})
return return_value
def tag_package_details(package_name, user):
package_manager = PackagesManager()
return_value = OrderedDict()
try:
package = package_manager.get_packages([package_name]).get()
pkg_details = package.package_details_json
if pkg_details and pkg_details.get('description'):
return_value.update(
{'package_desc': pkg_details['description']}
)
except:
# log event, passing for now
pass
else:
return_value.update(
{'package': package,
'user': user}
)
return return_value
def tag_stats_diff(package):
package_manager = PackagesManager()
return_value = OrderedDict()
try:
package_details = package_manager.get_packages([package]).get()
except:
# log event, passing for now
pass
else:
stats_diff = package_details.stats_diff_json or {}
langs_out_of_sync = {}
for branch, diff in stats_diff.items():
langs_out_of_sync[branch] = {}
for lang, diff_percent in diff.items():
langs_out_of_sync[branch][lang] = diff_percent
return_value.update(
{'package_name': package_details.package_name,
'stats_diff': langs_out_of_sync}
)
return return_value
def tag_stats_diff(package):
package_manager = PackagesManager()
return_value = OrderedDict()
try:
package_details = package_manager.get_packages([package]).get()
except:
# log event, passing for now
pass
else:
stats_diff = package_details.stats_diff_json or {}
langs_out_of_sync = {}
for branch, diff in stats_diff.items():
langs_out_of_sync[branch] = {}
for lang, diff_percent in diff.items():
langs_out_of_sync[branch][lang] = diff_percent
return_value.update({
'package_name': package_details.package_name,
'stats_diff': langs_out_of_sync
})
return return_value
def task_sync_packages_with_build_system():
"""
sync all packages with build system
"""
package_manager = PackagesManager()
graph_manager = GraphManager()
reports_manager = ReportsManager()
job_template_manager = JobTemplateManager()
location_manager = GeoLocationManager()
def _update_diff(package):
try:
package_stats = graph_manager.get_trans_stats_by_package(
package.package_name)
graph_manager.package_manager.calculate_stats_diff(
package.package_name, package_stats,
package.release_branch_mapping_json)
except Exception:
# pass for now
pass
def _sync_build_system(template, params):
if package_manager.is_package_build_latest(params):
return
t_params = template.job_template_params
if len(t_params) == len(params):
job_data = {
field.upper(): param
for field, param in zip(t_params, params)
}
job_data.update({
'YML_FILE':
yaml.dump(template.job_template_json,
default_flow_style=False).replace("\'", "")
})
job_data.update({'SCRATCH': True})
temp_path = 'false/{0}/'.format('-'.join(params))
job_manager = YMLBasedJobManager(
**job_data, **{
'params': [p.upper() for p in t_params],
'type': TS_JOB_TYPES[3]
}, **{'active_user_email':
'*****@*****.**'}, **{'sandbox_path': temp_path},
**{'job_log_file': temp_path + '.log'})
try:
if os.path.isdir(temp_path):
shutil.rmtree(temp_path)
os.mkdir(temp_path)
job_manager.execute_job()
except Exception as e:
# pass for now
pass
finally:
shutil.rmtree(temp_path)
job_template = None
all_packages = package_manager.get_packages().filter(
release_branch_mapping__isnull=False)
job_templates = job_template_manager.get_job_templates(
job_template_type=TS_JOB_TYPES[3])
if job_templates:
job_template = job_templates.first()
if all_packages and job_template:
for package in all_packages:
candidates = []
mapping = package.release_branch_mapping_json or {}
for release, map_dict in mapping.items():
candidates.append((package.package_name,
map_dict.get(BRANCH_MAPPING_KEYS[1]),
map_dict.get(BRANCH_MAPPING_KEYS[2])))
for candidate in candidates:
th = threading.Thread(target=_sync_build_system,
args=(
job_template,
candidate,
))
th.start()
th.join()
time.sleep(6)
_update_diff(package)
logger.info("%s Packages sync'd with Build System" % len(all_packages))
if reports_manager.analyse_packages_status():
logger.info("Packages Summary Updated")
time.sleep(2)
if reports_manager.refresh_stats_required_by_territory():
logger.info("Location Summary Updated")
time.sleep(2)
if location_manager.save_territory_build_system_stats():
logger.info("Territory Summary Updated")