def generate_reports(request):
"""
Generates Reports
"""
if request.is_ajax():
post_params = request.POST.dict()
report_subject = post_params.get('subject', '')
reports_manager = ReportsManager()
if report_subject == 'releases':
releases_summary = reports_manager.analyse_releases_status()
if releases_summary:
context = Context(
{'META': request.META,
'relsummary': releases_summary,
'last_updated': datetime.now()}
)
template_string = """
{% load tag_releases_summary from custom_tags %}
{% tag_releases_summary %}
"""
return HttpResponse(Template(template_string).render(context))
if report_subject == 'packages':
packages_summary = reports_manager.analyse_packages_status()
if packages_summary:
context = Context(
{'META': request.META,
'pkgsummary': packages_summary,
'last_updated': datetime.now()}
)
template_string = """
{% load tag_packages_summary from custom_tags %}
{% tag_packages_summary %}
"""
return HttpResponse(Template(template_string).render(context))
return HttpResponse(status=500)
def task_sync_packages_with_platform():
"""
sync all packages with translation platform
"""
package_manager = PackagesManager()
reports_manager = ReportsManager()
def _sync_package(pkg):
package_manager.sync_update_package_stats(pkg)
all_packages = package_manager.get_packages().filter(
platform_last_updated__lte=timezone.now() -
timedelta(hours=6)).order_by('platform_url')
for package in all_packages:
th = threading.Thread(target=_sync_package,
args=(package.package_name, ))
th.start()
th.join()
time.sleep(4)
logger.info("%s Packages sync'd with Translation Platform" %
len(all_packages))
if reports_manager.analyse_releases_status():
logger.info("Releases Summary Updated")
if reports_manager.analyse_packages_status():
logger.info("Packages Summary Updated")
def tag_packages_summary():
return_value = OrderedDict()
reports_manager = ReportsManager()
packages_summary = reports_manager.get_reports('packages')
if packages_summary:
return_value.update(dict(
pkgsummary=json.loads(packages_summary.get().report_json_str),
last_updated=packages_summary.get().report_updated
))
return return_value
def tag_packages_summary():
return_value = OrderedDict()
reports_manager = ReportsManager()
packages_summary = reports_manager.get_reports('packages')
if packages_summary:
return_value.update(dict(
pkgsummary=packages_summary.get().report_json,
last_updated=packages_summary.get().report_updated
))
return return_value
def tag_releases_summary():
return_value = OrderedDict()
reports_manager = ReportsManager()
releases_summary = reports_manager.get_reports('releases')
if releases_summary:
return_value.update(dict(
relsummary=releases_summary.get().report_json,
last_updated=releases_summary.get().report_updated
))
return return_value
def tag_packages_summary():
return_value = OrderedDict()
package_manager = PackagesManager()
reports_manager = ReportsManager()
packages_summary = reports_manager.get_reports('packages')
if packages_summary:
return_value.update(
dict(
pkgsummary=json.loads(packages_summary.get().report_json_str),
last_updated=packages_summary.get().report_updated,
package_count=package_manager.count_packages(),
))
return return_value
def tag_releases_summary():
return_value = OrderedDict()
reports_manager = ReportsManager()
releases_summary = reports_manager.get_reports('releases')
if releases_summary:
report = releases_summary.get().report_json_str
release_report_json = json.loads(report) if isinstance(report, str) else {}
pkg_manager = PackagesManager()
lang_locale_dict = {lang: locale for locale, lang in pkg_manager.get_locale_lang_tuple()}
for release, summary in release_report_json.items():
if summary.get('languages'):
release_report_json[release]['languages'] = \
OrderedDict(sorted(summary['languages'].items()))
return_value.update(dict(
relsummary=release_report_json,
last_updated=releases_summary.get().report_updated,
lang_locale=lang_locale_dict
))
return return_value
def task_sync_packages_with_platform():
"""
sync all packages with translation platform
"""
package_manager = PackagesManager()
reports_manager = ReportsManager()
def _sync_package(pkg):
package_manager.sync_update_package_stats(pkg)
package_manager.fetch_latest_builds(pkg)
all_packages = []
packages_except_weblate_fedora = package_manager.get_packages().filter(
platform_last_updated__lte=timezone.now() -
timedelta(hours=12)).order_by('platform_url').exclude(
platform_slug_id=WEBLATE_SLUGS[1])
weblate_fedora_packages = package_manager.get_packages().filter(
platform_last_updated__lte=timezone.now() - timedelta(hours=60),
platform_slug_id=WEBLATE_SLUGS[1]).order_by('platform_url')
if packages_except_weblate_fedora:
all_packages.extend(packages_except_weblate_fedora)
if weblate_fedora_packages:
all_packages.extend(weblate_fedora_packages)
for package in all_packages:
th = threading.Thread(target=_sync_package,
args=(package.package_name, ))
th.start()
th.join()
time.sleep(randrange(5, 10))
logger.info("%s Packages sync'd with Translation Platform" %
len(all_packages))
if reports_manager.analyse_releases_status():
logger.info("Releases Summary Updated")
if reports_manager.analyse_packages_status():
logger.info("Packages Summary Updated")
def tag_trending_languages():
return_value = OrderedDict()
reports_manager = ReportsManager()
releases_summary = reports_manager.get_reports('releases')
release_manager = ReleaseBranchManager()
latest_release = release_manager.get_latest_release()
pkg_manager = PackagesManager()
lang_locale_dict = {
lang: locale
for locale, lang in pkg_manager.get_locale_lang_tuple()
}
if releases_summary and latest_release:
releases_summary = releases_summary.get()
trending_languages = reports_manager.get_trending_languages(
releases_summary.report_json, *latest_release)
if trending_languages and isinstance(trending_languages,
(list, tuple)):
return_value["trending_languages"] = trending_languages[:9]
return_value["lang_locale_dict"] = lang_locale_dict
return_value["latest_release"] = latest_release
return return_value
def tag_releases_summary():
return_value = OrderedDict()
reports_manager = ReportsManager()
releases_summary = reports_manager.get_reports('releases')
if releases_summary:
report = releases_summary.get().report_json_str
release_report_json = json.loads(report) if isinstance(report,
str) else {}
pkg_manager = PackagesManager()
lang_locale_dict = {
lang: locale
for locale, lang in pkg_manager.get_locale_lang_tuple()
}
for release, summary in release_report_json.items():
if summary.get('languages'):
release_report_json[release]['languages'] = \
OrderedDict(sorted(summary['languages'].items()))
return_value.update(
dict(relsummary=release_report_json,
last_updated=releases_summary.get().report_updated,
lang_locale=lang_locale_dict))
return return_value
class Command(BaseCommand):
help = 'Sync packages with their respective translation platform.'
graph_manager = GraphManager()
package_manager = PackagesManager()
reports_manager = ReportsManager()
job_template_manager = JobTemplateManager()
location_manager = GeoLocationManager()
def _sync_package(self, pkg):
self.package_manager.sync_update_package_stats(pkg)
def sync_with_platform(self):
all_packages = self.package_manager.get_packages().filter(
platform_last_updated__lte=timezone.now() -
timedelta(hours=6)).order_by('platform_url')
for package in all_packages:
th = threading.Thread(target=self._sync_package,
args=(package.package_name, ))
th.start()
th.join()
time.sleep(2)
self.reports_manager.analyse_releases_status()
self.reports_manager.analyse_packages_status()
def _update_diff(self, package):
try:
package_stats = self.graph_manager.get_trans_stats_by_package(
package.package_name)
self.graph_manager.package_manager.calculate_stats_diff(
package.package_name, package_stats,
package.release_branch_mapping_json)
except Exception:
# pass for now
pass
def _sync_build_system(self, template, params):
if self.package_manager.is_package_build_latest(params):
return
t_params = template.job_template_params
if len(t_params) == len(params):
job_data = {
field.upper(): param
for field, param in zip(t_params, params)
}
job_data.update({
'YML_FILE':
yaml.dump(template.job_template_json,
default_flow_style=False).replace("\'", "")
})
job_data.update({'SCRATCH': True})
temp_path = 'false/{0}/'.format('-'.join(params))
job_manager = YMLBasedJobManager(
**job_data, **{
'params': [p.upper() for p in t_params],
'type': TS_JOB_TYPES[3]
}, **{'active_user_email':
'*****@*****.**'}, **{'sandbox_path': temp_path},
**{'job_log_file': temp_path + '.log'})
try:
if os.path.isdir(temp_path):
shutil.rmtree(temp_path)
os.mkdir(temp_path)
job_manager.execute_job()
except Exception as e:
# pass for now
pass
finally:
os.rmdir(temp_path)
def sync_with_build_system(self):
job_template = None
all_packages = self.package_manager.get_packages().filter(
release_branch_mapping__isnull=False)
job_templates = self.job_template_manager.get_job_templates(
job_template_type=TS_JOB_TYPES[3])
if job_templates:
job_template = job_templates.first()
if all_packages and job_template:
for package in all_packages:
candidates = []
mapping = package.release_branch_mapping_json or {}
for release, map_dict in mapping.items():
candidates.append((package.package_name,
map_dict.get(BRANCH_MAPPING_KEYS[1]),
map_dict.get(BRANCH_MAPPING_KEYS[2])))
for candidate in candidates:
th = threading.Thread(target=self._sync_build_system,
args=(
job_template,
candidate,
))
th.start()
th.join()
time.sleep(5)
self._update_diff(package)
self.reports_manager.analyse_packages_status()
self.reports_manager.refresh_stats_required_by_territory()
self.location_manager.save_territory_build_system_stats()
def add_arguments(self, parser):
# Named (optional) arguments
parser.add_argument(
'--platform',
action='store_true',
help='Sync packages with translation platform only.',
)
parser.add_argument(
'--build-system',
action='store_true',
help='Sync packages with build system only.',
)
def handle(self, *args, **options):
cmd_combinations_options = ['platform', 'build_system', 'default_both']
cmd_combinations = {
cmd_combinations_options[0]:
self.sync_with_platform,
cmd_combinations_options[1]:
self.sync_with_build_system,
cmd_combinations_options[2]:
[self.sync_with_platform, self.sync_with_build_system]
}
if options.get(cmd_combinations_options[0]):
cmd_combinations.get(cmd_combinations_options[0])()
elif options.get(cmd_combinations_options[1]):
cmd_combinations.get(cmd_combinations_options[1])()
else:
[m() for m in cmd_combinations.get(cmd_combinations_options[2])]
def task_sync_packages_with_build_system():
"""
sync all packages with build system
"""
package_manager = PackagesManager()
graph_manager = GraphManager()
reports_manager = ReportsManager()
job_template_manager = JobTemplateManager()
location_manager = GeoLocationManager()
def _update_diff(package):
try:
package_stats = graph_manager.get_trans_stats_by_package(
package.package_name)
graph_manager.package_manager.calculate_stats_diff(
package.package_name, package_stats,
package.release_branch_mapping_json)
except Exception:
# pass for now
pass
def _sync_build_system(template, params):
if package_manager.is_package_build_latest(params):
return
t_params = template.job_template_params
if len(t_params) == len(params):
job_data = {
field.upper(): param
for field, param in zip(t_params, params)
}
job_data.update({
'YML_FILE':
yaml.dump(template.job_template_json,
default_flow_style=False).replace("\'", "")
})
job_data.update({'SCRATCH': True})
temp_path = 'false/{0}/'.format('-'.join(params))
job_manager = YMLBasedJobManager(
**job_data, **{
'params': [p.upper() for p in t_params],
'type': TS_JOB_TYPES[3]
}, **{'active_user_email':
'*****@*****.**'}, **{'sandbox_path': temp_path},
**{'job_log_file': temp_path + '.log'})
try:
if os.path.isdir(temp_path):
shutil.rmtree(temp_path)
os.mkdir(temp_path)
job_manager.execute_job()
except Exception as e:
# pass for now
pass
finally:
shutil.rmtree(temp_path)
job_template = None
all_packages = package_manager.get_packages().filter(
release_branch_mapping__isnull=False)
job_templates = job_template_manager.get_job_templates(
job_template_type=TS_JOB_TYPES[3])
if job_templates:
job_template = job_templates.first()
if all_packages and job_template:
for package in all_packages:
candidates = []
mapping = package.release_branch_mapping_json or {}
for release, map_dict in mapping.items():
candidates.append((package.package_name,
map_dict.get(BRANCH_MAPPING_KEYS[1]),
map_dict.get(BRANCH_MAPPING_KEYS[2])))
for candidate in candidates:
th = threading.Thread(target=_sync_build_system,
args=(
job_template,
candidate,
))
th.start()
th.join()
time.sleep(6)
_update_diff(package)
logger.info("%s Packages sync'd with Build System" % len(all_packages))
if reports_manager.analyse_packages_status():
logger.info("Packages Summary Updated")
time.sleep(2)
if reports_manager.refresh_stats_required_by_territory():
logger.info("Location Summary Updated")
time.sleep(2)
if location_manager.save_territory_build_system_stats():
logger.info("Territory Summary Updated")