def tag_workload_detailed(relbranch):
return_value = OrderedDict()
graph_manager = GraphManager()
headers, workload = graph_manager.get_workload_detailed(relbranch)
return_value.update(dict(headers=headers))
return_value.update(dict(packages=workload.items()))
return return_value
class ManagersMixin(object):
"""
Managers Mixin
"""
inventory_manager = InventoryManager()
packages_manager = PackagesManager()
jobs_log_manager = JobsLogManager()
release_branch_manager = ReleaseBranchManager()
graph_manager = GraphManager()
def get_summary(self):
"""
Application Inventory Stats
"""
locales_set = self.inventory_manager.get_locales_set()
summary = {}
summary['locales_len'] = len(locales_set[0]) \
if isinstance(locales_set, tuple) else 0
platforms = self.inventory_manager.get_transplatform_slug_url()
summary['platforms_len'] = len(platforms) if platforms else 0
relstreams = self.inventory_manager.get_relstream_slug_name()
summary['products_len'] = len(relstreams) if relstreams else 0
relbranches = self.release_branch_manager.get_release_branches()
summary['releases_len'] = relbranches.count() if relbranches else 0
summary['packages_len'] = self.packages_manager.count_packages()
jobs_count, last_ran_on, last_ran_type = \
self.jobs_log_manager.get_joblog_stats()
summary['jobs_len'] = jobs_count
graph_rules = self.graph_manager.get_graph_rules(only_active=True)
summary['graph_rules_len'] = graph_rules.count() if graph_rules else 0
return summary
def tag_workload_per_lang(relbranch, lang_id):
return_value = OrderedDict()
graph_manager = GraphManager()
headers, workload = \
graph_manager.get_workload_estimate(relbranch,
locale=lang_id)
return_value.update(dict(headers=headers))
return_value.update(dict(packages=workload.items()))
return return_value
class ManagersMixin(object):
"""
Managers Mixin
"""
inventory_manager = InventoryManager()
packages_manager = PackagesManager()
jobs_log_manager = JobsLogManager()
release_branch_manager = ReleaseBranchManager()
graph_manager = GraphManager()
def tag_tabular_form(package):
return_value = OrderedDict()
graph_manager = GraphManager()
stats_dict = graph_manager.get_trans_stats_by_package(package)
headers = stats_dict['ticks']
stats_data = stats_dict['graph_data']
return_value.update(
{'headers': [lang for index, lang in headers],
'stats_data': stats_data,
'pkg_desc': stats_dict['pkg_desc']}
)
return return_value
def tag_coverage_view(coverage_rule):
return_value = OrderedDict()
graph_manager = GraphManager()
rule_data, pkg_len, locale_len, tag_len, release_name = \
graph_manager.get_trans_stats_by_rule(coverage_rule)
return_value.update(
dict(coverage_rule=coverage_rule,
rule_data=rule_data,
package_len=pkg_len,
locale_len=locale_len,
build_tag_len=tag_len,
release=release_name))
return return_value
def tag_threshold_based(relbranch, threshold):
return_value = OrderedDict()
graph_manager = GraphManager()
if threshold and not isinstance(threshold, int):
threshold = int(threshold)
headers, workload, locales = graph_manager.get_threshold_based(
relbranch, threshold)
return_value.update(dict(headers=headers))
return_value.update(dict(languages=workload))
return_value.update(dict(locales=locales))
return_value.update(dict(threshold=threshold))
return_value.update(dict(release=relbranch))
return return_value
def graph_data(request):
"""
Prepares and dispatch graph data
"""
graph_dataset = {}
if request.is_ajax():
graph_manager = GraphManager()
if 'package' in request.POST.dict() and 'locale' in request.POST.dict():
package = request.POST.dict().get('package')
locale = request.POST.dict().get('locale')
graph_dataset = graph_manager.get_stats_by_pkg_per_lang(package, locale)
elif 'package' in request.POST.dict():
package = request.POST.dict().get('package')
graph_dataset = graph_manager.get_trans_stats_by_package(package)
elif 'graph_rule' in request.POST.dict():
graph_rule = request.POST.dict().get('graph_rule')
graph_dataset = graph_manager.get_trans_stats_by_rule(graph_rule)
return JsonResponse(graph_dataset)
def workload_graph(request):
"""
Generates workload graph
"""
graph_dataset = {}
if request.is_ajax():
post_params = request.POST.dict()
if post_params.get('relbranch') and post_params.get('lang'):
context = Context({
'META': request.META,
'relbranch': post_params['relbranch'],
'locale': post_params['lang']
})
template_string = """
{% load tag_workload_per_lang from custom_tags %}
{% tag_workload_per_lang relbranch locale %}
"""
return HttpResponse(Template(template_string).render(context))
elif post_params.get('relbranch') and post_params.get('combine'):
context = Context({
'META': request.META,
'relbranch': post_params['relbranch']
})
template_string = """
{% load tag_workload_combined from custom_tags %}
{% tag_workload_combined relbranch %}
"""
return HttpResponse(Template(template_string).render(context))
elif post_params.get('relbranch') and post_params.get('detail'):
context = Context({
'META': request.META,
'relbranch': post_params['relbranch']
})
template_string = """
{% load tag_workload_detailed from custom_tags %}
{% tag_workload_detailed relbranch %}
"""
return HttpResponse(Template(template_string).render(context))
elif post_params.get('relbranch'):
graph_manager = GraphManager()
graph_dataset = graph_manager.get_workload_graph_data(
post_params['relbranch'])
return JsonResponse(graph_dataset)
class APIMixin(object):
"""
Required Managers
"""
graph_manager = GraphManager()
class GraphManagerMixin(object):
"""
Required Manager
"""
graph_manager = GraphManager()
class Command(BaseCommand):
help = 'Sync packages with their respective translation platform.'
graph_manager = GraphManager()
package_manager = PackagesManager()
reports_manager = ReportsManager()
job_template_manager = JobTemplateManager()
location_manager = GeoLocationManager()
def _sync_package(self, pkg):
self.package_manager.sync_update_package_stats(pkg)
def sync_with_platform(self):
all_packages = self.package_manager.get_packages().filter(
platform_last_updated__lte=timezone.now() -
timedelta(hours=6)).order_by('platform_url')
for package in all_packages:
th = threading.Thread(target=self._sync_package,
args=(package.package_name, ))
th.start()
th.join()
time.sleep(2)
self.reports_manager.analyse_releases_status()
self.reports_manager.analyse_packages_status()
def _update_diff(self, package):
try:
package_stats = self.graph_manager.get_trans_stats_by_package(
package.package_name)
self.graph_manager.package_manager.calculate_stats_diff(
package.package_name, package_stats,
package.release_branch_mapping_json)
except Exception:
# pass for now
pass
def _sync_build_system(self, template, params):
if self.package_manager.is_package_build_latest(params):
return
t_params = template.job_template_params
if len(t_params) == len(params):
job_data = {
field.upper(): param
for field, param in zip(t_params, params)
}
job_data.update({
'YML_FILE':
yaml.dump(template.job_template_json,
default_flow_style=False).replace("\'", "")
})
job_data.update({'SCRATCH': True})
temp_path = 'false/{0}/'.format('-'.join(params))
job_manager = YMLBasedJobManager(
**job_data, **{
'params': [p.upper() for p in t_params],
'type': TS_JOB_TYPES[3]
}, **{'active_user_email':
'*****@*****.**'}, **{'sandbox_path': temp_path},
**{'job_log_file': temp_path + '.log'})
try:
if os.path.isdir(temp_path):
shutil.rmtree(temp_path)
os.mkdir(temp_path)
job_manager.execute_job()
except Exception as e:
# pass for now
pass
finally:
os.rmdir(temp_path)
def sync_with_build_system(self):
job_template = None
all_packages = self.package_manager.get_packages().filter(
release_branch_mapping__isnull=False)
job_templates = self.job_template_manager.get_job_templates(
job_template_type=TS_JOB_TYPES[3])
if job_templates:
job_template = job_templates.first()
if all_packages and job_template:
for package in all_packages:
candidates = []
mapping = package.release_branch_mapping_json or {}
for release, map_dict in mapping.items():
candidates.append((package.package_name,
map_dict.get(BRANCH_MAPPING_KEYS[1]),
map_dict.get(BRANCH_MAPPING_KEYS[2])))
for candidate in candidates:
th = threading.Thread(target=self._sync_build_system,
args=(
job_template,
candidate,
))
th.start()
th.join()
time.sleep(5)
self._update_diff(package)
self.reports_manager.analyse_packages_status()
self.reports_manager.refresh_stats_required_by_territory()
self.location_manager.save_territory_build_system_stats()
def add_arguments(self, parser):
# Named (optional) arguments
parser.add_argument(
'--platform',
action='store_true',
help='Sync packages with translation platform only.',
)
parser.add_argument(
'--build-system',
action='store_true',
help='Sync packages with build system only.',
)
def handle(self, *args, **options):
cmd_combinations_options = ['platform', 'build_system', 'default_both']
cmd_combinations = {
cmd_combinations_options[0]:
self.sync_with_platform,
cmd_combinations_options[1]:
self.sync_with_build_system,
cmd_combinations_options[2]:
[self.sync_with_platform, self.sync_with_build_system]
}
if options.get(cmd_combinations_options[0]):
cmd_combinations.get(cmd_combinations_options[0])()
elif options.get(cmd_combinations_options[1]):
cmd_combinations.get(cmd_combinations_options[1])()
else:
[m() for m in cmd_combinations.get(cmd_combinations_options[2])]
def task_sync_packages_with_build_system():
"""
sync all packages with build system
"""
package_manager = PackagesManager()
graph_manager = GraphManager()
reports_manager = ReportsManager()
job_template_manager = JobTemplateManager()
location_manager = GeoLocationManager()
def _update_diff(package):
try:
package_stats = graph_manager.get_trans_stats_by_package(
package.package_name)
graph_manager.package_manager.calculate_stats_diff(
package.package_name, package_stats,
package.release_branch_mapping_json)
except Exception:
# pass for now
pass
def _sync_build_system(template, params):
if package_manager.is_package_build_latest(params):
return
t_params = template.job_template_params
if len(t_params) == len(params):
job_data = {
field.upper(): param
for field, param in zip(t_params, params)
}
job_data.update({
'YML_FILE':
yaml.dump(template.job_template_json,
default_flow_style=False).replace("\'", "")
})
job_data.update({'SCRATCH': True})
temp_path = 'false/{0}/'.format('-'.join(params))
job_manager = YMLBasedJobManager(
**job_data, **{
'params': [p.upper() for p in t_params],
'type': TS_JOB_TYPES[3]
}, **{'active_user_email':
'*****@*****.**'}, **{'sandbox_path': temp_path},
**{'job_log_file': temp_path + '.log'})
try:
if os.path.isdir(temp_path):
shutil.rmtree(temp_path)
os.mkdir(temp_path)
job_manager.execute_job()
except Exception as e:
# pass for now
pass
finally:
shutil.rmtree(temp_path)
job_template = None
all_packages = package_manager.get_packages().filter(
release_branch_mapping__isnull=False)
job_templates = job_template_manager.get_job_templates(
job_template_type=TS_JOB_TYPES[3])
if job_templates:
job_template = job_templates.first()
if all_packages and job_template:
for package in all_packages:
candidates = []
mapping = package.release_branch_mapping_json or {}
for release, map_dict in mapping.items():
candidates.append((package.package_name,
map_dict.get(BRANCH_MAPPING_KEYS[1]),
map_dict.get(BRANCH_MAPPING_KEYS[2])))
for candidate in candidates:
th = threading.Thread(target=_sync_build_system,
args=(
job_template,
candidate,
))
th.start()
th.join()
time.sleep(6)
_update_diff(package)
logger.info("%s Packages sync'd with Build System" % len(all_packages))
if reports_manager.analyse_packages_status():
logger.info("Packages Summary Updated")
time.sleep(2)
if reports_manager.refresh_stats_required_by_territory():
logger.info("Location Summary Updated")
time.sleep(2)
if location_manager.save_territory_build_system_stats():
logger.info("Territory Summary Updated")