def __init__(self, app, uploaded_workbook, lang_prefix='default_'):
self.app = app
self.uploaded_workbook = uploaded_workbook
self.headers = None
self.expected_rows = None
self.lang_prefix = lang_prefix
self.default_language_column = self.lang_prefix + self.app.default_language
self.app_translation_generator = AppTranslationsGenerator(
self.app.domain, self.app.get_id, None, self.app.default_language,
self.app.default_language, self.lang_prefix)
def _setup(self):
self._validate_sheet()
self.single_sheet = is_single_sheet_workbook(self.uploaded_workbook)
self.lang_cols_to_compare = [self.lang_prefix + self.lang_to_compare]
if self.lang_to_compare != self.app.default_language:
default_lang_col = self.lang_prefix + self.app.default_language
if default_lang_col in self.uploaded_workbook.worksheets[
0].fieldnames:
self.lang_cols_to_compare.append(default_lang_col)
self.app_translation_generator = AppTranslationsGenerator(
self.app.domain, self.app.get_id, None, self.app.default_language,
self.lang_to_compare, self.lang_prefix)
self.current_sheet_name_to_module_or_form_type_and_id = dict()
self.uploaded_sheet_name_to_module_or_form_type_and_id = dict()
def __init__(self, app, uploaded_workbook, lang_to_compare, lang_prefix='default_'):
self.app = app
self.uploaded_workbook = uploaded_workbook
self.headers = None
self.expected_rows = None
self.lang_prefix = lang_prefix
self.lang_cols_to_compare = [self.lang_prefix + self.app.default_language]
self.default_language_column = self.lang_prefix + self.app.default_language
if lang_to_compare in app.langs and lang_to_compare != self.app.default_language:
self.lang_cols_to_compare.append(self.lang_prefix + lang_to_compare)
target_lang = lang_to_compare
else:
target_lang = self.app.default_language
self.app_translation_generator = AppTranslationsGenerator(
self.app.domain, self.app.get_id, None, self.app.default_language, target_lang,
self.lang_prefix)
def email_project_from_hq(domain, data, email):
"""Emails the requester with an excel file translations to be sent to Transifex.
Used to verify translations before sending to Transifex
"""
lang = data.get('source_lang')
project_slug = data.get('transifex_project_slug')
quacks_like_a_transifex = AppTranslationsGenerator(domain,
data.get('app_id'),
data.get('version'),
key_lang=lang,
source_lang=lang,
lang_prefix='default_')
parser = TranslationsParser(quacks_like_a_transifex)
try:
translation_file, __ = parser.generate_excel_file()
with open(translation_file.name, 'rb') as file_obj:
email = EmailMessage(
subject='[{}] - HQ translation download'.format(
settings.SERVER_ENVIRONMENT),
body="Translations from HQ",
to=[email],
from_email=settings.DEFAULT_FROM_EMAIL)
filename = "{project}-{lang}-translations.xls".format(
project=project_slug, lang=lang)
email.attach(filename=filename, content=file_obj.read())
email.send()
finally:
try:
os.remove(translation_file.name)
except (NameError, OSError):
pass
def _setup(self):
self.single_sheet = self._is_single_sheet()
if self.single_sheet:
self.lang_cols_to_compare = [
self.lang_prefix + self.lang_to_compare
]
else:
self.lang_cols_to_compare = [
self.lang_prefix + self.app.default_language
]
# compare lang to compare along with default language, if not same
if self.lang_to_compare != self.app.default_language:
self.lang_cols_to_compare.append(self.lang_prefix +
self.lang_to_compare)
self.app_translation_generator = AppTranslationsGenerator(
self.app.domain, self.app.get_id, None, self.app.default_language,
self.lang_to_compare, self.lang_prefix)
self.current_sheet_name_to_module_or_form_type_and_id = dict()
self.uploaded_sheet_name_to_module_or_form_type_and_id = dict()
def __init__(self, app, uploaded_workbook, lang_prefix='default_'):
self.app = app
self.uploaded_workbook = uploaded_workbook
self.headers = None
self.expected_rows = None
self.lang_prefix = lang_prefix
self.default_language_column = self.lang_prefix + self.app.default_language
self.app_translation_generator = AppTranslationsGenerator(
self.app.domain, self.app.get_id, None, self.app.default_language, self.app.default_language,
self.lang_prefix)
def send_translation_files(self):
"""
submit translation files to transifex
"""
app_trans_generator = AppTranslationsGenerator(
self.domain, self.app_id_to_build, self.version,
self.key_lang, self.source_lang, self.lang_prefix,
self.exclude_if_default, self.use_version_postfix)
with PoFileGenerator(app_trans_generator.translations,
app_trans_generator.metadata) as po_file_generator:
generated_files = po_file_generator.generate_translation_files()
return self._send_files_to_transifex(generated_files, app_trans_generator)
def setUpClass(cls):
super(TestTransifexBlacklist, cls).setUpClass()
factory = AppFactory()
app = factory.app
app.langs = ['en', 'hin']
module, form = factory.new_basic_module('register', 'case')
form.source = cls.get_xml('transifex_blacklist').decode('utf-8')
with patch(
'corehq.apps.translations.generators.get_current_app') as mock:
mock.return_value = app
trans_gen = AppTranslationsGenerator('domain', 'app_id', 1, 'en',
'en', 'default_')
translations = trans_gen.translations
first_form_translations = translations['form_register_form_0_v1']
cls.labels_sent_to_transifex = [
trans.msgctxt for trans in first_form_translations
]
class UploadedTranslationsValidator(object):
"""
this compares the excel sheet uploaded with translations with what would be generated
with current app state and flags any discrepancies found between the two
"""
def __init__(self,
app,
uploaded_workbook,
lang_to_compare,
lang_prefix='default_'):
self.app = app
self.uploaded_workbook = uploaded_workbook
self.uploaded_headers = dict() # sheet_name: list of headers
self.current_headers = dict() # module_or_form_id: list of headers
self.current_rows = dict() # module_or_form_id: translations
self.lang_prefix = lang_prefix
self.default_language_column = self.lang_prefix + self.app.default_language
self.lang_to_compare = lang_to_compare or self.app.default_language
self.single_sheet = False
self._setup()
def _setup(self):
self._validate_sheet()
self.single_sheet = is_single_sheet_workbook(self.uploaded_workbook)
self.lang_cols_to_compare = [self.lang_prefix + self.lang_to_compare]
if self.lang_to_compare != self.app.default_language:
default_lang_col = self.lang_prefix + self.app.default_language
if default_lang_col in self.uploaded_workbook.worksheets[
0].fieldnames:
self.lang_cols_to_compare.append(default_lang_col)
self.app_translation_generator = AppTranslationsGenerator(
self.app.domain, self.app.get_id, None, self.app.default_language,
self.lang_to_compare, self.lang_prefix)
self.current_sheet_name_to_module_or_form_type_and_id = dict()
self.uploaded_sheet_name_to_module_or_form_type_and_id = dict()
def _validate_sheet(self):
lang_col = self.lang_prefix + self.lang_to_compare
if lang_col not in self.uploaded_workbook.worksheets[0].fieldnames:
raise BulkAppTranslationsException(
_("Missing translations for {} in uploaded sheet").format(
self.lang_to_compare))
def _generate_current_headers_and_rows(self):
lang = self.lang_to_compare if len(
self.lang_cols_to_compare) == 1 else None
self.current_headers = {
mod_or_form_id: headers
for mod_or_form_id, headers in get_bulk_app_sheet_headers(
self.app,
lang=lang,
eligible_for_transifex_only=True,
single_sheet=self.single_sheet,
)
}
if self.single_sheet:
# single sheet supports only single language
self.current_rows = get_bulk_app_single_sheet_by_name(
self.app,
self.lang_to_compare,
eligible_for_transifex_only=True)
else:
self.current_rows = get_bulk_app_sheets_by_name(
self.app, lang, eligible_for_transifex_only=True)
self._set_current_sheet_name_to_module_or_form_mapping()
self._map_ids_to_headers()
self._map_ids_to_translations()
def _set_current_sheet_name_to_module_or_form_mapping(self):
# iterate the first sheet to get unique ids for forms/modules
all_module_and_form_details = self.current_rows[
MODULES_AND_FORMS_SHEET_NAME]
sheet_name_column_index = self._get_current_header_index(
MODULES_AND_FORMS_SHEET_NAME, 'menu_or_form')
unique_id_column_index = self._get_current_header_index(
MODULES_AND_FORMS_SHEET_NAME, 'unique_id')
type_column_index = self._get_current_header_index(
MODULES_AND_FORMS_SHEET_NAME, 'Type')
for row in all_module_and_form_details:
self.current_sheet_name_to_module_or_form_type_and_id[
row[sheet_name_column_index]] = Unique_ID(
row[type_column_index], row[unique_id_column_index])
def _map_ids_to_headers(self):
sheet_names = list(self.current_headers.keys())
for sheet_name in sheet_names:
if sheet_name != MODULES_AND_FORMS_SHEET_NAME:
mapping = self.current_sheet_name_to_module_or_form_type_and_id.get(
sheet_name)
self.current_headers[mapping.id] = self.current_headers.pop(
sheet_name)
def _map_ids_to_translations(self):
sheet_names = list(self.current_rows.keys())
for sheet_name in sheet_names:
if sheet_name != MODULES_AND_FORMS_SHEET_NAME:
mapping = self.current_sheet_name_to_module_or_form_type_and_id.get(
sheet_name)
self.current_rows[mapping.id] = self.current_rows.pop(
sheet_name)
@memoized
def _get_current_header_index(self, module_or_form_id, header):
if module_or_form_id not in self.current_headers:
raise BulkAppTranslationsException(
_(f"Could not find a module or form with ID '{module_or_form_id}' in app '{self.app.name}'"
))
for index, _column_name in enumerate(
self.current_headers[module_or_form_id]):
if _column_name == header:
return index
def _filter_rows(self, for_type, rows, module_or_form_id):
if for_type == 'form':
return self.app_translation_generator.filter_invalid_rows_for_form(
rows, module_or_form_id,
self._get_current_header_index(module_or_form_id, 'label'))
elif for_type == 'module':
return self.app_translation_generator.filter_invalid_rows_for_module(
rows, module_or_form_id,
self._get_current_header_index(module_or_form_id,
'case_property'),
self._get_current_header_index(module_or_form_id,
'list_or_detail'),
self._get_current_header_index(module_or_form_id,
self.default_language_column))
elif for_type == 'module_and_form':
return rows
assert False, "Unexpected type"
def _compare_sheet(self, module_or_form_id, uploaded_rows, for_type):
"""
:param uploaded_rows: dict
:param for_type: type of sheet, module_and_forms, module, form
:return: diff generated by ghdiff or None
"""
columns_to_compare = COLUMNS_TO_COMPARE[
for_type] + self.lang_cols_to_compare
header_indices = {
column_name:
self._get_current_header_index(module_or_form_id, column_name)
for column_name in columns_to_compare
}
if None in header_indices.values():
raise BulkAppTranslationsException(
_("Could not find column(s) '{}'").format(", ".join(
[str(k) for k, v in header_indices.items() if v is None])))
if module_or_form_id not in self.current_rows:
return self._missing_module_or_form_diff(for_type)
if self.lang_prefix + self.app.default_language in self.lang_cols_to_compare:
current_rows = self._filter_rows(
for_type, self.current_rows[module_or_form_id],
module_or_form_id)
else:
current_rows = self.current_rows[module_or_form_id]
parsed_current_rows = []
parsed_uploaded_rows = []
for current_row in current_rows:
parsed_current_rows.append([
current_row[header_indices[column_name]]
for column_name in columns_to_compare
])
for uploaded_row in uploaded_rows:
parsed_uploaded_rows.append([
uploaded_row.get(column_name)
for column_name in columns_to_compare
])
return self._generate_diff(parsed_current_rows, parsed_uploaded_rows)
@memoized
def _missing_module_or_form_diff(self, for_type):
return ghdiff.diff(_('{} not found').format(for_type), [], css=False)
@staticmethod
def _generate_diff(parsed_current_rows, parsed_uploaded_rows):
current_rows_as_string = '\n'.join(
[', '.join(row) for row in parsed_current_rows])
uploaded_rows_as_string = '\n'.join(
[', '.join(row) for row in parsed_uploaded_rows])
diff = ghdiff.diff(current_rows_as_string,
uploaded_rows_as_string,
css=False)
if diff == NO_GHDIFF_MESSAGE:
return None
return diff
def compare(self):
self._generate_current_headers_and_rows()
if self.single_sheet:
return self._compare_single_sheet()
else:
return self._compare_multiple_sheets()
def _compare_single_sheet(self):
sheet = self.uploaded_workbook.worksheets[0]
columns_to_compare = SINGLE_SHEET_STATIC_HEADERS + self.lang_cols_to_compare
parsed_expected_rows = self._processed_single_sheet_expected_rows(
self.current_rows[sheet.title], columns_to_compare)
parsed_uploaded_rows = self._processed_single_sheet_uploaded_rows(
get_unicode_dicts(sheet), columns_to_compare)
error_msgs = self._generate_diff(parsed_expected_rows,
parsed_uploaded_rows)
return {sheet.title: error_msgs} if error_msgs else {}
def _processed_single_sheet_expected_rows(self, expected_rows,
columns_to_compare):
parsed_expected_rows = []
for expected_row in expected_rows:
_parsed_row = []
for column_name in columns_to_compare:
column_value = expected_row[self._get_current_header_index(
SINGLE_SHEET_NAME, column_name)]
if column_value is None:
column_value = ''
_parsed_row.append(column_value)
parsed_expected_rows.append(_parsed_row)
return parsed_expected_rows
@staticmethod
def _processed_single_sheet_uploaded_rows(uploaded_rows,
columns_to_compare):
return [[
uploaded_row.get(column_name) for column_name in columns_to_compare
] for uploaded_row in uploaded_rows]
def _compare_multiple_sheets(self):
msgs = {}
sheets_rows = self._parse_uploaded_worksheet()
for sheet_name in sheets_rows:
if sheet_name == MODULES_AND_FORMS_SHEET_NAME:
uploaded_module_or_form_id = MODULES_AND_FORMS_SHEET_NAME
else:
uploaded_module_or_form_id = self.uploaded_sheet_name_to_module_or_form_type_and_id[
sheet_name].id
rows = sheets_rows[sheet_name]
if is_modules_and_forms_sheet(sheet_name):
error_msgs = self._compare_sheet(uploaded_module_or_form_id,
rows, 'module_and_form')
elif is_module_sheet(sheet_name):
error_msgs = self._compare_sheet(uploaded_module_or_form_id,
rows, 'module')
elif is_form_sheet(sheet_name):
error_msgs = self._compare_sheet(uploaded_module_or_form_id,
rows, 'form')
else:
raise Exception("Got unexpected sheet name %s" % sheet_name)
if error_msgs:
msgs[sheet_name] = error_msgs
return msgs
def _parse_uploaded_worksheet(self):
sheets_rows = dict()
for sheet in self.uploaded_workbook.worksheets:
sheets_rows[sheet.worksheet.title] = get_unicode_dicts(sheet)
self._generate_uploaded_headers()
self._set_uploaded_sheet_name_to_module_or_form_mapping(
sheets_rows[MODULES_AND_FORMS_SHEET_NAME])
return sheets_rows
def _generate_uploaded_headers(self):
for sheet in self.uploaded_workbook.worksheets:
self.uploaded_headers[sheet.title] = sheet.headers
def _set_uploaded_sheet_name_to_module_or_form_mapping(
self, all_module_and_form_details):
for row in all_module_and_form_details:
self.uploaded_sheet_name_to_module_or_form_type_and_id[
row['menu_or_form']] = Unique_ID(row['Type'], row['unique_id'])
class UploadedTranslationsValidator(object):
"""
this compares the excel sheet uploaded with translations with what would be generated
with current app state and flags any discrepancies found between the two
"""
def __init__(self, app, uploaded_workbook, lang_prefix='default_'):
self.app = app
self.uploaded_workbook = uploaded_workbook
self.headers = None
self.expected_rows = None
self.lang_prefix = lang_prefix
self.default_language_column = self.lang_prefix + self.app.default_language
self.app_translation_generator = AppTranslationsGenerator(
self.app.domain, self.app.get_id, None, self.app.default_language,
self.app.default_language, self.lang_prefix)
def _generate_expected_headers_and_rows(self):
self.headers = {
h[0]: h[1]
for h in get_bulk_app_sheet_headers(self.app)
}
self.expected_rows = get_bulk_app_sheet_rows(
self.app,
exclude_module=lambda module: SKIP_TRANSFEX_STRING in module.
comment,
exclude_form=lambda form: SKIP_TRANSFEX_STRING in form.comment)
@memoized
def _get_header_index(self, sheet_name, header):
for index, _column_name in enumerate(self.headers[sheet_name]):
if _column_name == header:
return index
def _filter_rows(self, for_type, expected_rows, sheet_name):
if for_type == 'form':
return self.app_translation_generator._filter_invalid_rows_for_form(
expected_rows, self.app_translation_generator.
sheet_name_to_module_or_form_type_and_id[sheet_name].id,
self._get_header_index(sheet_name, 'label'))
elif for_type == 'module':
return self.app_translation_generator._filter_invalid_rows_for_module(
expected_rows, self.app_translation_generator.
sheet_name_to_module_or_form_type_and_id[sheet_name].id,
self._get_header_index(sheet_name, 'case_property'),
self._get_header_index(sheet_name, 'list_or_detail'),
self._get_header_index(sheet_name,
self.default_language_column))
elif for_type == 'module_and_form':
return expected_rows
assert False, "Unexpected type"
def _compare_sheet(self, sheet_name, uploaded_rows, for_type):
"""
:param uploaded_rows: dict
:param for_type: type of sheet, module_and_forms, module, form
:return: diff generated by ghdiff or None
"""
columns_to_compare = COLUMNS_TO_COMPARE[for_type] + [
self.default_language_column
]
expected_rows = self._filter_rows(for_type,
self.expected_rows[sheet_name],
sheet_name)
iterate_on = [expected_rows, uploaded_rows]
parsed_expected_rows = []
parsed_uploaded_rows = []
for i, (expected_row, uploaded_row) in enumerate(zip(*iterate_on), 2):
parsed_uploaded_row = [
uploaded_row.get(column_name)
for column_name in columns_to_compare
]
parsed_expected_row = [
expected_row[self._get_header_index(sheet_name, column_name)]
for column_name in columns_to_compare
]
parsed_expected_rows.append(parsed_expected_row)
parsed_uploaded_rows.append(parsed_uploaded_row)
expected_rows_as_string = '\n'.join(
[', '.join(row) for row in parsed_expected_rows])
uploaded_rows_as_string = '\n'.join(
[', '.join(row) for row in parsed_uploaded_rows])
diff = ghdiff.diff(expected_rows_as_string,
uploaded_rows_as_string,
css=False)
if diff == NO_GHDIFF_MESSAGE:
return None
return diff
def compare(self):
msgs = {}
self._generate_expected_headers_and_rows()
for sheet in self.uploaded_workbook.worksheets:
sheet_name = sheet.worksheet.title
# if sheet is not in the expected rows, ignore it. This can happen if the module/form sheet is excluded
# from transifex integration
if sheet_name not in self.expected_rows:
continue
rows = get_unicode_dicts(sheet)
if sheet_name == MODULES_AND_FORMS_SHEET_NAME:
error_msgs = self._compare_sheet(sheet_name, rows,
'module_and_form')
elif 'module' in sheet_name and 'form' not in sheet_name:
error_msgs = self._compare_sheet(sheet_name, rows, 'module')
elif 'module' in sheet_name and 'form' in sheet_name:
error_msgs = self._compare_sheet(sheet_name, rows, 'form')
else:
raise Exception("Got unexpected sheet name %s" % sheet_name)
if error_msgs:
msgs[sheet_name] = error_msgs
return msgs
class UploadedTranslationsValidator(object):
"""
this compares the excel sheet uploaded with translations with what would be generated
with current app state and flags any discrepancies found between the two
"""
def __init__(self, app, uploaded_workbook, lang_prefix='default_'):
self.app = app
self.uploaded_workbook = uploaded_workbook
self.headers = None
self.expected_rows = None
self.lang_prefix = lang_prefix
self.default_language_column = self.lang_prefix + self.app.default_language
self.app_translation_generator = AppTranslationsGenerator(
self.app.domain, self.app.get_id, None, self.app.default_language, self.app.default_language,
self.lang_prefix)
def _generate_expected_headers_and_rows(self):
self.headers = {h[0]: h[1] for h in expected_bulk_app_sheet_headers(self.app)}
self.expected_rows = expected_bulk_app_sheet_rows(
self.app,
exclude_module=lambda module: SKIP_TRANSFEX_STRING in module.comment,
exclude_form=lambda form: SKIP_TRANSFEX_STRING in form.comment
)
@memoized
def _get_header_index(self, sheet_name, header):
for index, _column_name in enumerate(self.headers[sheet_name]):
if _column_name == header:
return index
def _filter_rows(self, for_type, expected_rows, sheet_name):
if for_type == 'form':
return self.app_translation_generator._filter_invalid_rows_for_form(
expected_rows,
self.app_translation_generator.sheet_name_to_module_or_form_type_and_id[sheet_name].id,
self._get_header_index(sheet_name, 'label')
)
elif for_type == 'module':
return self.app_translation_generator._filter_invalid_rows_for_module(
expected_rows,
self.app_translation_generator.sheet_name_to_module_or_form_type_and_id[sheet_name].id,
self._get_header_index(sheet_name, 'case_property'),
self._get_header_index(sheet_name, 'list_or_detail'),
self._get_header_index(sheet_name, self.default_language_column)
)
assert False, "Unexpected type"
def _compare_sheet(self, sheet_name, uploaded_rows, for_type):
"""
:param uploaded_rows: dict
:param for_type: type of sheet, module_and_forms, module, form
:return: diff generated by ghdiff or None
"""
columns_to_compare = COLUMNS_TO_COMPARE[for_type] + [self.default_language_column]
expected_rows = self._filter_rows(for_type, self.expected_rows[sheet_name], sheet_name)
iterate_on = [expected_rows, uploaded_rows]
parsed_expected_rows = []
parsed_uploaded_rows = []
for i, (expected_row, uploaded_row) in enumerate(zip(*iterate_on), 2):
parsed_expected_row = [uploaded_row.get(column_name) for column_name in columns_to_compare]
parsed_uploaded_row = [expected_row[self._get_header_index(sheet_name, column_name)]
for column_name in columns_to_compare]
parsed_expected_rows.append(parsed_expected_row)
parsed_uploaded_rows.append(parsed_uploaded_row)
expected_rows_as_string = '\n'.join([', '.join(row) for row in parsed_expected_rows])
uploaded_rows_as_string = '\n'.join([', '.join(row) for row in parsed_uploaded_rows])
diff = ghdiff.diff(expected_rows_as_string, uploaded_rows_as_string, css=False)
if diff == NO_GHDIFF_MESSAGE:
return None
return diff
def compare(self):
msgs = {}
self._generate_expected_headers_and_rows()
for sheet in self.uploaded_workbook.worksheets:
rows = get_unicode_dicts(sheet)
sheet_name = sheet.worksheet.title
if sheet_name == MODULES_AND_FORMS_SHEET_NAME:
error_msgs = self._compare_sheet(sheet_name, rows, 'module_and_form')
elif 'module' in sheet_name and 'form' not in sheet_name:
error_msgs = self._compare_sheet(sheet_name, rows, 'module')
elif 'module' in sheet_name and 'form' in sheet_name:
error_msgs = self._compare_sheet(sheet_name, rows, 'form')
else:
raise Exception("Got unexpected sheet name %s" % sheet_name)
if error_msgs:
msgs[sheet_name] = error_msgs
return msgs
