def _parse_uploaded_worksheet(self):
sheets_rows = dict()
for sheet in self.uploaded_workbook.worksheets:
sheets_rows[sheet.worksheet.title] = get_unicode_dicts(sheet)
self._generate_uploaded_headers()
self._set_uploaded_sheet_name_to_module_or_form_mapping(
sheets_rows[MODULES_AND_FORMS_SHEET_NAME])
return sheets_rows
def _compare_single_sheet(self):
sheet = self.uploaded_workbook.worksheets[0]
columns_to_compare = SINGLE_SHEET_STATIC_HEADERS + self.lang_cols_to_compare
parsed_expected_rows = self._processed_single_sheet_expected_rows(
self.current_rows[sheet.title], columns_to_compare)
parsed_uploaded_rows = self._processed_single_sheet_uploaded_rows(
get_unicode_dicts(sheet), columns_to_compare)
error_msgs = self._generate_diff(parsed_expected_rows,
parsed_uploaded_rows)
return {sheet.title: error_msgs} if error_msgs else {}
def get_sheet_name_to_unique_id_map(file_or_filename, lang):
"""
Returns a map of sheet names to unique IDs, so that when modules or
forms have been moved we can use their ID and not their (changed) name.
This function is called before we process the upload so that we can use
the sheet-name-to-unique-ID map to check the sheets before they are
processed.
`file_or_filename` is a file not a workbook because we read uploaded
Excel files using WorkbookJSONReader, and it can only iterate sheet
rows once. This function opens its own Reader to parse the first sheet.
"""
def get_sheet_name():
return MODULES_AND_FORMS_SHEET_NAME if is_multisheet(
) else SINGLE_SHEET_NAME
def is_multisheet():
return not lang
def is_modules_and_forms_row(row):
"""
Returns the rows about modules and forms in single-sheet uploads.
They are the rows that include the unique IDs.
"""
return not row['case_property'] and not row[
'list_or_detail'] and not row['label']
sheet_name_to_unique_id = {}
try:
worksheet = get_single_worksheet(file_or_filename,
title=get_sheet_name())
except WorkbookJSONError:
# There is something wrong with the file. The problem will happen
# again when we try to process the upload. To preserve current
# behaviour, just return silently.
return sheet_name_to_unique_id
if is_multisheet():
rows = worksheet
else:
rows = (row for row in worksheet if is_modules_and_forms_row(row))
for row in get_unicode_dicts(rows):
sheet_name = row.get('menu_or_form', '')
unique_id = row.get('unique_id')
if unique_id and sheet_name not in sheet_name_to_unique_id:
sheet_name_to_unique_id[sheet_name] = unique_id
return sheet_name_to_unique_id
def _update_report_module_rows(self, rows):
new_headers = [None for i in self.module.report_configs]
new_descriptions = [None for i in self.module.report_configs]
allow_update = True
for row in get_unicode_dicts(rows):
match = re.search(r'^Report (\d+) (Display Text|Description)$',
row['case_property'])
if not match:
message = _(
"Found unexpected row \"{0}\" for menu {1}. No changes were made for menu "
"{1}.").format(row['case_property'], self.module.id + 1)
self.msgs.append((messages.error, message))
allow_update = False
continue
index = int(match.group(1))
try:
config = self.module.report_configs[index]
except IndexError:
message = _(
"Expected {0} reports for menu {1} but found row for Report {2}. No changes were made "
"for menu {1}.").format(len(self.module.report_configs),
self.module.id + 1, index)
self.msgs.append((messages.error, message))
allow_update = False
continue
if match.group(2) == "Display Text":
new_headers[index] = row
else:
if config.use_xpath_description:
message = _(
"Found row for {0}, but this report uses an xpath description, which is not "
"localizable. Description not updated.").format(
row['case_property'])
self.msgs.append((messages.error, message))
continue
new_descriptions[index] = row
if not allow_update:
return
for index, config in enumerate(self.module.report_configs):
if new_headers[index]:
self._update_translation(new_headers[index], config.header)
if new_descriptions[index]:
self._update_translation(new_descriptions[index],
config.localized_description)
def update(self, rows):
"""
This handles updating module/form names and menu media
(the contents of the "Menus and forms" sheet in the multi-tab upload).
"""
self.msgs = []
for row in get_unicode_dicts(rows):
sheet_name = row.get('menu_or_form', row.get('sheet_name', ''))
# The unique_id column is populated on the "Menus_and_forms" sheet in multi-sheet translation files,
# and in the "name / menu media" row in single-sheet translation files.
unique_id = row.get('unique_id')
if unique_id and sheet_name not in self.sheet_name_to_unique_id:
# If we have a value for unique_id, save it in self.sheet_name_to_unique_id so we can look it up
# for rows where the unique_id column is not populated.
self.sheet_name_to_unique_id[sheet_name] = unique_id
elif not unique_id and sheet_name in self.sheet_name_to_unique_id:
# If we don't have a value for unique_id, try to fetch it from self.sheet_name_to_unique_id
unique_id = self.sheet_name_to_unique_id[sheet_name]
try:
if unique_id:
document = get_menu_or_form_by_unique_id(self.app, unique_id, sheet_name)
else:
document = get_menu_or_form_by_sheet_name(self.app, sheet_name)
except (ModuleNotFoundException, FormNotFoundException, ValueError) as err:
self.msgs.append((messages.error, six.text_type(err)))
continue
self.update_translation_dict('default_', document.name, row)
# Update menu media
# For backwards compatibility with previous code, accept old "filepath" header names
for lang in self.langs:
image_header = 'image_%s' % lang
if image_header not in row:
image_header = 'icon_filepath_%s' % lang
if image_header in row:
document.set_icon(lang, row[image_header])
audio_header = 'audio_%s' % lang
if audio_header not in row:
audio_header = 'audio_filepath_%s' % lang
if audio_header in row:
document.set_audio(lang, row[audio_header])
return self.msgs
def update(self, rows):
"""
This handles updating module/form names and menu media
(the contents of the "Menus and forms" sheet in the multi-tab upload).
"""
self.msgs = []
for row in get_unicode_dicts(rows):
sheet_name = row.get('menu_or_form', '')
# The unique_id column is populated on the "Menus_and_forms" sheet in multi-sheet translation files,
# and in the "name / menu media" row in single-sheet translation files.
unique_id = row.get('unique_id')
if not unique_id and sheet_name in self.sheet_name_to_unique_id:
# If we don't have a value for unique_id, try to fetch it from self.sheet_name_to_unique_id
unique_id = self.sheet_name_to_unique_id[sheet_name]
try:
if unique_id:
document = get_menu_or_form_by_unique_id(
self.app, unique_id, sheet_name)
else:
document = get_menu_or_form_by_sheet_name(
self.app, sheet_name)
except (ModuleNotFoundException, FormNotFoundException,
ValueError) as err:
self.msgs.append((messages.error, six.text_type(err)))
continue
self.update_translation_dict('default_', document.name, row)
# Update menu media
for lang in self.langs:
image_header = 'image_%s' % lang
if image_header in row:
document.set_icon(lang, row[image_header])
audio_header = 'audio_%s' % lang
if audio_header in row:
document.set_audio(lang, row[audio_header])
return self.msgs
def update(self, rows):
try:
self._check_for_shadow_form_error()
except BulkAppTranslationsException as e:
return [(messages.error, str(e))]
if not self.itext:
# This form is empty or malformed. Ignore it.
return []
# Setup
rows = get_unicode_dicts(rows)
template_translation_el = self._get_template_translation_el()
self._add_missing_translation_elements_to_itext(
template_translation_el)
self._populate_markdown_stats(rows)
self.msgs = []
# Skip labels that have no translation provided
label_ids_to_skip = self._get_label_ids_to_skip(rows)
# Update the translations
for lang in self.langs:
translation_node = self.itext.find("./{f}translation[@lang='%s']" %
lang)
assert (translation_node.exists())
for row in rows:
if row['label'] in label_ids_to_skip:
continue
try:
self._add_or_remove_translations(lang, row)
except BulkAppTranslationsException as e:
self.msgs.append((messages.warning, str(e)))
save_xform(self.app, self.form,
etree.tostring(self.xform.xml, encoding='utf-8'))
return [(t, _('Error in {sheet}: {msg}').format(sheet=self.sheet_name,
msg=m))
for (t, m) in self.msgs]
def compare(self):
msgs = {}
self._generate_expected_headers_and_rows()
for sheet in self.uploaded_workbook.worksheets:
sheet_name = sheet.worksheet.title
# if sheet is not in the expected rows, ignore it. This can happen if the module/form sheet is excluded
# from transifex integration
if sheet_name not in self.expected_rows:
continue
rows = get_unicode_dicts(sheet)
if is_modules_and_forms_sheet(sheet.worksheet.title):
error_msgs = self._compare_sheet(sheet_name, rows, 'module_and_form')
elif is_module_sheet(sheet.worksheet.title):
error_msgs = self._compare_sheet(sheet_name, rows, 'module')
elif is_form_sheet(sheet.worksheet.title):
error_msgs = self._compare_sheet(sheet_name, rows, 'form')
else:
raise Exception("Got unexpected sheet name %s" % sheet_name)
if error_msgs:
msgs[sheet_name] = error_msgs
return msgs
def update(self, rows):
try:
self._check_for_shadow_form_error()
except BulkAppTranslationsException as e:
return [(messages.error, six.text_type(e))]
if not self.itext:
# This form is empty or malformed. Ignore it.
return []
# Setup
rows = get_unicode_dicts(rows)
template_translation_el = self._get_template_translation_el()
self._add_missing_translation_elements_to_itext(template_translation_el)
self._populate_markdown_stats(rows)
self.msgs = []
# Skip labels that have no translation provided
label_ids_to_skip = self._get_label_ids_to_skip(rows)
# Update the translations
for lang in self.langs:
translation_node = self.itext.find("./{f}translation[@lang='%s']" % lang)
assert(translation_node.exists())
for row in rows:
if row['label'] in label_ids_to_skip:
continue
try:
self._add_or_remove_translations(lang, row)
except BulkAppTranslationsException as e:
self.msgs.append((messages.warning, six.text_type(e)))
save_xform(self.app, self.form, etree.tostring(self.xform.xml))
return [(t, _('Error in {sheet}: {msg}').format(sheet=self.sheet_name, msg=m)) for (t, m) in self.msgs]
def _get_condensed_rows(self, rows):
'''
Reconfigure the given sheet into objects that are easier to process.
The major change is to nest mapping and graph config rows under their
"parent" rows, so that there's one row per case proeprty.
This function also pulls out case detail tab headers and the case list form label,
which will be processed separately from the case proeprty rows.
Populates class attributes condensed_rows, case_list_form_label, case_list_menu_item_label,
and tab_headers.
'''
self.condensed_rows = []
self.case_list_form_label = None
self.case_list_menu_item_label = None
self.tab_headers = [None for i in self.module.case_details.long.tabs]
index_of_last_enum_in_condensed = -1
index_of_last_graph_in_condensed = -1
for i, row in enumerate(get_unicode_dicts(rows)):
# If it's an enum case property, set index_of_last_enum_in_condensed
if row['case_property'].endswith(" (ID Mapping Text)"):
row['id'] = self._remove_description_from_case_property(row)
self.condensed_rows.append(row)
index_of_last_enum_in_condensed = len(self.condensed_rows) - 1
# If it's an enum value, add it to its parent enum property
elif row['case_property'].endswith(" (ID Mapping Value)"):
row['id'] = self._remove_description_from_case_property(row)
parent = self.condensed_rows[index_of_last_enum_in_condensed]
parent['mappings'] = parent.get('mappings', []) + [row]
# If it's a graph case property, set index_of_last_graph_in_condensed
elif row['case_property'].endswith(" (graph)"):
row['id'] = self._remove_description_from_case_property(row)
self.condensed_rows.append(row)
index_of_last_graph_in_condensed = len(self.condensed_rows) - 1
# If it's a graph configuration item, add it to its parent
elif row['case_property'].endswith(" (graph config)"):
row['id'] = self._remove_description_from_case_property(row)
parent = self.condensed_rows[index_of_last_graph_in_condensed]
parent['configs'] = parent.get('configs', []) + [row]
# If it's a graph series configuration item, add it to its parent
elif row['case_property'].endswith(" (graph series config)"):
trimmed_property = self._remove_description_from_case_property(
row)
row['id'] = trimmed_property.split(" ")[0]
row['series_index'] = trimmed_property.split(" ")[1]
parent = self.condensed_rows[index_of_last_graph_in_condensed]
parent['series_configs'] = parent.get('series_configs',
[]) + [row]
# If it's a graph annotation, add it to its parent
elif row['case_property'].startswith("graph annotation "):
row['id'] = int(row['case_property'].split(" ")[-1])
parent = self.condensed_rows[index_of_last_graph_in_condensed]
parent['annotations'] = parent.get('annotations', []) + [row]
# It's the case list registration form label. Don't add it to condensed rows
elif row['case_property'] == 'case_list_form_label':
self.case_list_form_label = row
# It's the case list menu item label. Don't add it to condensed rows
elif row['case_property'] == 'case_list_menu_item_label':
self.case_list_menu_item_label = row
# If it's a tab header, don't add it to condensed rows
elif re.search(r'^Tab \d+$', row['case_property']):
index = int(row['case_property'].split(' ')[-1])
if index < len(self.tab_headers):
self.tab_headers[index] = row
else:
message = _(
"Expected {0} case detail tabs for menu {1} but found row for Tab {2}. No changes "
"were made for menu {1}.").format(
len(self.tab_headers), self.module.id + 1, index)
self.msgs.append((messages.error, message))
# It's a normal case property
else:
row['id'] = row['case_property']
self.condensed_rows.append(row)
def _get_condensed_rows(self, rows):
'''
Reconfigure the given sheet into objects that are easier to process.
The major change is to nest mapping and graph config rows under their
"parent" rows, so that there's one row per case proeprty.
This function also pulls out case detail tab headers and the case list form label,
which will be processed separately from the case proeprty rows.
Populates class attributes condensed_rows, case_list_form_label, and tab_headers.
'''
self.condensed_rows = []
self.case_list_form_label = None
self.tab_headers = [None for i in self.module.case_details.long.tabs]
index_of_last_enum_in_condensed = -1
index_of_last_graph_in_condensed = -1
for i, row in enumerate(get_unicode_dicts(rows)):
# If it's an enum case property, set index_of_last_enum_in_condensed
if row['case_property'].endswith(" (ID Mapping Text)"):
row['id'] = self._remove_description_from_case_property(row)
self.condensed_rows.append(row)
index_of_last_enum_in_condensed = len(self.condensed_rows) - 1
# If it's an enum value, add it to its parent enum property
elif row['case_property'].endswith(" (ID Mapping Value)"):
row['id'] = self._remove_description_from_case_property(row)
parent = self.condensed_rows[index_of_last_enum_in_condensed]
parent['mappings'] = parent.get('mappings', []) + [row]
# If it's a graph case property, set index_of_last_graph_in_condensed
elif row['case_property'].endswith(" (graph)"):
row['id'] = self._remove_description_from_case_property(row)
self.condensed_rows.append(row)
index_of_last_graph_in_condensed = len(self.condensed_rows) - 1
# If it's a graph configuration item, add it to its parent
elif row['case_property'].endswith(" (graph config)"):
row['id'] = self._remove_description_from_case_property(row)
parent = self.condensed_rows[index_of_last_graph_in_condensed]
parent['configs'] = parent.get('configs', []) + [row]
# If it's a graph series configuration item, add it to its parent
elif row['case_property'].endswith(" (graph series config)"):
trimmed_property = self._remove_description_from_case_property(row)
row['id'] = trimmed_property.split(" ")[0]
row['series_index'] = trimmed_property.split(" ")[1]
parent = self.condensed_rows[index_of_last_graph_in_condensed]
parent['series_configs'] = parent.get('series_configs', []) + [row]
# If it's a graph annotation, add it to its parent
elif row['case_property'].startswith("graph annotation "):
row['id'] = int(row['case_property'].split(" ")[-1])
parent = self.condensed_rows[index_of_last_graph_in_condensed]
parent['annotations'] = parent.get('annotations', []) + [row]
# It's a case list registration form label. Don't add it to condensed rows
elif row['case_property'] == 'case_list_form_label':
self.case_list_form_label = row
# If it's a tab header, don't add it to condensed rows
elif re.search(r'^Tab \d+$', row['case_property']):
index = int(row['case_property'].split(' ')[-1])
if index < len(self.tab_headers):
self.tab_headers[index] = row
else:
message = _("Expected {0} case detail tabs for menu {1} but found row for Tab {2}. No changes "
"were made for menu {1}.").format(len(self.tab_headers), self.module.id + 1, index)
self.msgs.append((messages.error, message))
# It's a normal case property
else:
row['id'] = row['case_property']
self.condensed_rows.append(row)
