def _parse_uploaded_worksheet(self): sheets_rows = dict() for sheet in self.uploaded_workbook.worksheets: sheets_rows[sheet.worksheet.title] = get_unicode_dicts(sheet) self._generate_uploaded_headers() self._set_uploaded_sheet_name_to_module_or_form_mapping( sheets_rows[MODULES_AND_FORMS_SHEET_NAME]) return sheets_rows
def _compare_single_sheet(self): sheet = self.uploaded_workbook.worksheets[0] columns_to_compare = SINGLE_SHEET_STATIC_HEADERS + self.lang_cols_to_compare parsed_expected_rows = self._processed_single_sheet_expected_rows( self.current_rows[sheet.title], columns_to_compare) parsed_uploaded_rows = self._processed_single_sheet_uploaded_rows( get_unicode_dicts(sheet), columns_to_compare) error_msgs = self._generate_diff(parsed_expected_rows, parsed_uploaded_rows) return {sheet.title: error_msgs} if error_msgs else {}
def get_sheet_name_to_unique_id_map(file_or_filename, lang): """ Returns a map of sheet names to unique IDs, so that when modules or forms have been moved we can use their ID and not their (changed) name. This function is called before we process the upload so that we can use the sheet-name-to-unique-ID map to check the sheets before they are processed. `file_or_filename` is a file not a workbook because we read uploaded Excel files using WorkbookJSONReader, and it can only iterate sheet rows once. This function opens its own Reader to parse the first sheet. """ def get_sheet_name(): return MODULES_AND_FORMS_SHEET_NAME if is_multisheet( ) else SINGLE_SHEET_NAME def is_multisheet(): return not lang def is_modules_and_forms_row(row): """ Returns the rows about modules and forms in single-sheet uploads. They are the rows that include the unique IDs. """ return not row['case_property'] and not row[ 'list_or_detail'] and not row['label'] sheet_name_to_unique_id = {} try: worksheet = get_single_worksheet(file_or_filename, title=get_sheet_name()) except WorkbookJSONError: # There is something wrong with the file. The problem will happen # again when we try to process the upload. To preserve current # behaviour, just return silently. return sheet_name_to_unique_id if is_multisheet(): rows = worksheet else: rows = (row for row in worksheet if is_modules_and_forms_row(row)) for row in get_unicode_dicts(rows): sheet_name = row.get('menu_or_form', '') unique_id = row.get('unique_id') if unique_id and sheet_name not in sheet_name_to_unique_id: sheet_name_to_unique_id[sheet_name] = unique_id return sheet_name_to_unique_id
def _update_report_module_rows(self, rows): new_headers = [None for i in self.module.report_configs] new_descriptions = [None for i in self.module.report_configs] allow_update = True for row in get_unicode_dicts(rows): match = re.search(r'^Report (\d+) (Display Text|Description)$', row['case_property']) if not match: message = _( "Found unexpected row \"{0}\" for menu {1}. No changes were made for menu " "{1}.").format(row['case_property'], self.module.id + 1) self.msgs.append((messages.error, message)) allow_update = False continue index = int(match.group(1)) try: config = self.module.report_configs[index] except IndexError: message = _( "Expected {0} reports for menu {1} but found row for Report {2}. No changes were made " "for menu {1}.").format(len(self.module.report_configs), self.module.id + 1, index) self.msgs.append((messages.error, message)) allow_update = False continue if match.group(2) == "Display Text": new_headers[index] = row else: if config.use_xpath_description: message = _( "Found row for {0}, but this report uses an xpath description, which is not " "localizable. Description not updated.").format( row['case_property']) self.msgs.append((messages.error, message)) continue new_descriptions[index] = row if not allow_update: return for index, config in enumerate(self.module.report_configs): if new_headers[index]: self._update_translation(new_headers[index], config.header) if new_descriptions[index]: self._update_translation(new_descriptions[index], config.localized_description)
def update(self, rows): """ This handles updating module/form names and menu media (the contents of the "Menus and forms" sheet in the multi-tab upload). """ self.msgs = [] for row in get_unicode_dicts(rows): sheet_name = row.get('menu_or_form', row.get('sheet_name', '')) # The unique_id column is populated on the "Menus_and_forms" sheet in multi-sheet translation files, # and in the "name / menu media" row in single-sheet translation files. unique_id = row.get('unique_id') if unique_id and sheet_name not in self.sheet_name_to_unique_id: # If we have a value for unique_id, save it in self.sheet_name_to_unique_id so we can look it up # for rows where the unique_id column is not populated. self.sheet_name_to_unique_id[sheet_name] = unique_id elif not unique_id and sheet_name in self.sheet_name_to_unique_id: # If we don't have a value for unique_id, try to fetch it from self.sheet_name_to_unique_id unique_id = self.sheet_name_to_unique_id[sheet_name] try: if unique_id: document = get_menu_or_form_by_unique_id(self.app, unique_id, sheet_name) else: document = get_menu_or_form_by_sheet_name(self.app, sheet_name) except (ModuleNotFoundException, FormNotFoundException, ValueError) as err: self.msgs.append((messages.error, six.text_type(err))) continue self.update_translation_dict('default_', document.name, row) # Update menu media # For backwards compatibility with previous code, accept old "filepath" header names for lang in self.langs: image_header = 'image_%s' % lang if image_header not in row: image_header = 'icon_filepath_%s' % lang if image_header in row: document.set_icon(lang, row[image_header]) audio_header = 'audio_%s' % lang if audio_header not in row: audio_header = 'audio_filepath_%s' % lang if audio_header in row: document.set_audio(lang, row[audio_header]) return self.msgs
def update(self, rows): """ This handles updating module/form names and menu media (the contents of the "Menus and forms" sheet in the multi-tab upload). """ self.msgs = [] for row in get_unicode_dicts(rows): sheet_name = row.get('menu_or_form', '') # The unique_id column is populated on the "Menus_and_forms" sheet in multi-sheet translation files, # and in the "name / menu media" row in single-sheet translation files. unique_id = row.get('unique_id') if not unique_id and sheet_name in self.sheet_name_to_unique_id: # If we don't have a value for unique_id, try to fetch it from self.sheet_name_to_unique_id unique_id = self.sheet_name_to_unique_id[sheet_name] try: if unique_id: document = get_menu_or_form_by_unique_id( self.app, unique_id, sheet_name) else: document = get_menu_or_form_by_sheet_name( self.app, sheet_name) except (ModuleNotFoundException, FormNotFoundException, ValueError) as err: self.msgs.append((messages.error, six.text_type(err))) continue self.update_translation_dict('default_', document.name, row) # Update menu media for lang in self.langs: image_header = 'image_%s' % lang if image_header in row: document.set_icon(lang, row[image_header]) audio_header = 'audio_%s' % lang if audio_header in row: document.set_audio(lang, row[audio_header]) return self.msgs
def update(self, rows): try: self._check_for_shadow_form_error() except BulkAppTranslationsException as e: return [(messages.error, str(e))] if not self.itext: # This form is empty or malformed. Ignore it. return [] # Setup rows = get_unicode_dicts(rows) template_translation_el = self._get_template_translation_el() self._add_missing_translation_elements_to_itext( template_translation_el) self._populate_markdown_stats(rows) self.msgs = [] # Skip labels that have no translation provided label_ids_to_skip = self._get_label_ids_to_skip(rows) # Update the translations for lang in self.langs: translation_node = self.itext.find("./{f}translation[@lang='%s']" % lang) assert (translation_node.exists()) for row in rows: if row['label'] in label_ids_to_skip: continue try: self._add_or_remove_translations(lang, row) except BulkAppTranslationsException as e: self.msgs.append((messages.warning, str(e))) save_xform(self.app, self.form, etree.tostring(self.xform.xml, encoding='utf-8')) return [(t, _('Error in {sheet}: {msg}').format(sheet=self.sheet_name, msg=m)) for (t, m) in self.msgs]
def compare(self): msgs = {} self._generate_expected_headers_and_rows() for sheet in self.uploaded_workbook.worksheets: sheet_name = sheet.worksheet.title # if sheet is not in the expected rows, ignore it. This can happen if the module/form sheet is excluded # from transifex integration if sheet_name not in self.expected_rows: continue rows = get_unicode_dicts(sheet) if is_modules_and_forms_sheet(sheet.worksheet.title): error_msgs = self._compare_sheet(sheet_name, rows, 'module_and_form') elif is_module_sheet(sheet.worksheet.title): error_msgs = self._compare_sheet(sheet_name, rows, 'module') elif is_form_sheet(sheet.worksheet.title): error_msgs = self._compare_sheet(sheet_name, rows, 'form') else: raise Exception("Got unexpected sheet name %s" % sheet_name) if error_msgs: msgs[sheet_name] = error_msgs return msgs
def update(self, rows): try: self._check_for_shadow_form_error() except BulkAppTranslationsException as e: return [(messages.error, six.text_type(e))] if not self.itext: # This form is empty or malformed. Ignore it. return [] # Setup rows = get_unicode_dicts(rows) template_translation_el = self._get_template_translation_el() self._add_missing_translation_elements_to_itext(template_translation_el) self._populate_markdown_stats(rows) self.msgs = [] # Skip labels that have no translation provided label_ids_to_skip = self._get_label_ids_to_skip(rows) # Update the translations for lang in self.langs: translation_node = self.itext.find("./{f}translation[@lang='%s']" % lang) assert(translation_node.exists()) for row in rows: if row['label'] in label_ids_to_skip: continue try: self._add_or_remove_translations(lang, row) except BulkAppTranslationsException as e: self.msgs.append((messages.warning, six.text_type(e))) save_xform(self.app, self.form, etree.tostring(self.xform.xml)) return [(t, _('Error in {sheet}: {msg}').format(sheet=self.sheet_name, msg=m)) for (t, m) in self.msgs]
def _get_condensed_rows(self, rows): ''' Reconfigure the given sheet into objects that are easier to process. The major change is to nest mapping and graph config rows under their "parent" rows, so that there's one row per case proeprty. This function also pulls out case detail tab headers and the case list form label, which will be processed separately from the case proeprty rows. Populates class attributes condensed_rows, case_list_form_label, case_list_menu_item_label, and tab_headers. ''' self.condensed_rows = [] self.case_list_form_label = None self.case_list_menu_item_label = None self.tab_headers = [None for i in self.module.case_details.long.tabs] index_of_last_enum_in_condensed = -1 index_of_last_graph_in_condensed = -1 for i, row in enumerate(get_unicode_dicts(rows)): # If it's an enum case property, set index_of_last_enum_in_condensed if row['case_property'].endswith(" (ID Mapping Text)"): row['id'] = self._remove_description_from_case_property(row) self.condensed_rows.append(row) index_of_last_enum_in_condensed = len(self.condensed_rows) - 1 # If it's an enum value, add it to its parent enum property elif row['case_property'].endswith(" (ID Mapping Value)"): row['id'] = self._remove_description_from_case_property(row) parent = self.condensed_rows[index_of_last_enum_in_condensed] parent['mappings'] = parent.get('mappings', []) + [row] # If it's a graph case property, set index_of_last_graph_in_condensed elif row['case_property'].endswith(" (graph)"): row['id'] = self._remove_description_from_case_property(row) self.condensed_rows.append(row) index_of_last_graph_in_condensed = len(self.condensed_rows) - 1 # If it's a graph configuration item, add it to its parent elif row['case_property'].endswith(" (graph config)"): row['id'] = self._remove_description_from_case_property(row) parent = self.condensed_rows[index_of_last_graph_in_condensed] parent['configs'] = parent.get('configs', []) + [row] # If it's a graph series configuration item, add it to its parent elif row['case_property'].endswith(" (graph series config)"): trimmed_property = self._remove_description_from_case_property( row) row['id'] = trimmed_property.split(" ")[0] row['series_index'] = trimmed_property.split(" ")[1] parent = self.condensed_rows[index_of_last_graph_in_condensed] parent['series_configs'] = parent.get('series_configs', []) + [row] # If it's a graph annotation, add it to its parent elif row['case_property'].startswith("graph annotation "): row['id'] = int(row['case_property'].split(" ")[-1]) parent = self.condensed_rows[index_of_last_graph_in_condensed] parent['annotations'] = parent.get('annotations', []) + [row] # It's the case list registration form label. Don't add it to condensed rows elif row['case_property'] == 'case_list_form_label': self.case_list_form_label = row # It's the case list menu item label. Don't add it to condensed rows elif row['case_property'] == 'case_list_menu_item_label': self.case_list_menu_item_label = row # If it's a tab header, don't add it to condensed rows elif re.search(r'^Tab \d+$', row['case_property']): index = int(row['case_property'].split(' ')[-1]) if index < len(self.tab_headers): self.tab_headers[index] = row else: message = _( "Expected {0} case detail tabs for menu {1} but found row for Tab {2}. No changes " "were made for menu {1}.").format( len(self.tab_headers), self.module.id + 1, index) self.msgs.append((messages.error, message)) # It's a normal case property else: row['id'] = row['case_property'] self.condensed_rows.append(row)
def _get_condensed_rows(self, rows): ''' Reconfigure the given sheet into objects that are easier to process. The major change is to nest mapping and graph config rows under their "parent" rows, so that there's one row per case proeprty. This function also pulls out case detail tab headers and the case list form label, which will be processed separately from the case proeprty rows. Populates class attributes condensed_rows, case_list_form_label, and tab_headers. ''' self.condensed_rows = [] self.case_list_form_label = None self.tab_headers = [None for i in self.module.case_details.long.tabs] index_of_last_enum_in_condensed = -1 index_of_last_graph_in_condensed = -1 for i, row in enumerate(get_unicode_dicts(rows)): # If it's an enum case property, set index_of_last_enum_in_condensed if row['case_property'].endswith(" (ID Mapping Text)"): row['id'] = self._remove_description_from_case_property(row) self.condensed_rows.append(row) index_of_last_enum_in_condensed = len(self.condensed_rows) - 1 # If it's an enum value, add it to its parent enum property elif row['case_property'].endswith(" (ID Mapping Value)"): row['id'] = self._remove_description_from_case_property(row) parent = self.condensed_rows[index_of_last_enum_in_condensed] parent['mappings'] = parent.get('mappings', []) + [row] # If it's a graph case property, set index_of_last_graph_in_condensed elif row['case_property'].endswith(" (graph)"): row['id'] = self._remove_description_from_case_property(row) self.condensed_rows.append(row) index_of_last_graph_in_condensed = len(self.condensed_rows) - 1 # If it's a graph configuration item, add it to its parent elif row['case_property'].endswith(" (graph config)"): row['id'] = self._remove_description_from_case_property(row) parent = self.condensed_rows[index_of_last_graph_in_condensed] parent['configs'] = parent.get('configs', []) + [row] # If it's a graph series configuration item, add it to its parent elif row['case_property'].endswith(" (graph series config)"): trimmed_property = self._remove_description_from_case_property(row) row['id'] = trimmed_property.split(" ")[0] row['series_index'] = trimmed_property.split(" ")[1] parent = self.condensed_rows[index_of_last_graph_in_condensed] parent['series_configs'] = parent.get('series_configs', []) + [row] # If it's a graph annotation, add it to its parent elif row['case_property'].startswith("graph annotation "): row['id'] = int(row['case_property'].split(" ")[-1]) parent = self.condensed_rows[index_of_last_graph_in_condensed] parent['annotations'] = parent.get('annotations', []) + [row] # It's a case list registration form label. Don't add it to condensed rows elif row['case_property'] == 'case_list_form_label': self.case_list_form_label = row # If it's a tab header, don't add it to condensed rows elif re.search(r'^Tab \d+$', row['case_property']): index = int(row['case_property'].split(' ')[-1]) if index < len(self.tab_headers): self.tab_headers[index] = row else: message = _("Expected {0} case detail tabs for menu {1} but found row for Tab {2}. No changes " "were made for menu {1}.").format(len(self.tab_headers), self.module.id + 1, index) self.msgs.append((messages.error, message)) # It's a normal case property else: row['id'] = row['case_property'] self.condensed_rows.append(row)