def get_resource_files(self, resource): """ Get a dict for all files assigned to a resource. First we calculate the files matching the file expression and then we apply all translation excpetions. The resulting dict will be in this format: { 'en': 'path/foo/en/bar.po', 'de': 'path/foo/de/bar.po', 'es': 'path/exceptions/es.po'} NOTE: All paths are relative to the root of the project """ tr_files = {} if self.config.has_section(resource): try: file_filter = self.config.get(resource, "file_filter") except ConfigParser.NoOptionError: file_filter = "$^" source_lang = self.config.get(resource, "source_lang") source_file = self.get_resource_option(resource, 'source_file') or None if source_file is not None: source_file = native_path(source_file) expr_re = regex_from_filefilter(file_filter, self.root) expr_rec = re.compile(expr_re) for f_path in files_in_project(self.root): match = expr_rec.match(posix_path(f_path)) if match: lang = match.group(1) if lang != source_lang: f_path = os.path.relpath(f_path, self.root) if f_path != source_file: tr_files.update({lang: f_path}) for (name, value) in self.config.items(resource): if name.startswith("trans."): value = native_path(value) lang = name.split('.')[1] # delete language which has same file if value in tr_files.values(): keys = [] for k, v in tr_files.iteritems(): if v == value: keys.append(k) if len(keys) == 1: del tr_files[keys[0]] else: raise Exception("Your configuration seems wrong."\ " You have multiple languages pointing to"\ " the same file.") # Add language with correct file tr_files.update({lang: value}) return tr_files return None
def get_resource_files(self, resource): """ Get a dict for all files assigned to a resource. First we calculate the files matching the file expression and then we apply all translation excpetions. The resulting dict will be in this format: { 'en': 'path/foo/en/bar.po', 'de': 'path/foo/de/bar.po', 'es': 'path/exceptions/es.po'} NOTE: All paths are relative to the root of the project """ tr_files = {} if self.config.has_section(resource): try: file_filter = self.config.get(resource, "file_filter") except ConfigParser.NoOptionError: file_filter = "$^" source_lang = self.config.get(resource, "source_lang") source_file = self.get_resource_option(resource, 'source_file') or None if source_file is not None: source_file = native_path(source_file) expr_re = regex_from_filefilter(file_filter, self.root) expr_rec = re.compile(expr_re) for f_path in files_in_project(self.root): match = expr_rec.match(posix_path(f_path)) if match: lang = match.group(1) if lang != source_lang: f_path = os.path.relpath(f_path, self.root) if f_path != source_file: tr_files.update({lang: f_path}) for (name, value) in self.config.items(resource): if name.startswith("trans."): value = native_path(value) lang = name.split('.')[1] # delete language which has same file if value in tr_files.values(): keys = [] for k, v in tr_files.iteritems(): if v == value: keys.append(k) if len(keys) == 1: del tr_files[keys[0]] else: raise Exception("Your configuration seems wrong."\ " You have multiple languages pointing to"\ " the same file.") # Add language with correct file tr_files.update({lang:value}) return tr_files return None
def get_source_file(self, resource): """Get source file for a resource.""" if self.config.has_section(resource): source_lang = self.config.get(resource, "source_lang") source_file = self.get_resource_option(resource, 'source_file')\ or None if source_file is None: try: file_filter = self.config.get(resource, "file_filter") filename = file_filter.replace('<lang>', source_lang) if os.path.exists(filename): return native_path(filename) except configparser.NoOptionError: pass else: return native_path(source_file)
def regex_from_filefilter(file_filter, root_path=os.path.curdir): """Create proper regex from <lang> expression.""" # Force expr to be a valid regex expr (escaped) but keep <lang> intact expr_re = re.escape( posix_path(os.path.join(root_path, native_path(file_filter)))) expr_re = expr_re.replace("\\<lang\\>", '<lang>').replace( '<lang>', '([^%(sep)s]+)' % {'sep': re.escape(posix_sep)}) return "^%s$" % expr_re
def regex_from_filefilter(file_filter, root_path = os.path.curdir): """Create proper regex from <lang> expression.""" # Force expr to be a valid regex expr (escaped) but keep <lang> intact expr_re = re.escape( posix_path(os.path.join(root_path, native_path(file_filter))) ) expr_re = expr_re.replace("\\<lang\\>", '<lang>').replace( '<lang>', '([^%(sep)s]+)' % { 'sep': re.escape(posix_sep)}) return "^%s$" % expr_re
def pull(self, languages=[], resources=[], overwrite=True, fetchall=False, fetchsource=False, force=False, skip=False, minimum_perc=0, mode=None): """Pull all translations file from transifex server.""" self.minimum_perc = minimum_perc resource_list = self.get_chosen_resources(resources) if mode == 'reviewed': url = 'pull_reviewed_file' elif mode == 'translator': url = 'pull_translator_file' elif mode == 'developer': url = 'pull_developer_file' else: url = 'pull_file' for resource in resource_list: logger.debug("Handling resource %s" % resource) self.resource = resource project_slug, resource_slug = resource.split('.') files = self.get_resource_files(resource) slang = self.get_resource_option(resource, 'source_lang') sfile = self.get_source_file(resource) lang_map = self.get_resource_lang_mapping(resource) host = self.get_resource_host(resource) verify_ssl(host) logger.debug("Language mapping is: %s" % lang_map) if mode is None: mode = self._get_option(resource, 'mode') self.url_info = { 'host': host, 'project': project_slug, 'resource': resource_slug } logger.debug("URL data are: %s" % self.url_info) stats = self._get_stats_for_resource() try: file_filter = self.config.get(resource, 'file_filter') except ConfigParser.NoOptionError: file_filter = None # Pull source file pull_languages = set([]) new_translations = set([]) if fetchall: new_translations = self._new_translations_to_add( files, slang, lang_map, stats, force) if new_translations: msg = "New translations found for the following languages: %s" logger.info(msg % ', '.join(new_translations)) existing, new = self._languages_to_pull(languages, files, lang_map, stats, force) pull_languages |= existing new_translations |= new logger.debug("Adding to new translations: %s" % new) if fetchsource: if sfile and slang not in pull_languages: pull_languages.add(slang) elif slang not in new_translations: new_translations.add(slang) if pull_languages: logger.debug("Pulling languages for: %s" % pull_languages) msg = "Pulling translations for resource %s (source: %s)" logger.info(msg % (resource, sfile)) for lang in pull_languages: local_lang = lang if lang in lang_map.values(): remote_lang = lang_map.flip[lang] else: remote_lang = lang if languages and lang not in pull_languages: logger.debug("Skipping language %s" % lang) continue if lang != slang: local_file = files.get(lang, None) or files[lang_map[lang]] else: local_file = sfile logger.debug("Using file %s" % local_file) kwargs = { 'lang': remote_lang, 'stats': stats, 'local_file': local_file, 'force': force, 'mode': mode, } if not self._should_update_translation(**kwargs): msg = "Skipping '%s' translation (file: %s)." logger.info(msg % (color_text(remote_lang, "RED"), local_file)) continue if not overwrite: local_file = ("%s.new" % local_file) logger.warning(" -> %s: %s" % (color_text(remote_lang, "RED"), local_file)) try: r = self.do_url_request(url, language=remote_lang) except Exception, e: if not skip: raise e else: logger.error(e) continue base_dir = os.path.split(local_file)[0] mkdir_p(base_dir) fd = open(local_file, 'wb') fd.write(r) fd.close() if new_translations: msg = "Pulling new translations for resource %s (source: %s)" logger.info(msg % (resource, sfile)) for lang in new_translations: if lang in lang_map.keys(): local_lang = lang_map[lang] else: local_lang = lang remote_lang = lang if file_filter: local_file = os.path.relpath( os.path.join( self.root, native_path( file_filter.replace('<lang>', local_lang))), os.curdir.decode('utf-8')) else: trans_dir = os.path.join(self.root, ".tx", resource) if not os.path.exists(trans_dir): os.mkdir(trans_dir) local_file = os.path.relpath( os.path.join(trans_dir, '%s_translation' % local_lang, os.curdir)) if lang != slang: satisfies_min = self._satisfies_min_translated( stats[remote_lang], mode) if not satisfies_min: msg = "Skipping language %s due to used options." logger.info(msg % lang) continue logger.warning( " -> %s: %s" % (color_text(remote_lang, "RED"), local_file)) r = self.do_url_request(url, language=remote_lang) base_dir = os.path.split(local_file)[0] mkdir_p(base_dir) fd = open(local_file, 'wb') fd.write(r) fd.close()
def _get_pseudo_file(self, slang, resource, file_filter): pseudo_file = file_filter.replace('<lang>', '%s_pseudo' % slang) return native_path(pseudo_file)
def push(self, source=False, translations=False, force=False, resources=[], languages=[], skip=False, no_interactive=False): """ Push all the resources """ resource_list = self.get_chosen_resources(resources) self.skip = skip self.force = force for resource in resource_list: push_languages = [] project_slug, resource_slug = resource.split('.') files = self.get_resource_files(resource) slang = self.get_resource_option(resource, 'source_lang') sfile = self.get_resource_option(resource, 'source_file') if sfile is not None: sfile = native_path(sfile) lang_map = self.get_resource_lang_mapping(resource) host = self.get_resource_host(resource) logger.debug("Language mapping is: %s" % lang_map) logger.debug("Using host %s" % host) self.url_info = { 'host': host, 'project': project_slug, 'resource': resource_slug } logger.info("Pushing translations for resource %s:" % resource) stats = self._get_stats_for_resource() if force and not no_interactive: answer = raw_input("Warning: By using --force, the uploaded" " files will overwrite remote translations, even if they" " are newer than your uploaded files.\nAre you sure you" " want to continue? [y/N] ") if not answer in ["", 'Y', 'y', "yes", 'YES']: return if source: if sfile is None: logger.error("You don't seem to have a proper source file" " mapping for resource %s. Try without the --source" " option or set a source file first and then try again." % resource) continue # Push source file try: logger.warning("Pushing source file (%s)" % sfile) if not self._resource_exists(stats): logger.info("Resource does not exist. Creating...") fileinfo = "%s;%s" % (resource_slug, slang) filename = self.get_full_path(sfile) self._create_resource(resource, project_slug, fileinfo, filename) self.do_url_request( 'push_source', multipart=True, method="PUT", files=[( "%s;%s" % (resource_slug, slang) , self.get_full_path(sfile) )], ) except Exception, e: if not skip: raise else: logger.error(e) else: try: self.do_url_request('resource_details') except Exception, e: code = getattr(e, 'code', None) if code == 404: msg = "Resource %s doesn't exist on the server." logger.error(msg % resource) continue
def test_posix_path_in_windows_replaces_backslashes(self): orig_path = posix_path(os.path.abspath(os.getcwd())) expected_path = orig_path.replace("/", "\\") self.assertEqual(expected_path, native_path(orig_path))
def pull(self, languages=[], resources=[], overwrite=True, fetchall=False, fetchsource=False, force=False, skip=False, minimum_perc=0, mode=None, pseudo=False): """Pull all translations file from transifex server.""" self.minimum_perc = minimum_perc resource_list = self.get_chosen_resources(resources) skip_decode = False if mode == 'reviewed': url = 'pull_reviewed_file' elif mode == 'translator': url = 'pull_translator_file' elif mode == 'developer': url = 'pull_developer_file' elif mode == 'onlytranslated': url = 'pull_onlytranslated_file' elif mode == 'onlyreviewed': url = 'pull_onlyreviewed_file' else: url = 'pull_file' for resource in resource_list: logger.debug("Handling resource %s" % resource) self.resource = resource project_slug, resource_slug = resource.split('.', 1) files = self.get_resource_files(resource) slang = self.get_resource_option(resource, 'source_lang') sfile = self.get_source_file(resource) lang_map = self.get_resource_lang_mapping(resource) host = self.get_resource_host(resource) logger.debug("Language mapping is: %s" % lang_map) if mode is None: mode = self._get_option(resource, 'mode') self.url_info = { 'host': host, 'project': project_slug, 'resource': resource_slug } logger.debug("URL data are: %s" % self.url_info) stats = self._get_stats_for_resource() details_response, _ = self.do_url_request('resource_details') details = utils.parse_json(details_response) if details['i18n_type'] in self.SKIP_DECODE_I18N_TYPES: skip_decode = True try: file_filter = self.config.get(resource, 'file_filter') except configparser.NoOptionError: file_filter = None # Pull source file pull_languages = set([]) new_translations = set([]) if pseudo: pseudo_file = self._get_pseudo_file(slang, resource, file_filter) if self._should_download(slang, stats, local_file=pseudo_file): logger.info( "Pulling pseudo file for resource %s (%s)." % (resource, utils.color_text(pseudo_file, "RED"))) self._download_pseudo(project_slug, resource_slug, pseudo_file) if not languages: continue if fetchall: new_translations = self._new_translations_to_add( files, slang, lang_map, stats, force) if new_translations: msg = ("New translations found " "for the following languages:%s") logger.info(msg % ', '.join(new_translations)) existing, new = self._languages_to_pull(languages, files, lang_map, stats, force) pull_languages |= existing new_translations |= new logger.debug("Adding to new translations: %s" % new) if fetchsource: if sfile and slang not in pull_languages: pull_languages.add(slang) elif slang not in new_translations: new_translations.add(slang) if pull_languages: logger.debug("Pulling languages for: %s" % pull_languages) msg = "Pulling translations for resource %s (source: %s)" logger.info(msg % (resource, sfile)) for lang in pull_languages: local_lang = lang if lang in list(lang_map.values()): remote_lang = lang_map.flip[lang] else: remote_lang = lang if languages and lang not in pull_languages: logger.debug("Skipping language %s" % lang) continue if lang != slang: local_file = files.get(lang, None) or files[lang_map[lang]] else: local_file = sfile logger.debug("Using file %s" % local_file) kwargs = { 'lang': remote_lang, 'stats': stats, 'local_file': local_file, 'force': force, 'mode': mode, } if not self._should_update_translation(**kwargs): msg = "Skipping '%s' translation (file: %s)." logger.info( msg % (utils.color_text(remote_lang, "RED"), local_file)) continue if not overwrite: local_file = ("%s.new" % local_file) logger.warning( " -> %s: %s" % (utils.color_text(remote_lang, "RED"), local_file)) try: r, charset = self.do_url_request(url, language=remote_lang, skip_decode=skip_decode) except Exception as e: if isinstance(e, SSLError) or not skip: raise else: logger.error(e) continue self._save_file(local_file, charset, r) if new_translations: msg = "Pulling new translations for resource %s (source: %s)" logger.info(msg % (resource, sfile)) for lang in new_translations: if lang in list(lang_map.keys()): local_lang = lang_map[lang] else: local_lang = lang remote_lang = lang if file_filter: local_file = os.path.relpath( os.path.join( self.root, native_path( file_filter.replace('<lang>', local_lang))), os.curdir) else: trans_dir = os.path.join(self.root, ".tx", resource) if not os.path.exists(trans_dir): os.mkdir(trans_dir) local_file = os.path.relpath( os.path.join(trans_dir, '%s_translation' % local_lang, os.curdir)) if lang != slang: satisfies_min = self._satisfies_min_translated( stats[remote_lang], mode) if not satisfies_min: msg = "Skipping language %s due to used options." logger.info(msg % lang) continue logger.warning( " -> %s: %s" % (utils.color_text(remote_lang, "RED"), local_file)) r, charset = self.do_url_request(url, language=remote_lang, skip_decode=skip_decode) self._save_file(local_file, charset, r)
def push(self, source=False, translations=False, force=False, resources=[], languages=[], skip=False, no_interactive=False): """ Push all the resources """ resource_list = self.get_chosen_resources(resources) self.skip = skip self.force = force for resource in resource_list: push_languages = [] project_slug, resource_slug = resource.split('.') files = self.get_resource_files(resource) slang = self.get_resource_option(resource, 'source_lang') sfile = self.get_resource_option(resource, 'source_file') if sfile is not None: sfile = native_path(sfile) lang_map = self.get_resource_lang_mapping(resource) host = self.get_resource_host(resource) verify_ssl(host) logger.debug("Language mapping is: %s" % lang_map) logger.debug("Using host %s" % host) self.url_info = { 'host': host, 'project': project_slug, 'resource': resource_slug } logger.info("Pushing translations for resource %s:" % resource) stats = self._get_stats_for_resource() if force and not no_interactive: answer = raw_input( "Warning: By using --force, the uploaded" " files will overwrite remote translations, even if they" " are newer than your uploaded files.\nAre you sure you" " want to continue? [y/N] ") if not answer in ["", 'Y', 'y', "yes", 'YES']: return if source: if sfile is None: logger.error( "You don't seem to have a proper source file" " mapping for resource %s. Try without the --source" " option or set a source file first and then try again." % resource) continue # Push source file try: logger.warning("Pushing source file (%s)" % sfile) if not self._resource_exists(stats): logger.info("Resource does not exist. Creating...") fileinfo = "%s;%s" % (resource_slug, slang) filename = self.get_full_path(sfile) self._create_resource(resource, project_slug, fileinfo, filename) self.do_url_request( 'push_source', multipart=True, method="PUT", files=[("%s;%s" % (resource_slug, slang), self.get_full_path(sfile))], ) except Exception, e: if not skip: raise else: logger.error(e) else: try: self.do_url_request('resource_details') except Exception, e: code = getattr(e, 'code', None) if code == 404: msg = "Resource %s doesn't exist on the server." logger.error(msg % resource) continue
def test_posix_path_in_windows_replaces_backslashes(self): orig_path = posix_path(os.path.abspath(os.getcwd())) expected_path = orig_path.replace('/', '\\') self.assertEqual(expected_path, native_path(orig_path))
def test_native_path_in_unix_does_nothing(self): orig_path = os.path.abspath(os.getcwd()) path = native_path(orig_path) self.assertEqual(orig_path, path)
def pull(self, languages=[], resources=[], overwrite=True, fetchall=False, fetchsource=False, force=False, skip=False, minimum_perc=0, mode=None, pseudo=False): """Pull all translations file from transifex server.""" self.minimum_perc = minimum_perc resource_list = self.get_chosen_resources(resources) if mode == 'reviewed': url = 'pull_reviewed_file' elif mode == 'translator': url = 'pull_translator_file' elif mode == 'developer': url = 'pull_developer_file' else: url = 'pull_file' for resource in resource_list: logger.debug("Handling resource %s" % resource) self.resource = resource project_slug, resource_slug = resource.split('.', 1) files = self.get_resource_files(resource) slang = self.get_resource_option(resource, 'source_lang') sfile = self.get_source_file(resource) lang_map = self.get_resource_lang_mapping(resource) host = self.get_resource_host(resource) logger.debug("Language mapping is: %s" % lang_map) if mode is None: mode = self._get_option(resource, 'mode') self.url_info = { 'host': host, 'project': project_slug, 'resource': resource_slug } logger.debug("URL data are: %s" % self.url_info) stats = self._get_stats_for_resource() try: file_filter = self.config.get(resource, 'file_filter') except configparser.NoOptionError: file_filter = None # Pull source file pull_languages = set([]) new_translations = set([]) if pseudo: pseudo_file = self._get_pseudo_file( slang, resource, file_filter ) if self._should_download(slang, stats, local_file=pseudo_file): logger.info("Pulling pseudo file for resource %s (%s)." % ( resource, color_text(pseudo_file, "RED") )) self._download_pseudo( project_slug, resource_slug, pseudo_file ) if not languages: continue if fetchall: new_translations = self._new_translations_to_add( files, slang, lang_map, stats, force ) if new_translations: msg = "New translations found for the following languages: %s" logger.info(msg % ', '.join(new_translations)) existing, new = self._languages_to_pull( languages, files, lang_map, stats, force ) pull_languages |= existing new_translations |= new logger.debug("Adding to new translations: %s" % new) if fetchsource: if sfile and slang not in pull_languages: pull_languages.add(slang) elif slang not in new_translations: new_translations.add(slang) if pull_languages: logger.debug("Pulling languages for: %s" % pull_languages) msg = "Pulling translations for resource %s (source: %s)" logger.info(msg % (resource, sfile)) for lang in pull_languages: local_lang = lang if lang in list(lang_map.values()): remote_lang = lang_map.flip[lang] else: remote_lang = lang if languages and lang not in pull_languages: logger.debug("Skipping language %s" % lang) continue if lang != slang: local_file = files.get(lang, None) or files[lang_map[lang]] else: local_file = sfile logger.debug("Using file %s" % local_file) kwargs = { 'lang': remote_lang, 'stats': stats, 'local_file': local_file, 'force': force, 'mode': mode, } if not self._should_update_translation(**kwargs): msg = "Skipping '%s' translation (file: %s)." logger.info( msg % (color_text(remote_lang, "RED"), local_file) ) continue if not overwrite: local_file = ("%s.new" % local_file) logger.warning( " -> %s: %s" % (color_text(remote_lang, "RED"), local_file) ) try: r, charset = self.do_url_request(url, language=remote_lang) except Exception as e: if isinstance(e, SSLError) or not skip: raise else: logger.error(e) continue base_dir = os.path.split(local_file)[0] mkdir_p(base_dir) fd = open(local_file, 'wb') fd.write(r.encode(charset)) fd.close() if new_translations: msg = "Pulling new translations for resource %s (source: %s)" logger.info(msg % (resource, sfile)) for lang in new_translations: if lang in list(lang_map.keys()): local_lang = lang_map[lang] else: local_lang = lang remote_lang = lang if file_filter: local_file = os.path.relpath( os.path.join( self.root, native_path( file_filter.replace('<lang>', local_lang) ) ), os.curdir ) else: trans_dir = os.path.join(self.root, ".tx", resource) if not os.path.exists(trans_dir): os.mkdir(trans_dir) local_file = os.path.relpath(os.path.join(trans_dir, '%s_translation' % local_lang, os.curdir)) if lang != slang: satisfies_min = self._satisfies_min_translated( stats[remote_lang], mode ) if not satisfies_min: msg = "Skipping language %s due to used options." logger.info(msg % lang) continue logger.warning( " -> %s: %s" % (color_text(remote_lang, "RED"), local_file) ) r, charset = self.do_url_request(url, language=remote_lang) base_dir = os.path.split(local_file)[0] mkdir_p(base_dir) fd = open(local_file, 'wb') fd.write(r.encode(charset)) fd.close()