def test_unescape_of_quoted_newline(self): # regression test for #198 self.assertEqual(r'\n', pofile.unescape(r'"\\n"'))
def test_unescape(self): escaped = u'"Say:\\n \\"hello, world!\\"\\n"' unescaped = u'Say:\n "hello, world!"\n' self.assertNotEqual(unescaped, escaped) self.assertEqual(unescaped, pofile.unescape(escaped))
def main(): if not os.path.isdir('locale'): script_dir = os.path.dirname(os.path.realpath(__file__)) os.chdir(script_dir) if len(sys.argv) == 1: locale_dir = 'locale' elif len(sys.argv) == 2: locale_dir = sys.argv[1] else: print "Usage: i18n-extract.py [<locale path>]" sys.exit(1) locale_dir = os.path.realpath(locale_dir) splunk_home = os.environ.get('SPLUNK_HOME') if not splunk_home: print "SPLUNK_HOME environment variable was not set!" sys.exit(2) print locale_dir print splunk_home if locale_dir.startswith(splunk_home): strip = splunk_home.replace(os.path.sep, '/') strip = (strip+'/share/splunk', strip, 'etc/apps/') template_dir = os.path.join(splunk_home, 'share/splunk/search_mrsparkle') default_dir = os.path.join(splunk_home, 'etc/system/default') search_app = os.path.join(splunk_home, 'etc/apps/search') launcher_app = os.path.join(splunk_home, 'etc/apps/launcher') getting_started_app = os.path.join(splunk_home, 'etc/apps/gettingstarted') stubby_app = os.path.join(splunk_home, 'etc/apps/stubby') datapreview_app = os.path.join(splunk_home, 'etc/apps/splunk_datapreview') else: # assume this is an extraction from the source tree strip = ('../../../../web', '../../../../', 'cfg/bundles/') template_dir = '../../../../web/search_mrsparkle' default_dir = '../../../../cfg/bundles/default' search_app = '../../../../cfg/bundles/search' launcher_app = '../../../../cfg/bundles/launcher' getting_started_app = '../../../../cfg/bundles/gettingstarted' stubby_app = '../../../../cfg/bundles/stubby' datapreview_app = '../../../../cfg/bundles/splunk_datapreview' # this is always relative to the script directory search_helper_dir = '../../searchhelp' args = [ 'extract', '-F', os.path.join(locale_dir, 'babel.cfg'), '-c', 'TRANS:', '-k', 'deferred_ugettext', '-k', 'deferred_ungettext', '.', template_dir, default_dir, search_app, launcher_app, getting_started_app, datapreview_app #stubby_app ] sys.argv[1:] = args # Open the .pot file for write outfile = open(os.path.join(locale_dir, 'messages.pot'), 'w') # Capture Babel's stdout so we can translate the absolute pathnames into relative ones buf = StringIO() stdout_org = sys.stdout sys.stdout = buf # Do the extraction frontend.main() # restore stdout sys.stdout = stdout_org # Start reading the captured data from the top buf.reset() print >>outfile, HEADER currentfn = [] filemapping = {} msgid = msgid_plural = None for line in buf: line = line.strip() if line.startswith('# '): # strip the original comment header continue if line[:3] != '#: ': # filename:linenum references begin with #: print >>outfile, line if currentfn: # capture the translation associated with the current filename(s) if line.startswith('msgid '): msgid = unescape(line[6:]) elif line.startswith('msgid_plural '): msgid_plural = unescape(line[13:]) elif line.startswith('"'): # multi-line translation if msgid is not None: msgid = msgid + unescape(line) elif msgid_plural is not None: msgid_plural = msgid_plural + unescape(line) continue if msgid and currentfn: for fn in currentfn: fn = fn.lower() if fn.find('data' + os.sep + 'ui') > -1: fn = os.path.splitext(os.path.basename(fn))[0] filemapping.setdefault(fn, []).append( (msgid, msgid_plural) ) msgid = msgid_plural = None currentfn = [] newline = '#: ' fnpairs = line[3:].split(' ') for fnpair in fnpairs: fn, ln = fnpair.rsplit(':', 1) for prefix in strip: if fn.startswith(prefix): fn = fn[len(prefix):].strip('/') # keep track of js files if fn.endswith('.js') or fn.find('data' + os.sep + 'ui') > -1: currentfn.append(fn) newline += "%s:%s " % (fn, ln) print >>outfile, newline outfile.close() # collect the final message if msgid and currentfn: for fn in currentfn: if fn.find('data' + os.sep + 'ui') > -1: fn = os.path.splitext(os.path.basename(fn))[0] filemapping.setdefault(fn.lower(), []).append( (msgid, msgid_plural) ) # pickle the lookup data cachefile = open(os.path.join(locale_dir, "messages-filecache.bin"), 'wb') pickle.dump(filemapping, cachefile, 2) cachefile.close()
def find_trans_keys_for_file_direct(filename): """ Lookup a list of strings to be translated for a given file This relies on the messages.pot file being correctly marked up with the name of the source file associated with each string to be translated """ filename = os.path.realpath(filename).lower() app_prefix = util.get_apps_dir().lower() app_name = None if filename.startswith(app_prefix): # if this is for a file defined in an app, then check it's local messages.pot file app_name = filename[len(app_prefix):].split(os.path.sep, 2)[1] potfile = os.path.join(app_prefix, app_name, 'locale', 'messages.pot') if app_name in INTERNAL_APPS: potfile = os.path.join(LOCALE_PATH, 'messages.pot') else: # else use the system messages.pot potfile = os.path.join(LOCALE_PATH, 'messages.pot') result = [] try: f = open(potfile, 'r') except IOError: if app_name: logger.debug( "Application %s does not contain a messages.pot file" % app_name) return result else: raise # Can't find the appserver messages.pot file == fatal. matched = False msgid = msgid_plural = None for line in f: line = line.strip() if line[:2] == '#:': fn = line[2:].rsplit(':', 1)[0].strip() # split off the line number try: # strip the relative portion of the filename fn = fn[fn.rindex('../') + 3:] except ValueError: pass # hack for search_mrsparkle directories if fn.startswith('web/'): fn = fn[4:] if msgid: # process the existing entry for the last match result.append((msgid, msgid_plural)) msgid = msgid_plural = None matched = False if not matched: # multiple files may use this string; we only need to know if 1 matched matched = fn.lower() == filename[-len(fn):].replace( os.path.sep, "/") elif matched: if line.startswith('msgid '): msgid = unescape(line[6:]) elif line.startswith('msgid_plural '): msgid_plural = unescape(line[13:]) elif line.startswith('"'): if msgid is not None: msgid = msgid + unescape(line) elif msgid_plural is not None: msgid_plural = msgid_plural + unescape(line) if msgid: result.append((msgid, msgid_plural)) f.close() return result
def find_trans_keys_for_file_direct(filename): """ Lookup a list of strings to be translated for a given file This relies on the messages.pot file being correctly marked up with the name of the source file associated with each string to be translated """ filename = os.path.realpath(filename).lower() app_prefix = util.get_apps_dir().lower() app_name = None if filename.startswith(app_prefix): # if this is for a file defined in an app, then check it's local messages.pot file app_name =filename[len(app_prefix):].split(os.path.sep, 2)[1] potfile = os.path.join(app_prefix, app_name, 'locale', 'messages.pot') if app_name in INTERNAL_APPS: potfile = os.path.join(LOCALE_PATH, 'messages.pot') else: # else use the system messages.pot potfile = os.path.join(LOCALE_PATH, 'messages.pot') result = [] try: f = open(potfile, 'r') except IOError: if app_name: logger.debug("Application %s does not contain a messages.pot file" % app_name) return result else: raise # Can't find the appserver messages.pot file == fatal. matched = False msgid = msgid_plural = None for line in f: line = line.strip() if line[:2] == '#:': fn = line[2:].rsplit(':',1)[0].strip() # split off the line number try: # strip the relative portion of the filename fn = fn[fn.rindex('../')+3:] except ValueError: pass # hack for search_mrsparkle directories if fn.startswith('web/'): fn = fn[4:] if msgid: # process the existing entry for the last match result.append( (msgid, msgid_plural) ) msgid = msgid_plural = None matched = False if not matched: # multiple files may use this string; we only need to know if 1 matched matched = fn.lower() == filename[-len(fn):].replace(os.path.sep, "/") elif matched: if line.startswith('msgid '): msgid = unescape(line[6:]) elif line.startswith('msgid_plural '): msgid_plural = unescape(line[13:]) elif line.startswith('"'): if msgid is not None: msgid = msgid + unescape(line) elif msgid_plural is not None: msgid_plural = msgid_plural + unescape(line) if msgid: result.append( (msgid, msgid_plural) ) f.close() return result
def main(): if not os.path.isdir('locale'): script_dir = os.path.dirname(os.path.realpath(__file__)) os.chdir(script_dir) if len(sys.argv) == 1: locale_dir = 'locale' elif len(sys.argv) == 2: locale_dir = sys.argv[1] else: print "Usage: i18n-extract.py [<locale path>]" sys.exit(1) locale_dir = os.path.realpath(locale_dir) splunk_home = os.environ.get('SPLUNK_HOME') if not splunk_home: print "SPLUNK_HOME environment variable was not set!" sys.exit(2) print locale_dir print splunk_home if locale_dir.startswith(splunk_home): strip = splunk_home.replace(os.path.sep, '/') strip = (strip+'/share/splunk', strip, 'etc/apps/') template_dir = os.path.join(splunk_home, 'share/splunk/search_mrsparkle/templates') js_dir = os.path.join(splunk_home, 'share/splunk/search_mrsparkle/exposed/js') modules_dir = os.path.join(splunk_home, 'share/splunk/search_mrsparkle/modules') default_dir = os.path.join(splunk_home, 'etc/system/default') search_app = os.path.join(splunk_home, 'etc/apps/search') launcher_app = os.path.join(splunk_home, 'etc/apps/launcher') getting_started_app = os.path.join(splunk_home, 'etc/apps/gettingstarted') deployment_app = os.path.join(splunk_home, 'etc/apps/SplunkDeploymentMonitor') stubby_app = os.path.join(splunk_home, 'etc/apps/stubby') else: # assume this is an extraction from the source tree strip = ('../../../../web', '../../../../', 'cfg/bundles/') template_dir = '../../../../web/search_mrsparkle/templates' js_dir = '../../../../web/search_mrsparkle/exposed/js' modules_dir = '../../../../web/search_mrsparkle/modules' default_dir = '../../../../cfg/bundles/default' search_app = '../../../../cfg/bundles/search' launcher_app = '../../../../cfg/bundles/launcher' getting_started_app = '../../../../cfg/bundles/gettingstarted' deployment_app = '../../../../cfg/bundles/SplunkDeploymentMonitor' stubby_app = '../../../../cfg/bundles/stubby' # this is always relative to the script directory search_helper_dir = '../../searchhelp' args = [ 'extract', '-F', os.path.join(locale_dir, 'babel.cfg'), '-c', 'TRANS:', '-k', 'deferred_ugettext', '-k', 'deferred_ungettext', '.', template_dir, js_dir, modules_dir, default_dir, search_app, launcher_app, getting_started_app, deployment_app #stubby_app ] sys.argv[1:] = args # Open the .pot file for write outfile = open(os.path.join(locale_dir, 'messages.pot'), 'w') # Capture Babel's stdout so we can translate the absolute pathnames into relative ones buf = StringIO() stdout_org = sys.stdout sys.stdout = buf # Do the extraction frontend.main() # restore stdout sys.stdout = stdout_org # Start reading the captured data from the top buf.reset() print >>outfile, HEADER currentfn = [] filemapping = {} msgid = msgid_plural = None for line in buf: line = line.strip() if line.startswith('# '): # strip the original comment header continue if line[:3] != '#: ': # filename:linenum references begin with #: print >>outfile, line if currentfn: # capture the translation associated with the current filename(s) if line.startswith('msgid '): msgid = unescape(line[6:]) elif line.startswith('msgid_plural '): msgid_plural = unescape(line[13:]) elif line.startswith('"'): # multi-line translation if msgid is not None: msgid = msgid + unescape(line) elif msgid_plural is not None: msgid_plural = msgid_plural + unescape(line) continue if msgid and currentfn: for fn in currentfn: filemapping.setdefault(fn.lower(), []).append( (msgid, msgid_plural) ) msgid = msgid_plural = None currentfn = [] newline = '#: ' fnpairs = line[3:].split(' ') for fnpair in fnpairs: fn, ln = fnpair.rsplit(':', 1) for prefix in strip: if fn.startswith(prefix): fn = fn[len(prefix):].strip('/') # keep track of js files if fn.endswith('.js'): currentfn.append(fn) newline += "%s:%s " % (fn, ln) print >>outfile, newline outfile.close() # collect the final message if msgid and currentfn: for fn in currentfn: filemapping.setdefault(fn.lower(), []).append( (msgid, msgid_plural) ) # pickle the lookup data cachefile = open(os.path.join(locale_dir, "messages-filecache.bin"), 'wb') pickle.dump(filemapping, cachefile, 2) cachefile.close()