def test_unescape_of_quoted_newline(self):
# regression test for #198
self.assertEqual(r'\n', pofile.unescape(r'"\\n"'))
def test_unescape(self):
escaped = u'"Say:\\n \\"hello, world!\\"\\n"'
unescaped = u'Say:\n "hello, world!"\n'
self.assertNotEqual(unescaped, escaped)
self.assertEqual(unescaped, pofile.unescape(escaped))
def main():
if not os.path.isdir('locale'):
script_dir = os.path.dirname(os.path.realpath(__file__))
os.chdir(script_dir)
if len(sys.argv) == 1:
locale_dir = 'locale'
elif len(sys.argv) == 2:
locale_dir = sys.argv[1]
else:
print "Usage: i18n-extract.py [<locale path>]"
sys.exit(1)
locale_dir = os.path.realpath(locale_dir)
splunk_home = os.environ.get('SPLUNK_HOME')
if not splunk_home:
print "SPLUNK_HOME environment variable was not set!"
sys.exit(2)
print locale_dir
print splunk_home
if locale_dir.startswith(splunk_home):
strip = splunk_home.replace(os.path.sep, '/')
strip = (strip+'/share/splunk', strip, 'etc/apps/')
template_dir = os.path.join(splunk_home, 'share/splunk/search_mrsparkle')
default_dir = os.path.join(splunk_home, 'etc/system/default')
search_app = os.path.join(splunk_home, 'etc/apps/search')
launcher_app = os.path.join(splunk_home, 'etc/apps/launcher')
getting_started_app = os.path.join(splunk_home, 'etc/apps/gettingstarted')
stubby_app = os.path.join(splunk_home, 'etc/apps/stubby')
datapreview_app = os.path.join(splunk_home, 'etc/apps/splunk_datapreview')
else:
# assume this is an extraction from the source tree
strip = ('../../../../web', '../../../../', 'cfg/bundles/')
template_dir = '../../../../web/search_mrsparkle'
default_dir = '../../../../cfg/bundles/default'
search_app = '../../../../cfg/bundles/search'
launcher_app = '../../../../cfg/bundles/launcher'
getting_started_app = '../../../../cfg/bundles/gettingstarted'
stubby_app = '../../../../cfg/bundles/stubby'
datapreview_app = '../../../../cfg/bundles/splunk_datapreview'
# this is always relative to the script directory
search_helper_dir = '../../searchhelp'
args = [
'extract',
'-F', os.path.join(locale_dir, 'babel.cfg'),
'-c', 'TRANS:',
'-k', 'deferred_ugettext',
'-k', 'deferred_ungettext',
'.',
template_dir,
default_dir,
search_app,
launcher_app,
getting_started_app,
datapreview_app
#stubby_app
]
sys.argv[1:] = args
# Open the .pot file for write
outfile = open(os.path.join(locale_dir, 'messages.pot'), 'w')
# Capture Babel's stdout so we can translate the absolute pathnames into relative ones
buf = StringIO()
stdout_org = sys.stdout
sys.stdout = buf
# Do the extraction
frontend.main()
# restore stdout
sys.stdout = stdout_org
# Start reading the captured data from the top
buf.reset()
print >>outfile, HEADER
currentfn = []
filemapping = {}
msgid = msgid_plural = None
for line in buf:
line = line.strip()
if line.startswith('# '):
# strip the original comment header
continue
if line[:3] != '#: ': # filename:linenum references begin with #:
print >>outfile, line
if currentfn:
# capture the translation associated with the current filename(s)
if line.startswith('msgid '):
msgid = unescape(line[6:])
elif line.startswith('msgid_plural '):
msgid_plural = unescape(line[13:])
elif line.startswith('"'):
# multi-line translation
if msgid is not None:
msgid = msgid + unescape(line)
elif msgid_plural is not None:
msgid_plural = msgid_plural + unescape(line)
continue
if msgid and currentfn:
for fn in currentfn:
fn = fn.lower()
if fn.find('data' + os.sep + 'ui') > -1:
fn = os.path.splitext(os.path.basename(fn))[0]
filemapping.setdefault(fn, []).append( (msgid, msgid_plural) )
msgid = msgid_plural = None
currentfn = []
newline = '#: '
fnpairs = line[3:].split(' ')
for fnpair in fnpairs:
fn, ln = fnpair.rsplit(':', 1)
for prefix in strip:
if fn.startswith(prefix):
fn = fn[len(prefix):].strip('/')
# keep track of js files
if fn.endswith('.js') or fn.find('data' + os.sep + 'ui') > -1:
currentfn.append(fn)
newline += "%s:%s " % (fn, ln)
print >>outfile, newline
outfile.close()
# collect the final message
if msgid and currentfn:
for fn in currentfn:
if fn.find('data' + os.sep + 'ui') > -1:
fn = os.path.splitext(os.path.basename(fn))[0]
filemapping.setdefault(fn.lower(), []).append( (msgid, msgid_plural) )
# pickle the lookup data
cachefile = open(os.path.join(locale_dir, "messages-filecache.bin"), 'wb')
pickle.dump(filemapping, cachefile, 2)
cachefile.close()
def find_trans_keys_for_file_direct(filename):
"""
Lookup a list of strings to be translated for a given file
This relies on the messages.pot file being correctly marked up with
the name of the source file associated with each string to be translated
"""
filename = os.path.realpath(filename).lower()
app_prefix = util.get_apps_dir().lower()
app_name = None
if filename.startswith(app_prefix):
# if this is for a file defined in an app, then check it's local messages.pot file
app_name = filename[len(app_prefix):].split(os.path.sep, 2)[1]
potfile = os.path.join(app_prefix, app_name, 'locale', 'messages.pot')
if app_name in INTERNAL_APPS:
potfile = os.path.join(LOCALE_PATH, 'messages.pot')
else:
# else use the system messages.pot
potfile = os.path.join(LOCALE_PATH, 'messages.pot')
result = []
try:
f = open(potfile, 'r')
except IOError:
if app_name:
logger.debug(
"Application %s does not contain a messages.pot file" %
app_name)
return result
else:
raise # Can't find the appserver messages.pot file == fatal.
matched = False
msgid = msgid_plural = None
for line in f:
line = line.strip()
if line[:2] == '#:':
fn = line[2:].rsplit(':',
1)[0].strip() # split off the line number
try:
# strip the relative portion of the filename
fn = fn[fn.rindex('../') + 3:]
except ValueError:
pass
# hack for search_mrsparkle directories
if fn.startswith('web/'):
fn = fn[4:]
if msgid:
# process the existing entry for the last match
result.append((msgid, msgid_plural))
msgid = msgid_plural = None
matched = False
if not matched:
# multiple files may use this string; we only need to know if 1 matched
matched = fn.lower() == filename[-len(fn):].replace(
os.path.sep, "/")
elif matched:
if line.startswith('msgid '):
msgid = unescape(line[6:])
elif line.startswith('msgid_plural '):
msgid_plural = unescape(line[13:])
elif line.startswith('"'):
if msgid is not None:
msgid = msgid + unescape(line)
elif msgid_plural is not None:
msgid_plural = msgid_plural + unescape(line)
if msgid:
result.append((msgid, msgid_plural))
f.close()
return result
def find_trans_keys_for_file_direct(filename):
"""
Lookup a list of strings to be translated for a given file
This relies on the messages.pot file being correctly marked up with
the name of the source file associated with each string to be translated
"""
filename = os.path.realpath(filename).lower()
app_prefix = util.get_apps_dir().lower()
app_name = None
if filename.startswith(app_prefix):
# if this is for a file defined in an app, then check it's local messages.pot file
app_name =filename[len(app_prefix):].split(os.path.sep, 2)[1]
potfile = os.path.join(app_prefix, app_name, 'locale', 'messages.pot')
if app_name in INTERNAL_APPS:
potfile = os.path.join(LOCALE_PATH, 'messages.pot')
else:
# else use the system messages.pot
potfile = os.path.join(LOCALE_PATH, 'messages.pot')
result = []
try:
f = open(potfile, 'r')
except IOError:
if app_name:
logger.debug("Application %s does not contain a messages.pot file" % app_name)
return result
else:
raise # Can't find the appserver messages.pot file == fatal.
matched = False
msgid = msgid_plural = None
for line in f:
line = line.strip()
if line[:2] == '#:':
fn = line[2:].rsplit(':',1)[0].strip() # split off the line number
try:
# strip the relative portion of the filename
fn = fn[fn.rindex('../')+3:]
except ValueError:
pass
# hack for search_mrsparkle directories
if fn.startswith('web/'):
fn = fn[4:]
if msgid:
# process the existing entry for the last match
result.append( (msgid, msgid_plural) )
msgid = msgid_plural = None
matched = False
if not matched:
# multiple files may use this string; we only need to know if 1 matched
matched = fn.lower() == filename[-len(fn):].replace(os.path.sep, "/")
elif matched:
if line.startswith('msgid '):
msgid = unescape(line[6:])
elif line.startswith('msgid_plural '):
msgid_plural = unescape(line[13:])
elif line.startswith('"'):
if msgid is not None:
msgid = msgid + unescape(line)
elif msgid_plural is not None:
msgid_plural = msgid_plural + unescape(line)
if msgid:
result.append( (msgid, msgid_plural) )
f.close()
return result
def main():
if not os.path.isdir('locale'):
script_dir = os.path.dirname(os.path.realpath(__file__))
os.chdir(script_dir)
if len(sys.argv) == 1:
locale_dir = 'locale'
elif len(sys.argv) == 2:
locale_dir = sys.argv[1]
else:
print "Usage: i18n-extract.py [<locale path>]"
sys.exit(1)
locale_dir = os.path.realpath(locale_dir)
splunk_home = os.environ.get('SPLUNK_HOME')
if not splunk_home:
print "SPLUNK_HOME environment variable was not set!"
sys.exit(2)
print locale_dir
print splunk_home
if locale_dir.startswith(splunk_home):
strip = splunk_home.replace(os.path.sep, '/')
strip = (strip+'/share/splunk', strip, 'etc/apps/')
template_dir = os.path.join(splunk_home, 'share/splunk/search_mrsparkle/templates')
js_dir = os.path.join(splunk_home, 'share/splunk/search_mrsparkle/exposed/js')
modules_dir = os.path.join(splunk_home, 'share/splunk/search_mrsparkle/modules')
default_dir = os.path.join(splunk_home, 'etc/system/default')
search_app = os.path.join(splunk_home, 'etc/apps/search')
launcher_app = os.path.join(splunk_home, 'etc/apps/launcher')
getting_started_app = os.path.join(splunk_home, 'etc/apps/gettingstarted')
deployment_app = os.path.join(splunk_home, 'etc/apps/SplunkDeploymentMonitor')
stubby_app = os.path.join(splunk_home, 'etc/apps/stubby')
else:
# assume this is an extraction from the source tree
strip = ('../../../../web', '../../../../', 'cfg/bundles/')
template_dir = '../../../../web/search_mrsparkle/templates'
js_dir = '../../../../web/search_mrsparkle/exposed/js'
modules_dir = '../../../../web/search_mrsparkle/modules'
default_dir = '../../../../cfg/bundles/default'
search_app = '../../../../cfg/bundles/search'
launcher_app = '../../../../cfg/bundles/launcher'
getting_started_app = '../../../../cfg/bundles/gettingstarted'
deployment_app = '../../../../cfg/bundles/SplunkDeploymentMonitor'
stubby_app = '../../../../cfg/bundles/stubby'
# this is always relative to the script directory
search_helper_dir = '../../searchhelp'
args = [
'extract',
'-F', os.path.join(locale_dir, 'babel.cfg'),
'-c', 'TRANS:',
'-k', 'deferred_ugettext',
'-k', 'deferred_ungettext',
'.',
template_dir,
js_dir,
modules_dir,
default_dir,
search_app,
launcher_app,
getting_started_app,
deployment_app
#stubby_app
]
sys.argv[1:] = args
# Open the .pot file for write
outfile = open(os.path.join(locale_dir, 'messages.pot'), 'w')
# Capture Babel's stdout so we can translate the absolute pathnames into relative ones
buf = StringIO()
stdout_org = sys.stdout
sys.stdout = buf
# Do the extraction
frontend.main()
# restore stdout
sys.stdout = stdout_org
# Start reading the captured data from the top
buf.reset()
print >>outfile, HEADER
currentfn = []
filemapping = {}
msgid = msgid_plural = None
for line in buf:
line = line.strip()
if line.startswith('# '):
# strip the original comment header
continue
if line[:3] != '#: ': # filename:linenum references begin with #:
print >>outfile, line
if currentfn:
# capture the translation associated with the current filename(s)
if line.startswith('msgid '):
msgid = unescape(line[6:])
elif line.startswith('msgid_plural '):
msgid_plural = unescape(line[13:])
elif line.startswith('"'):
# multi-line translation
if msgid is not None:
msgid = msgid + unescape(line)
elif msgid_plural is not None:
msgid_plural = msgid_plural + unescape(line)
continue
if msgid and currentfn:
for fn in currentfn:
filemapping.setdefault(fn.lower(), []).append( (msgid, msgid_plural) )
msgid = msgid_plural = None
currentfn = []
newline = '#: '
fnpairs = line[3:].split(' ')
for fnpair in fnpairs:
fn, ln = fnpair.rsplit(':', 1)
for prefix in strip:
if fn.startswith(prefix):
fn = fn[len(prefix):].strip('/')
# keep track of js files
if fn.endswith('.js'):
currentfn.append(fn)
newline += "%s:%s " % (fn, ln)
print >>outfile, newline
outfile.close()
# collect the final message
if msgid and currentfn:
for fn in currentfn:
filemapping.setdefault(fn.lower(), []).append( (msgid, msgid_plural) )
# pickle the lookup data
cachefile = open(os.path.join(locale_dir, "messages-filecache.bin"), 'wb')
pickle.dump(filemapping, cachefile, 2)
cachefile.close()
