def exit_on_exception(func, cleanup=None):
"""
Gracefully exit a Pology script when an exception is received.
Any error message will be printed, any progress lines will be cleared,
and keyboard interrupt will exist silently.
The backtrace can be shown instead (on non-keyboard interrupt exceptions)
by setting C{[global]/show-backtrace} user configuration field to true.
@param func: a zero-argument function
@type func: () -> any
@param cleanup: a zero-argument function to execute before exiting
@type cleanup: () -> any
@returns: path of current working directory
@rtype: string
"""
try:
func()
except KeyboardInterrupt:
report("", newline=False)
if cleanup:
cleanup()
exit(100)
except Exception, e:
report("", newline=False)
if cleanup:
cleanup()
if pology.config.section("global").boolean("show-backtrace"):
raise
else:
error(str_to_unicode(str(e)), code=1)
def _value(self, typ, name, default=None, typename=None):
if not _config.has_option(self.name, name):
return default
value = _config.get(self.name, name)
if typ is not bool:
try:
cvalue = typ(value)
except:
cvalue = None
else:
cvalue = strbool(value)
if cvalue is None:
if typename:
error(
_("@info", "User configuration: value '%(val)s' "
"of field '%(field)s' in section '%(sec)s' "
"cannot be converted into '%(type)s' type.",
val=value,
field=name,
sec=self.name,
type=typename))
else:
error(
_("@info", "User configuration: value '%(val)s' "
"of field '%(field)s' in section '%(sec)s' "
"cannot be converted into requested type.",
val=value,
field=name,
sec=self.name))
return cvalue
def translate(self, texts):
import urllib
try:
import simplejson
except:
error(
_("@info", "Python module '%(mod)s' not available. "
"Try installing the '%(pkg)s' package.",
mod="simplejson",
pkg="python-simplejson"))
baseurl = "https://www.googleapis.com/language/translate/v2"
baseparams = (("key", self.apikey), ("source", self.lang_in),
("target", self.lang_out), ("target", "json"))
texts_tr = []
for text in texts:
params = baseparams + (("q", text.encode("utf8")), )
parfmt = "&".join(
["%s=%s" % (p, urllib.quote_plus(v)) for p, v in params])
execurl = "%s?%s" % (baseurl, parfmt)
try:
res = simplejson.load(urllib.FancyURLopener().open(execurl))
text_tr = unicode(
res["data"]["translations"][0]["translatedText"])
except:
text_tr = u""
texts_tr.append(text_tr)
return texts_tr
def __init__(self, slang, tlang, options):
cmdpath = options.transerv_bin or "apertium"
try:
subprocess.call(cmdpath,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError:
error(
_("@info Apertium is machine translation software",
"Apertium executable not found at '%(path)s'.",
path=cmdpath))
if options.tmode is not None:
mode = options.tmode
else:
mode = "%s-%s" % (slang, tlang)
optional_parameters = u""
if options.data_directory:
optional_parameters = u"-d %s" % options.data_directory
self.cmdline = u"%s -u -f html-noent %s %s" % (
cmdpath, optional_parameters, mode)
entpath = os.path.join(datadir(), "spec", "html.entities")
self.htmlents = read_entities(entpath)
def _raise_or_abort(errmsg, abort, exc=PologyError):
"""
Raise an exception or abort execution with given error message,
based on the value of C{abort}.
"""
if abort:
error(errmsg)
else:
raise exc(errmsg)
def main ():
locale.setlocale(locale.LC_ALL, "")
# Get defaults for command line options from global config.
cfgsec = pology_config.section("porewrap")
# Setup options and parse the command line.
usage = _("@info command usage",
"%(cmd)s [options] POFILE...",
cmd="%prog")
desc = _("@info command description",
"Rewrap message strings in PO files.")
ver = _("@info command version",
u"%(cmd)s (Pology) %(version)s\n"
u"Copyright © 2007, 2008, 2009, 2010 "
u"Chusslove Illich (Часлав Илић) <%(email)s>",
cmd="%prog", version=version(), email="*****@*****.**")
opars = ColorOptionParser(usage=usage, description=desc, version=ver)
opars.add_option(
"-v", "--verbose",
action="store_true", dest="verbose", default=False,
help=_("@info command line option description",
"More detailed progress information."))
add_cmdopt_wrapping(opars)
add_cmdopt_filesfrom(opars)
(op, fargs) = opars.parse_args()
if len(fargs) < 1 and not op.files_from:
error(_("@info", "No input files given."))
# Could use some speedup.
try:
import psyco
psyco.full()
except ImportError:
pass
# Assemble list of files.
fnames = collect_paths_cmdline(rawpaths=fargs,
filesfrom=op.files_from,
respathf=collect_catalogs,
abort=True)
# Rewrap all catalogs.
for fname in fnames:
if op.verbose:
report(_("@info:progress", "Rewrapping: %(file)s", file=fname))
cat = Catalog(fname, monitored=False)
wrapping = select_field_wrapping(cfgsec, cat, op)
cat.set_wrapping(wrapping)
cat.sync(force=True)
def assert_system(cmdline, echo=False, wdir=None):
"""
Execute command line and assert success.
If the command exits with non-zero zero state, the program aborts.
C{cmdline} can be either a monolithic string, in which case it is
executed through a shell, or a list of argument strings,
when the process is started directly with these arguments.
C{cmdline} is processed with L{unicode_to_str} to convert any
unicode strings to raw byte strings in expected system encoding.
@param cmdline: command line to execute
@type cmdline: string
@param echo: whether to echo the supplied command line
@type echo: bool
@param wdir: working directory for the command (CWD if none given)
@type wdir: path
"""
if echo:
if isinstance(cmdline, basestring):
cmdstr = cmdline
else:
cmdstr = " ".join(map(escape_sh, cmdline))
report(cmdstr)
if wdir is not None:
cwd = getucwd()
os.chdir(wdir)
if isinstance(cmdline, basestring):
cmdline = unicode_to_str(cmdline)
shell = True
else:
cmdline = map(unicode_to_str, cmdline)
shell = False
ret = subprocess.call(cmdline, shell=shell)
if wdir is not None:
os.chdir(cwd)
if ret:
if echo:
error(_("@info", "Non-zero exit from the previous command."))
else:
error(
_("@info",
"Non-zero exit from the command:\n%(cmdline)s",
cmdline=cmdline))
def get_transerv(slang, tlang, scat, tcat, tsbuilder):
if not slang:
slang = scat.header.get_field_value("Language")
if not slang:
error(
_("@info",
"Cannot determine language of source catalog '%(file)s'.",
file=scat.filename))
if not tlang:
tlang = tcat.header.get_field_value("Language")
if not tlang:
error(
_("@info",
"Cannot determine language of target catalog '%(file)s'.",
file=tcat.filename))
trdir = (slang, tlang)
if trdir not in _transervs:
_transervs[trdir] = tsbuilder(slang, tlang)
return _transervs[trdir]
def main():
locale.setlocale(locale.LC_ALL, "")
# FIXME: Use pology.colors.ColorOptionParser.
reminv = False
paths = []
for arg in sys.argv[1:]:
if arg.startswith("-"):
if arg in ("-r", "--remove-invalid"):
reminv = True
else:
error(
_("@info",
"Unknown option '%(opt)s' in command line.",
opt=arg))
else:
paths.append(arg)
if len(paths) < 1:
usage()
for path in paths:
organize(path, reminv)
def error_on_msg(text, msg, cat, code=1, subsrc=None, file=sys.stderr):
"""
Error on a PO message (aborts the execution).
Outputs the message reference (catalog name and message position),
along with the error text. Aborts execution with the given code.
@param text: text to report
@type text: string
@param msg: the message for which the text is reported
@type msg: L{Message_base}
@param cat: the catalog where the message lives
@type cat: L{Catalog}
@param code: the exit code
@type code: int
@param subsrc: more detailed source of the message
@type subsrc: C{None} or string
@param file: send output to this file descriptor
@type file: C{file}
"""
posinfo = _msg_pos_fmt(cat.filename, msg.refline, msg.refentry)
text = cinterp("%s: %s", posinfo, text)
error(text, code=code, subsrc=subsrc, showcmd=True)
def strdlist(self, name, default=None):
"""
Get a configuration field as a list of delimited strings.
Delimiter is taken to be the non-alphanumeric character with
which the field value starts. In this example::
afield = /foo/bar/baz/
the delimiter is C{/}.
If the field value does not start with a non-alphanumeric,
or it does not end with the delimiter, error is signalled.
@rtype: unicode or as C{default}
"""
value = self._value(unicode, name, None, "string")
if value is None:
return default
value = value.strip()
if len(value) < 2:
error(
_("@info",
"User configuration: value '%(val)s' of field '%(field)s' "
"in section '%(sec)s' is too short for a delimited list.",
val=value,
field=name,
sec=self.name))
if value[0].isalnum():
error(
_("@info",
"User configuration: value '%(val)s' of field '%(field)s' "
"in section '%(sec)s' does not start with "
"a non-alphanumeric delimiter character.",
val=value,
field=name,
sec=self.name))
delim = value[0]
if value[-1] != delim:
error(
_("@info",
"User configuration: value '%(val)s' of field '%(field)s' "
"in section '%(sec)s' does not end with "
"the delimiter character with which it starts.",
val=value,
field=name,
sec=self.name))
lst = value[1:-1].split(delim)
return lst
def translate_parallel(paths, tsbuilder, options):
pathrepl = options.parcats
comppath = options.parcomp
slang = options.slang
tlang = options.tlang
ccat = None
if comppath is not None:
if not os.path.isfile(comppath):
error(
_("@info",
"Compendium '%(file)s' does not exist.",
file=comppath))
ccat = Catalog(comppath, monitored=False)
if pathrepl is not None:
lst = pathrepl.split(":")
if len(lst) != 2:
error(
_("@info",
"Invalid search and replace specification '%(spec)s'.",
spec=pathrepl))
pathsrch, pathrepl = lst
catpaths = collect_catalogs(paths)
for catpath in catpaths:
# Open parallel catalog if it exists.
pcat = None
if pathrepl is not None:
pcatpath = catpath.replace(pathsrch, pathrepl, 1)
if catpath == pcatpath:
error(
_("@info",
"Parallel catalog and target catalog are same files "
"for '%(file)s'.",
file=catpath))
if os.path.isfile(pcatpath):
pcat = Catalog(pcatpath, monitored=False)
# If there is neither the parallel catalog nor the compendium,
# skip processing current target catalog.
if not pcat and not ccat:
continue
# Collect messages and texts to translate.
cat = Catalog(catpath)
pmsgs, psmsgs, ptexts = [], [], []
cmsgs, csmsgs, ctexts = [], [], []
for msg in cat:
if to_translate(msg, options):
# Priority: parallel catalog, then compendium.
for scat, msgs, smsgs, texts in (
(pcat, pmsgs, psmsgs, ptexts),
(ccat, cmsgs, csmsgs, ctexts),
):
if scat and msg in scat:
smsg = scat[msg]
if smsg.translated:
msgs.append(msg)
smsgs.append(smsg)
texts.extend(smsg.msgstr)
break
# Translate collected texts.
texts_tr = []
for texts, scat in ((ptexts, pcat), (ctexts, ccat)):
transerv = get_transerv(slang, tlang, scat, cat, tsbuilder)
texts_tr.append(transerv.translate(texts) if texts else [])
if texts_tr[-1] is None:
texts_tr = None
break
if texts_tr is None:
warning(
_("@info",
"Translation service failure on '%(file)s'.",
file=catpath))
continue
ptexts_tr, ctexts_tr = texts_tr
# Put translated texts into messages.
# For plural messages, assume 1-1 match to parallel language.
for msgs, smsgs, texts in (
(pmsgs, psmsgs, ptexts_tr),
(cmsgs, csmsgs, ctexts_tr),
):
for msg, smsg in zip(msgs, smsgs):
ctexts = []
for i in range(len(smsg.msgstr)):
text = texts.pop(0)
text = reduce_for_encoding(text, cat.encoding())
ctexts.append(text)
for i in range(len(msg.msgstr)):
msg.msgstr[i] = i < len(ctexts) and ctexts[i] or ctexts[-1]
decorate(msg, options)
sync_rep(cat, pmsgs + cmsgs)
def error_wcl (msg, norem=set()):
if not isinstance(norem, set):
norem = set(norem)
cleanup_tmppaths(norem)
error(msg)
def _main():
locale.setlocale(locale.LC_ALL, "")
usage = _("@info command usage",
"%(cmd)s [OPTIONS] VCS [POPATHS...]",
cmd="%prog")
desc = _(
"@info command description",
"Compose hybridized Ijekavian-Ekavian translation out of "
"translation modified from Ekavian to Ijekavian or vice-versa.")
ver = _("@info command version", u"%(cmd)s (Pology) %(version)s\n"
u"Copyright © 2009, 2010 "
u"Chusslove Illich (Часлав Илић) <%(email)s>",
cmd="%prog",
version=version(),
email="*****@*****.**")
opars = ColorOptionParser(usage=usage, description=desc, version=ver)
opars.add_option(
"-a",
"--accept-changes",
action="store_true",
dest="accept_changes",
default=False,
help=_(
"@info command line option description",
"Accept messages which have some changes between base "
"and reconstructed base text."))
opars.add_option("-r",
"--base-revision",
metavar=_("@info command line value placeholder",
"REVISION"),
action="store",
dest="base_revision",
default=None,
help=_(
"@info command line option description",
"Use the given revision as base for hybridization, "
"instead of local latest revision."))
add_cmdopt_filesfrom(opars)
(options, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
try:
import psyco
psyco.full()
except ImportError:
pass
# Create VCS.
if len(free_args) < 1:
showvcs = list(set(available_vcs()).difference(["none"]))
showvcs.sort()
error(
_("@info", "Version control system not given "
"(can be one of: %(vcslist)s).",
vcslist=format_item_list(showvcs)))
vcskey = free_args.pop(0)
if vcskey not in available_vcs(flat=True):
error(
_("@info", "Unknown version control system '%(vcs)s'.",
vcs=vcskey))
vcs = make_vcs(vcskey)
# Collect PO files in given paths.
popaths = collect_paths_cmdline(rawpaths=free_args,
filesfrom=options.files_from,
elsecwd=True,
respathf=collect_catalogs,
abort=True)
# Catalogs must be under version control.
for path in popaths:
if not vcs.is_versioned(path):
error(
_("@info",
"Catalog '%(file)s' is not under version control.",
file=path))
# Go by modified PO file and hybridize it.
for path in popaths:
# Extract local head counterpart.
tmpf = NamedTemporaryFile(prefix="pohybdl-export-", suffix=".po")
if not vcs.export(path, options.base_revision, tmpf.name):
error(
_("@info",
"Version control system cannot export file '%(file)s'.",
file=path))
# Hybridize by comparing local head and modified file.
hybdl(path, tmpf.name, options.accept_changes)
def main ():
locale.setlocale(locale.LC_ALL, "")
# Get defaults for command line options from global config.
cfgsec = pology_config.section("poselfmerge")
def_minwnex = cfgsec.integer("min-words-exact", 0)
def_minasfz = cfgsec.real("min-adjsim-fuzzy", 0.0)
def_fuzzex = cfgsec.boolean("fuzzy-exact", False)
def_refuzz = cfgsec.boolean("rebase-fuzzies", False)
# Setup options and parse the command line.
usage = _("@info command usage",
"%(cmd)s [options] POFILE...",
cmd="%prog")
desc = _("@info command description",
"Merge PO file with itself or compendium, "
"to produce fuzzy matches on similar messages.")
ver = _("@info command version",
u"%(cmd)s (Pology) %(version)s\n"
u"Copyright © 2009, 2010 "
u"Chusslove Illich (Часлав Илић) <%(email)s>",
cmd="%prog", version=version(), email="*****@*****.**")
opars = ColorOptionParser(usage=usage, description=desc, version=ver)
opars.add_option(
"-A", "--min-adjsim-fuzzy",
metavar=_("@info command line value placeholder", "RATIO"),
action="store", dest="min_adjsim_fuzzy", default=def_minasfz,
help=_("@info command line option description",
"On fuzzy matches, the minimum adjusted similarity "
"to accept the match, or else the message is left untranslated. "
"Range is 0.0-1.0, where 0 means always to accept the match, "
"and 1 never to accept; a practical range is 0.6-0.8."))
opars.add_option(
"-b", "--rebase-fuzzies",
action="store_true", dest="rebase_fuzzies", default=def_refuzz,
help=_("@info command line option description",
"Before merging, clear those fuzzy messages whose predecessor "
"(determined by previous fields) is still in the catalog."))
opars.add_option(
"-C", "--compendium",
metavar=_("@info command line value placeholder", "POFILE"),
action="append", dest="compendiums", default=[],
help=_("@info command line option description",
"Catalog with existing translations, to additionally use for "
"direct and fuzzy matches. Can be repeated."))
opars.add_option(
"-v", "--verbose",
action="store_true", dest="verbose", default=False,
help=_("@info command line option description",
"More detailed progress information."))
opars.add_option(
"-W", "--min-words-exact",
metavar=_("@info command line value placeholder", "NUMBER"),
action="store", dest="min_words_exact", default=def_minwnex,
help=_("@info command line option description",
"When using compendium, in case of exact match, "
"minimum number of words that original text must have "
"to accept translation without making it fuzzy. "
"Zero means to always accept an exact match."))
opars.add_option(
"-x", "--fuzzy-exact",
action="store_true", dest="fuzzy_exact", default=def_fuzzex,
help=_("@info command line option description",
"When using compendium, make all exact matches fuzzy."))
add_cmdopt_wrapping(opars)
add_cmdopt_filesfrom(opars)
(op, fargs) = opars.parse_args()
if len(fargs) < 1 and not op.files_from:
error(_("@info", "No input files given."))
# Could use some speedup.
try:
import psyco
psyco.full()
except ImportError:
pass
# Convert non-string options to needed types.
try:
op.min_words_exact = int(op.min_words_exact)
except:
error(_("@info",
"Value to option %(opt)s must be an integer number, "
"given '%(val)s' instead.",
opt="--min-words-exact", val=op.min_words_exact))
try:
op.min_adjsim_fuzzy = float(op.min_adjsim_fuzzy)
except:
error(_("@info",
"Value to option %(opt)s must be a real number, "
"given '%(val)s' instead.",
opt="--min-adjsim-fuzzy", val=op.min_ajdsim_fuzzy))
# Assemble list of files.
fnames = collect_paths_cmdline(rawpaths=fargs,
filesfrom=op.files_from,
respathf=collect_catalogs,
abort=True)
# Self-merge all catalogs.
for fname in fnames:
if op.verbose:
report(_("@info:progress", "Self-merging: %(file)s", file=fname))
self_merge_pofile(fname, op.compendiums,
op.fuzzy_exact, op.min_words_exact,
op.min_adjsim_fuzzy, op.rebase_fuzzies,
cfgsec, op)
@type abort: bool
@return: matcher function
@rtype: (msgf, msg, cat, hl=[])->bool
"""
mopts = _prep_attrobj(mopts, dict(case=False, ))
try:
expr, p = _build_expr_r(exprstr, 0, len(exprstr), mopts)
if p < len(exprstr):
raise ExprError(exprstr,
_("@item:intext", "premature end of expression"))
except ExprError, e:
if abort:
error(str_to_unicode(str(e)))
else:
raise
return expr
def make_msg_fmatcher(exprstr,
mopts=None,
accels=None,
filters=[],
abort=False):
"""
Build expression matcher for messages, with filtering.
Like L{make_msg_matcher}, except that matchers built by this function
do their own filtering, and so omit the first argument.
def main():
locale.setlocale(locale.LC_ALL, "")
# Get defaults for command line options from global config.
cfgsec = pology_config.section("posieve")
def_do_skip = cfgsec.boolean("skip-on-error", True)
def_msgfmt_check = cfgsec.boolean("msgfmt-check", False)
def_skip_obsolete = cfgsec.boolean("skip-obsolete", False)
# Setup options and parse the command line.
usage = _("@info command usage",
"%(cmd)s [OPTIONS] SIEVE [POPATHS...]",
cmd="%prog")
desc = _(
"@info command description",
"Apply sieves to PO paths, which may be either single PO files or "
"directories to search recursively for PO files. "
"Some of the sieves only examine PO files, while others "
"modify them as well. "
"The first non-option argument is the sieve name; "
"a list of several comma-separated sieves can be given too.")
ver = _("@info command version", u"%(cmd)s (Pology) %(version)s\n"
u"Copyright © 2007, 2008, 2009, 2010 "
u"Chusslove Illich (Часлав Илић) <%(email)s>",
cmd="%prog",
version=version(),
email="*****@*****.**")
opars = ColorOptionParser(usage=usage, description=desc, version=ver)
opars.add_option(
"-a",
"--announce-entry",
action="store_true",
dest="announce_entry",
default=False,
help=_("@info command line option description",
"Announce that header or message is just about to be sieved."))
opars.add_option("-b",
"--skip-obsolete",
action="store_true",
dest="skip_obsolete",
default=def_skip_obsolete,
help=_("@info command line option description",
"Do not sieve obsolete messages."))
opars.add_option(
"-c",
"--msgfmt-check",
action="store_true",
dest="msgfmt_check",
default=def_msgfmt_check,
help=_("@info command line option description",
"Check catalogs by %(cmd)s and skip those which do not pass.",
cmd="msgfmt -c"))
opars.add_option("-u",
"--single-entry",
metavar=_("@info command line value placeholder",
"ENTRY_NUMBER"),
action="store",
dest="single_entry",
default=0,
help=_("@info command line option description",
"Only perform the check on this ENTRY_NUMBER."))
opars.add_option(
"--force-sync",
action="store_true",
dest="force_sync",
default=False,
help=_("@info command line option description",
"Force rewriting of all messages, whether modified or not."))
opars.add_option("-H",
"--help-sieves",
action="store_true",
dest="help_sieves",
default=False,
help=_("@info command line option description",
"Show help for applied sieves."))
opars.add_option("--issued-params",
action="store_true",
dest="issued_params",
default=False,
help=_(
"@info command line option description",
"Show all issued sieve parameters "
"(from command line and user configuration)."))
opars.add_option("-l",
"--list-sieves",
action="store_true",
dest="list_sieves",
default=False,
help=_("@info command line option description",
"List available internal sieves."))
opars.add_option("--list-options",
action="store_true",
dest="list_options",
default=False,
help=_("@info command line option description",
"List the names of available options."))
opars.add_option("--list-sieve-names",
action="store_true",
dest="list_sieve_names",
default=False,
help=_("@info command line option description",
"List the names of available internal sieves."))
opars.add_option("--list-sieve-params",
action="store_true",
dest="list_sieve_params",
default=False,
help=_("@info command line option description",
"List the parameters known to issued sieves."))
opars.add_option("-m",
"--output-modified",
metavar=_("@info command line value placeholder", "FILE"),
action="store",
dest="output_modified",
default=None,
help=_("@info command line option description",
"Output names of modified files into FILE."))
opars.add_option("--no-skip",
action="store_false",
dest="do_skip",
default=def_do_skip,
help=_(
"@info command line option description",
"Do not try to skip catalogs which signal errors."))
opars.add_option("--no-sync",
action="store_false",
dest="do_sync",
default=True,
help=_("@info command line option description",
"Do not write any modifications to catalogs."))
opars.add_option("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help=_(
"@info command line option description",
"Do not display any progress info "
"(does not influence sieves themselves)."))
opars.add_option("-s",
metavar=_("@info command line value placeholder",
"NAME[:VALUE]"),
action="append",
dest="sieve_params",
default=[],
help=_("@info command line option description",
"Pass a parameter to sieves."))
opars.add_option(
"-S",
metavar=_("@info command line value placeholder", "NAME[:VALUE]"),
action="append",
dest="sieve_no_params",
default=[],
help=_(
"@info command line option description",
"Remove a parameter to sieves "
"(e.g. if it was issued through user configuration)."))
opars.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help=_("@info command line option description",
"Output more detailed progress information."))
add_cmdopt_filesfrom(opars)
add_cmdopt_incexc(opars)
add_cmdopt_colors(opars)
(op, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
if op.list_options:
report(list_options(opars))
sys.exit(0)
if len(free_args) < 1 and not (op.list_sieves or op.list_sieve_names):
error(_("@info", "No sieve to apply given."))
op.raw_sieves = []
op.raw_paths = []
if len(free_args) > 2 and op.single_entry != 0:
error(
_("@info",
"With single entry mode, you can only give one input file."))
if len(free_args) >= 1:
op.raw_sieves = free_args[0]
op.raw_paths = free_args[1:]
# Could use some speedup.
try:
import psyco
psyco.full()
except ImportError:
pass
set_coloring_globals(ctype=op.coloring_type, outdep=(not op.raw_colors))
# Dummy-set all internal sieves as requested if sieve listing required.
sieves_requested = []
if op.list_sieves or op.list_sieve_names:
# Global sieves.
modpaths = glob.glob(os.path.join(datadir(), "sieve", "[a-z]*.py"))
modpaths.sort()
for modpath in modpaths:
sname = os.path.basename(modpath)[:-3] # minus .py
sname = sname.replace("_", "-")
sieves_requested.append(sname)
# Language-specific sieves.
modpaths = glob.glob(
os.path.join(datadir(), "lang", "*", "sieve", "[a-z]*.py"))
modpaths.sort()
for modpath in modpaths:
sname = os.path.basename(modpath)[:-3] # minus .py
sname = sname.replace("_", "-")
lang = os.path.basename(os.path.dirname(os.path.dirname(modpath)))
sieves_requested.append(lang + ":" + sname)
# No need to load and setup sieves if only listing sieve names requested.
if op.list_sieve_names:
report("\n".join(sieves_requested))
sys.exit(0)
# Load sieve modules from supplied names in the command line.
if not sieves_requested:
sieves_requested = op.raw_sieves.split(",")
sieve_modules = []
for sieve_name in sieves_requested:
# Resolve sieve file.
if not sieve_name.endswith(".py"):
# One of internal sieves.
if ":" in sieve_name:
# Language-specific internal sieve.
lang, name = sieve_name.split(":")
sieve_path_base = os.path.join("lang", lang, "sieve", name)
else:
sieve_path_base = os.path.join("sieve", sieve_name)
sieve_path_base = sieve_path_base.replace("-", "_") + ".py"
sieve_path = os.path.join(datadir(), sieve_path_base)
else:
# Sieve name is its path.
sieve_path = sieve_name
try:
sieve_file = open(unicode_to_str(sieve_path))
# ...unicode_to_str because of exec below.
except IOError:
error(_("@info", "Cannot load sieve '%(file)s'.", file=sieve_path))
# Load file into new module.
sieve_mod_name = "sieve_" + str(len(sieve_modules))
sieve_mod = imp.new_module(sieve_mod_name)
exec sieve_file in sieve_mod.__dict__
sieve_file.close()
sys.modules[sieve_mod_name] = sieve_mod # to avoid garbage collection
sieve_modules.append((sieve_name, sieve_mod))
if not hasattr(sieve_mod, "Sieve"):
error(
_("@info",
"Module '%(file)s' does not define %(classname)s class.",
file=sieve_path,
classname="Sieve"))
# Setup sieves (description, known parameters...)
pp = ParamParser()
snames = []
for name, mod in sieve_modules:
scview = pp.add_subcmd(name)
if hasattr(mod, "setup_sieve"):
mod.setup_sieve(scview)
snames.append(name)
# If info on sieves requested, report and exit.
if op.list_sieves:
report(_("@info", "Available internal sieves:"))
report(pp.listcmd(snames))
sys.exit(0)
elif op.list_sieve_params:
params = set()
for scview in pp.cmdviews():
params.update(scview.params(addcol=True))
report("\n".join(sorted(params)))
sys.exit(0)
elif op.help_sieves:
report(_("@info", "Help for sieves:"))
report("")
report(pp.help(snames))
sys.exit(0)
# Prepare sieve parameters for parsing.
sieve_params = list(op.sieve_params)
# - append paramaters according to configuration
sieve_params.extend(read_config_params(pp.cmdviews(), sieve_params))
# - remove paramaters according to command line
if op.sieve_no_params:
sieve_params_mod = []
for parspec in sieve_params:
if parspec.split(":", 1)[0] not in op.sieve_no_params:
sieve_params_mod.append(parspec)
sieve_params = sieve_params_mod
# If assembly of issued parameters requested, report and exit.
if op.issued_params:
escparams = []
for parspec in sieve_params:
if ":" in parspec:
param, value = parspec.split(":", 1)
escparam = "%s:%s" % (param, escape_sh(value))
else:
escparam = parspec
escparams.append(escparam)
fmtparams = " ".join(["-s%s" % x for x in sorted(escparams)])
if fmtparams:
report(fmtparams)
sys.exit(0)
# Parse sieve parameters.
sparams, nacc_params = pp.parse(sieve_params, snames)
if nacc_params:
error(
_("@info", "Parameters not accepted by any of issued subcommands: "
"%(paramlist)s.",
paramlist=format_item_list(nacc_params)))
# ========================================
# FIXME: Think of something less ugly.
# Add as special parameter to each sieve:
# - root paths from which the catalogs are collected
# - whether destination independent coloring is in effect
# - test function for catalog selection
root_paths = []
if op.raw_paths:
root_paths.extend(op.raw_paths)
if op.files_from:
for ffpath in op.files_from:
root_paths.extend(collect_paths_from_file(ffpath))
if not op.raw_paths and not op.files_from:
root_paths = ["."]
is_cat_included = build_path_selector(incnames=op.include_names,
incpaths=op.include_paths,
excnames=op.exclude_names,
excpaths=op.exclude_paths)
for p in sparams.values():
p.root_paths = root_paths
p.raw_colors = op.raw_colors
p.is_cat_included = is_cat_included
# ========================================
# Create sieves.
sieves = []
for name, mod in sieve_modules:
sieves.append(mod.Sieve(sparams[name]))
# Get the message monitoring indicator from the sieves.
# Monitor unless all sieves have requested otherwise.
use_monitored = False
for sieve in sieves:
if getattr(sieve, "caller_monitored", True):
use_monitored = True
break
if op.verbose and not use_monitored:
report(_("@info:progress", "--> Not monitoring messages."))
# Get the sync indicator from the sieves.
# Sync unless all sieves have requested otherwise,
# and unless syncing is disabled globally in command line.
do_sync = False
for sieve in sieves:
if getattr(sieve, "caller_sync", True):
do_sync = True
break
if not op.do_sync:
do_sync = False
if op.verbose and not do_sync:
report(_("@info:progress", "--> Not syncing after sieving."))
# Open in header-only mode if no sieve has message processor.
# Categorize sieves by the presence of message/header processors.
use_headonly = True
header_sieves = []
header_sieves_last = []
message_sieves = []
for sieve in sieves:
if hasattr(sieve, "process"):
use_headonly = False
message_sieves.append(sieve)
if hasattr(sieve, "process_header"):
header_sieves.append(sieve)
if hasattr(sieve, "process_header_last"):
header_sieves_last.append(sieve)
if op.verbose and use_headonly:
report(_("@info:progress",
"--> Opening catalogs in header-only mode."))
# Collect catalog paths.
fnames = collect_paths_cmdline(rawpaths=op.raw_paths,
incnames=op.include_names,
incpaths=op.include_paths,
excnames=op.exclude_names,
excpaths=op.exclude_paths,
filesfrom=op.files_from,
elsecwd=True,
respathf=collect_catalogs,
abort=True)
if op.do_skip:
errwarn = warning
errwarn_on_msg = warning_on_msg
else:
errwarn = error
errwarn_on_msg = error_on_msg
# Prepare inline progress indicator.
if not op.quiet:
update_progress = init_file_progress(fnames,
addfmt=t_("@info:progress",
"Sieving: %(file)s"))
# Sieve catalogs.
modified_files = []
for fname in fnames:
if op.verbose:
report(_("@info:progress", "Sieving %(file)s...", file=fname))
elif not op.quiet:
update_progress(fname)
if op.msgfmt_check:
d1, oerr, ret = collect_system(
["msgfmt", "-o", "/dev/null", "-c", fname])
if ret != 0:
oerr = oerr.strip()
errwarn(
_("@info:progress", "%(file)s: %(cmd)s check failed:\n"
"%(msg)s",
file=fname,
cmd="msgfmt -c",
msg=oerr))
warning(
_("@info:progress",
"Skipping catalog due to syntax check failure."))
continue
try:
cat = Catalog(fname,
monitored=use_monitored,
headonly=use_headonly,
single_entry=int(op.single_entry))
except CatalogSyntaxError, e:
errwarn(
_("@info:progress",
"%(file)s: Parsing failed: %(msg)s",
file=fname,
msg=e))
warning(
_("@info:progress",
"Skipping catalog due to parsing failure."))
continue
skip = False
# First run all header sieves.
if header_sieves and op.announce_entry:
report(
_("@info:progress",
"Sieving header of %(file)s...",
file=fname))
for sieve in header_sieves:
try:
ret = sieve.process_header(cat.header, cat)
except SieveCatalogError, e:
errwarn(
_("@info:progress",
"%(file)s:header: Sieving failed: %(msg)s",
file=fname,
msg=e))
skip = True
break
if ret not in (None, 0):
break
def abort_or_raise(e):
error(str_to_unicode(str(e)))
def main():
locale.setlocale(locale.LC_ALL, "")
# Get defaults for command line options from global config.
cfgsec = pology_config.section("pomtrans")
showservs = list()
showservs.sort()
# Setup options and parse the command line.
usage = _("@info command usage",
"%(cmd)s [OPTIONS] TRANSERV PATHS...",
cmd="%prog")
desc = _("@info command description",
"Perform machine translation of PO files.")
ver = _("@info command version", u"%(cmd)s (Pology) %(version)s\n"
u"Copyright © 2009, 2010 "
u"Chusslove Illich (Часлав Илић) <%(email)s>",
cmd="%prog",
version=version(),
email="*****@*****.**")
opars = ColorOptionParser(usage=usage, description=desc, version=ver)
opars.add_option("-a",
"--accelerator",
dest="accel",
metavar=_("@info command line value placeholder", "CHAR"),
help=_(
"@info command line option description",
"Accelerator marker character used in messages. "
"Detected from catalogs if not given."))
opars.add_option("-c",
"--parallel-compendium",
dest="parcomp",
metavar=_("@info command line value placeholder", "FILE"),
help=_(
"@info command line option description",
"Translate from translation to another language, "
"found in compendium file at the given path."))
opars.add_option("-l",
"--list-transervs",
action="store_true",
dest="list_transervs",
default=False,
help="List available translation services.")
opars.add_option("-m",
"--flag-%s" % _flag_mtrans,
action="store_true",
dest="flag_mtrans",
default=False,
help=_("@info command line option description",
"Add '%(flag)s' flag to translated messages.",
flag=_flag_mtrans))
opars.add_option(
"-M",
"--translation-mode",
dest="tmode",
metavar=_("@info command line value placeholder", "MODE"),
help=_(
"@info command line option description",
"Translation mode for the chosen translation service. "
"Overrides the default translation mode constructed "
"based on source and target language. "
"Mode string format is translation service dependent."))
opars.add_option("-n",
"--no-fuzzy-flag",
action="store_false",
dest="flag_fuzzy",
default=True,
help=_(
"@info command line option description",
"Do not add '%(flag)s' flag to translated messages.",
flag="fuzzy"))
opars.add_option(
"-p",
"--parallel-catalogs",
dest="parcats",
metavar=_("@info command line value placeholder", "SEARCH:REPLACE"),
help=_(
"@info command line option description",
"Translate from translation to another language "
"found in parallel catalogs. "
"For given target catalog path, the path to parallel catalog "
"is constructed by replacing once SEARCH with REPLACE."))
opars.add_option("-s",
"--source-lang",
dest="slang",
metavar=_("@info command line value placeholder", "LANG"),
help=_(
"@info command line option description",
"Source language code. "
"Detected from catalogs if not given."))
opars.add_option("-t",
"--target-lang",
dest="tlang",
metavar=_("@info command line value placeholder", "LANG"),
help=_(
"@info command line option description",
"Target language code. "
"Detected from catalogs if not given."))
opars.add_option("-T",
"--transerv-bin",
dest="transerv_bin",
metavar=_("@info command line value placeholder", "PATH"),
help=_(
"@info command line option description",
"Custom path to translation service executable "
"(where applicable)."))
opars.add_option(
"-d",
"--data-directory",
dest="data_directory",
metavar=_("@info command line value placeholder", "FOLDER"),
help=_(
"@info command line option description",
"Custom path to a translation data directory (where applicable)."))
(op, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
# Could use some speedup.
try:
import psyco
psyco.full()
except ImportError:
pass
if op.list_transervs:
report("\n".join(sorted(_known_transervs.keys())))
sys.exit(0)
if len(free_args) < 1:
error(_("@info", "Translation service not specified."))
transervkey = free_args.pop(0)
if transervkey not in _known_transervs:
error(
_("@info",
"Translation service '%(serv)s' not known.",
serv=transervkey))
tsbuilder_wopts = _known_transervs[transervkey]
tsbuilder = lambda slang, tlang: tsbuilder_wopts(slang, tlang, op)
paths = free_args
if not op.parcomp and not op.parcats:
translate_direct(paths, tsbuilder, op)
else:
translate_parallel(paths, tsbuilder, op)
def organize(dictPath, reminv=False):
report(dictPath)
dictEncDefault = "UTF-8"
dictFile = open(dictPath, "r", dictEncDefault)
# Parse the header for language and encoding.
header = dictFile.readline()
m = re.search(r"^(\S+)\s+(\S+)\s+(\d+)\s+(\S+)\s*", header)
if not m:
error(
_("@info",
"Malformed header of the dictionary file '%(file)s'.",
file=dictPath))
dictType, dictLang, numWords, dictEnc = m.groups()
expDictType = "personal_ws-1.1"
if dictType != expDictType:
dictType = expDictType
report(" " + _("@item:inlist",
"dictionary type changed to '%(dtype)s'",
dtype=expDictType))
# Reopen in correct encoding if not the default.
if dictEnc.lower() != dictEncDefault.lower():
dictFile.close()
dictFile = open(dictPath, "r", dictEnc)
# Read all words and eliminate duplicates.
words = set()
validCharacters = re.compile(ur"^[\w\d\'・-]+$", re.UNICODE)
lno = 0
for word in dictFile:
lno += 1
word = word.strip()
if not word or word.startswith("personal_ws"):
continue
if word in words:
report(" " +
_("@item:inlist", "duplicate removed: %(word)s", word=word))
elif not validCharacters.match(word):
if not reminv:
report(" " + _("@item:inlist",
"*** invalid word at %(line)s: %(word)s",
line=lno,
word=word))
words.add(word)
else:
report(" " + _("@item:inlist",
"invalid word removed: %(word)s",
word=word))
else:
words.add(word)
dictFile.close()
words = list(words)
numWords = len(words)
# Sort the list according to current locale, ignoring case.
words.sort(lambda x, y: locale.strcoll(x.lower(), y.lower()))
# Write back the updated dictionary.
dictFile = open(dictPath, "w", dictEnc)
dictFile.write("%s %s %d %s\n" % (dictType, dictLang, numWords, dictEnc))
dictFile.write("\n".join(words))
dictFile.write("\n")
dictFile.close()
report(" " + n_("@item:inlist",
"written %(num)d word",
"written %(num)d words",
num=len(words)))
def main():
locale.setlocale(locale.LC_ALL, "")
# Get defaults for command line options from global config.
cfgsec = pology_config.section("poepatch")
def_do_merge = cfgsec.boolean("merge", True)
# Setup options and parse the command line.
usage = _("@info command usage", "%(cmd)s [OPTIONS] [OPTIONS] < EDIFF\n"
"%(cmd)s -u [OPTIONS] PATHS...",
cmd="%prog")
desc = _("@info command description",
"Apply embedded diff of PO files as patch.")
ver = _("@info command version", u"%(cmd)s (Pology) %(version)s\n"
u"Copyright © 2009, 2010 "
u"Chusslove Illich (Часлав Илић) <%(email)s>",
cmd="%prog",
version=version(),
email="*****@*****.**")
opars = ColorOptionParser(usage=usage, description=desc, version=ver)
opars.add_option(
"-a",
"--aggressive",
action="store_true",
dest="aggressive",
default=False,
help=_(
"@info command line option description",
"Apply every message to its paired message in the target file, "
"irrespective of whether its non-pairing parts match too."))
opars.add_option(
"-d",
"--directory",
metavar=_("@info command line value placeholder", "DIR"),
dest="directory",
help=_(
"@info command line option description",
"Prepend this directory path to any resolved target file path."))
opars.add_option(
"-e",
"--embed",
action="store_true",
dest="embed",
default=False,
help=_(
"@info command line option description",
"Instead of applying resolved newer version of the message, "
"add the full embedded diff into the target file."))
opars.add_option(
"-i",
"--input",
metavar=_("@info command line value placeholder", "FILE"),
dest="input",
help=_(
"@info command line option description",
"Read the patch from the given file instead of standard input."))
opars.add_option(
"-n",
"--no-merge",
action="store_false",
dest="do_merge",
default=def_do_merge,
help=_("@info command line option description",
"Do not try to indirectly pair messages by merging catalogs."))
opars.add_option(
"-p",
"--strip",
metavar=_("@info command line value placeholder", "NUM"),
dest="strip",
help=_(
"@info command line option description",
"Strip the smallest prefix containing NUM leading slashes from "
"each file name found in the ediff file (like in patch(1)). "
"If not given, only the base name of each file is taken."))
opars.add_option(
"-u",
"--unembed",
action="store_true",
dest="unembed",
default=False,
help=_(
"@info command line option description",
"Instead of applying a patch, resolve all embedded differences "
"in given paths to newer versions of messages."))
(op, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
# Could use some speedup.
try:
import psyco
psyco.full()
except ImportError:
pass
if not op.unembed:
if free_args:
error(
_("@info",
"Too many arguments in command line: %(argspec)s",
argspec=" ".join(free_args)))
if op.strip and not op.strip.isdigit():
error(
_("@info",
"Option %(opt)s expects a positive integer value.",
opt="--strip"))
apply_ediff(op)
else:
paths = []
for path in free_args:
if not os.path.exists(path):
warning(
_("@info", "Path '%(path)s' does not exist.", path=path))
if os.path.isdir(path):
paths.extend(collect_catalogs(path))
else:
paths.append(path)
for path in paths:
unembed_ediff(path)
def apply_ediff(op):
# Read the ediff PO.
dummy_stream_path = "<stdin>"
if op.input:
if not os.path.isfile(op.input):
error(
_("@info",
"Path '%(path)s' is not a file or does not exist.",
path=op.input))
edfpath = op.input
readfh = None
else:
edfpath = dummy_stream_path
readfh = sys.stdin
try:
ecat = Catalog(edfpath, monitored=False, readfh=readfh)
except:
error(
_("@info ediff is shorthand for \"embedded difference\"",
"Error reading ediff '%(file)s'.",
file=edfpath))
# Split ediff by diffed catalog into original and new file paths,
# header message, and ordinary messages.
hmsgctxt = ecat.header.get_field_value(EDST.hmsgctxt_field)
if hmsgctxt is None:
error(
_("@info",
"Header field '%(field)s' is missing in the ediff.",
field=EDST.hmsgctxt_field))
edsplits = []
cehmsg = None
smsgid = u"\x00"
ecat.add_last(MessageUnsafe(dict(msgctxt=hmsgctxt,
msgid=smsgid))) # sentry
for emsg in ecat:
if emsg.msgctxt == hmsgctxt:
if cehmsg:
# Record previous section.
edsplits.append((fpaths, cehmsg, cemsgs))
if emsg.msgid == smsgid: # end sentry, avoid parsing below
break
# Mine original and new file paths out of header.
fpaths = []
for fpath in emsg.msgid.split("\n")[:2]:
# Strip leading "+ "/"- "
fpath = fpath[2:]
# Convert to planform path separators.
fpath = re.sub(r"/+", os.path.sep, fpath)
# Remove revision indicator.
p = fpath.find(EDST.filerev_sep)
if p >= 0:
fpath = fpath[:p]
# Strip path and append directory as requested.
if op.strip:
preflen = int(op.strip)
lst = fpath.split(os.path.sep, preflen)
if preflen + 1 == len(lst):
fpath = lst[preflen]
else:
fpath = os.path.basename(fpath)
else:
fpath = os.path.basename(fpath)
if op.directory and fpath:
fpath = os.path.join(op.directory, fpath)
# All done.
fpaths.append(fpath)
cehmsg = emsg
cemsgs = []
else:
cemsgs.append(emsg)
# Prepare catalog for rejects and merges.
rcat = Catalog("", create=True, monitored=False, wrapping=ecat.wrapping())
init_ediff_header(rcat.header, hmsgctxt=hmsgctxt, extitle="rejects")
# Apply diff to catalogs.
for fpaths, ehmsg, emsgs in edsplits:
# Open catalog for patching.
fpath1, fpath2 = fpaths
if fpath1:
# Diff from an existing catalog, open it.
if not os.path.isfile(fpath1):
warning(
_("@info",
"Path '%(path)s' is not a file or does not exist, "
"skipping it.",
path=fpath1))
continue
try:
cat = Catalog(fpath1)
except:
warning(
_("@info",
"Error reading catalog '%(file)s', skipping it.",
file=fpath1))
continue
elif fpath2:
# New catalog added in diff, create it (or open if it exists).
try:
mkdirpath(os.path.dirname(fpath2))
cat = Catalog(fpath2, create=True)
if cat.created():
cat.set_wrapping(ecat.wrapping())
except:
if os.path.isfile(fpath2):
warning(
_("@info",
"Error reading catalog '%(file)s', skipping it.",
file=fpath1))
else:
warning(
_("@info",
"Cannot create catalog '%(file)s', skipping it.",
file=fpath2))
continue
else:
error(_("@info", "Both catalogs in ediff indicated not to exist."))
# Do not try to patch catalog with embedded differences
# (i.e. previously patched using -e).
if cat.header.get_field_value(EDST.hmsgctxt_field) is not None:
warning(
_("@info", "Catalog '%(file)s' already contains "
"embedded differences, skipping it.",
file=cat.filename))
continue
# Do not try to patch catalog if the patch contains
# unresolved split differences.
if reduce(lambda r, x: r or _flag_ediff_to_new in x.flag, emsgs,
False):
warning(
_("@info", "Patch for catalog '%(file)s' contains unresolved "
"split differences, skipping it.",
file=cat.filename))
continue
# Patch the catalog.
rejected_ehmsg = patch_header(cat, ehmsg, ecat, op)
rejected_emsgs_flags = patch_messages(cat, emsgs, ecat, op)
any_rejected = rejected_ehmsg or rejected_emsgs_flags
if fpath2 or any_rejected:
created = cat.created()
if cat.sync():
if not created:
if any_rejected and op.embed:
report(
_("@info:progress E is for \"with embedding\"",
"Partially patched (E): %(file)s",
file=cat.filename))
elif any_rejected:
report(
_("@info:progress",
"Partially patched: %(file)s",
file=cat.filename))
elif op.embed:
report(
_("@info:progress E is for \"with embedding\"",
"Patched (E): %(file)s",
file=cat.filename))
else:
report(
_("@info:progress",
"Patched: %(file)s",
file=cat.filename))
else:
if op.embed:
report(
_("@info:progress E is for \"with embedding\"",
"Created (E): %(file)s",
file=cat.filename))
else:
report(
_("@info:progress",
"Created: %(file)s",
file=cat.filename))
else:
pass #report("unchanged: %s" % cat.filename)
else:
os.unlink(fpath1)
report(_("@info:progress", "Removed: %(file)s", file=fpath1))
# If there were any rejects and reembedding is not in effect,
# record the necessary to present them.
if any_rejected and not op.embed:
if not rejected_ehmsg:
# Clean header diff.
ehmsg.manual_comment = ehmsg.manual_comment[:1]
ehmsg.msgstr[0] = u""
rcat.add_last(ehmsg)
for emsg, flag in rejected_emsgs_flags:
# Reembed to avoid any conflicts.
msg1, msg2, msg1_s, msg2_s = resolve_diff_pair(emsg)
emsg = msg_ediff(msg1_s,
msg2_s,
emsg=msg2_s,
ecat=rcat,
enoctxt=hmsgctxt)
if flag:
emsg.flag.add(flag)
rcat.add_last(emsg)
# If there were any rejects, write them out.
if len(rcat) > 0:
# Construct paths for embedded diffs of rejects.
rsuff = "rej"
if ecat.filename != dummy_stream_path:
rpath = ecat.filename
p = rpath.rfind(".")
if p < 0:
p = len(rpath)
rpath = rpath[:p] + (".%s" % rsuff) + rpath[p:]
else:
rpath = "stdin.%s.po" % rsuff
rcat.filename = rpath
rcat.sync(force=True, noobsend=True)
report(
_(
"@info:progress file to which rejected parts of the patch "
"have been written to",
"*** Rejects: %(file)s",
file=rcat.filename))