def _read_dict_file(fname):
# Parse the header for encoding.
encDefault = "UTF-8"
file = open(fname, "r", encDefault)
header = file.readline()
m = re.search(r"^(\S+)\s+(\S+)\s+(\d+)\s+(\S+)\s*", header)
if not m:
warning(
_("@info",
"Malformed header in dictionary file '%(file)s'.",
file=filepath))
return []
enc = m.group(4)
# Reopen in correct encoding if not the default.
if enc.lower() != encDefault.lower():
file.close()
file = open(fname, "r", enc)
# Read words.
words = []
for line in file:
word = line.strip()
if word:
words.append(word)
return words
def _read_wlist_aspell(fname):
# Parse the header for encoding.
defenc = "UTF-8"
fl = codecs.open(fname, "r", defenc)
header = fl.readline()
m = re.search(r"^(\S+)\s+(\S+)\s+(\d+)\s+(\S+)\s*", header)
if not m:
warning(
_("@info",
"Malformed header in dictionary file '%(file)s'.",
file=fname))
return []
enc = m.group(4)
# Reopen in correct encoding if not the default.
if enc.lower() != defenc.lower():
fl.close()
fl = codecs.open(fname, "r", enc)
# Read words.
words = []
for line in fl:
word = line.strip()
if word:
words.append(word)
return words
def process_header(self, hdr, cat):
# Establish counts for this file.
if cat.filename not in self.counts:
self.counts[cat.filename] = self._count_zero()
self.count = self.counts[cat.filename]
# If template correspondence requested, handle template matching.
if (self.p.templates and not cat.filename.endswith(".pot")):
# Construct expected template path.
tpath = cat.filename.replace(self.tspec_srch, self.tspec_repl, 1)
pdot = tpath.rfind(".")
if pdot >= 0:
tpath = tpath[:pdot] + ".pot"
# Inform if the template does not exist.
if not os.path.isfile(tpath):
warning(
_("@info",
"Expected template catalog '%(file)s' is missing.",
file=tpath))
# Indicate the template has been matched.
if tpath not in self.matched_templates:
self.matched_templates[tpath] = True
# Force explicitly given accelerators.
if self.p.accel is not None:
cat.set_accelerator(self.p.accel)
def translate_direct(paths, tsbuilder, options):
transervs = {}
catpaths = collect_catalogs(paths)
for catpath in catpaths:
# Collect messages and texts to translate.
cat = Catalog(catpath)
if options.accel is not None: # force explicitly given accelerator
cat.set_accelerator(options.accel)
texts = []
msgs = []
for msg in cat:
if to_translate(msg, options):
msgf = MessageUnsafe(msg)
remove_accel_msg(msgf, cat)
texts.append(msgf.msgid)
if msg.msgid_plural is not None:
texts.append(msgf.msgid_plural)
msgs.append(msg)
# Translate collected texts.
slang = options.slang or "en"
transerv = get_transerv(slang, options.tlang, cat, cat, tsbuilder)
texts_tr = transerv.translate(texts) if texts else []
if texts_tr is None:
warning(
_("@info",
"Translation service failure on '%(file)s'.",
file=catpath))
continue
for i, text in enumerate(texts_tr):
text = reduce_for_encoding(text, cat.encoding())
texts_tr[i] = text
# Put translated texts into messages.
singlepls = cat.plural_indices_single()
for msg in msgs:
msgid_tr = texts_tr.pop(0)
if msg.msgid_plural is not None:
msgid_plural_tr = texts_tr.pop(0)
if msgid_tr:
if msg.msgid_plural is not None:
for i in range(len(msg.msgstr)):
if i in singlepls:
msg.msgstr[i] = msgid_tr
else:
msg.msgstr[i] = msgid_plural_tr
else:
msg.msgstr[0] = msgid_tr
decorate(msg, options)
sync_rep(cat, msgs)
def revert (self, path):
# Base override.
res = collect_system(["svn", "revert", "-R", path], env=self._env)
if res[-1] != 0:
warning(_("@info",
"Subversion reports it cannot revert path '%(path)s':\n"
"%(msg)s",
path=path, msg=res[1]))
return False
return True
def __init__(self, params):
self.p = params
# Collect user and project configuration.
prjsect = "project-" + params.proj
if not config.has_section(prjsect):
raise SieveError(
_("@info",
"Project '%(id)s' is not defined in user configuration.",
id=params.proj))
self.prjcfg = config.section(prjsect)
prjcfg = config.section(prjsect)
usrcfg = config.section("user")
# Collect project data.
self.name = prjcfg.string("name") or usrcfg.string("name")
if not self.name:
warning(
_("@info", "Field '%(field)s' is not set in "
"project or user configuration.",
field="name"))
self.email = prjcfg.string("email") or usrcfg.string("email")
if not self.email:
warning(
_("@info", "Field '%(field)s' is not set in "
"project or user configuration.",
field="email"))
self.langteam = prjcfg.string("language-team")
if not self.langteam:
warning(
_("@info", "Field '%(field)s' is not set in "
"project configuration.",
field="language-team"))
self.teamemail = prjcfg.string("team-email") # ok not to be present
self.langcode = prjcfg.string("language") or usrcfg.string("language")
if not self.langcode:
warning(
_("@info", "Field '%(field)s' is not set in "
"project configuration.",
field="language"))
self.encoding = (prjcfg.string("encoding") or usrcfg.string("encoding")
or u"UTF-8")
self.plforms = (prjcfg.string("plural-forms")
or usrcfg.string("plural-forms"))
if not self.plforms:
warning(
_("@info", "Field '%(field)s' is not set in "
"project configuration.",
field="plural-forms"))
self.poeditor = (prjcfg.string("po-editor")
or usrcfg.string("po-editor")) # ok not to be present
def revert (self, path):
# Base override.
res = collect_system(["git", "checkout", path],
wdir=root, env=self._env)
if res[-1] != 0:
warning(_("@info"
"Git reports it cannot revert path '%(path)s':\n"
"%(msg)s",
path=path, msg=res[1]))
return []
return True
def move (self, spath, dpath):
# Base override.
root1, spath = self._gitroot(spath)
root2, dpath = self._gitroot(dpath)
if root1 != root2:
warning(_("@info",
"Trying to move paths between different repositories."))
return False
if collect_system(["git", "mv", spath, dpath], wdir=root1)[2] != 0:
return False
return True
def remove (self, path):
# Base override.
if os.path.isdir(path):
warning(_("@info",
"Git cannot remove directories (tried on '%(path)s').",
path=path))
return False
root, path = self._gitroot(path)
if collect_system(["git", "rm", path], wdir=root)[2] != 0:
return False
return True
def diff (self, path, rev1=None, rev2=None):
# Base override.
root, path = self._gitroot(path)
if rev1 is not None and rev2 is not None:
rspec = "%s..%s" % (rev1, rev2)
elif rev1 is not None:
rspec = "%s" % rev1
elif rev2 is not None:
raise PologyError(
_("@info"
"Git cannot diff from non-staged paths to a commit."))
else:
rspec = ""
res = collect_system(["git", "diff", rspec, path],
wdir=root, env=self._env)
if res[-1] != 0:
warning(_("@info"
"Git reports it cannot diff path '%(path)s':\n"
"%(msg)s",
path=path, msg=res[1]))
return []
udiff = []
nskip = 0
for line in res[0].split("\n"):
if nskip > 0:
nskip -= 1
continue
if line.startswith("diff"):
m = re.search(r"a/(.*?) *b/", line)
udiff.append((":", m.group(1) if m else ""))
nskip = 3
elif line.startswith("@@"):
m = re.search(r"-(\d+),(\d+) *\+(\d+),(\d+)", line)
spans = tuple(map(int, m.groups())) if m else (0, 0, 0, 0)
udiff.append(("@", spans))
elif line.startswith(" "):
udiff.append((" ", line[1:]))
elif line.startswith("-"):
udiff.append(("-", line[1:]))
elif line.startswith("+"):
udiff.append(("+", line[1:]))
return udiff
def lines_from_file(filepath, encoding=None):
"""
Read content of a text file into list of lines.
Only CR, LF, and CR+LF are treated as line breaks.
If the given file path is not readable, or text cannot be decoded using
given encoding, exceptions are raised. If encoding is not given,
the encoding specified by the environment is used.
@param filepath: path of the file to read
@type filepath: string
@param encoding: text encoding for the file
@param encoding: string
@returns: lines
@rtype: [string...]
"""
if encoding is None:
encoding = locale.getpreferredencoding()
try:
ifl = codecs.open(filepath, "r", encoding)
except:
warning(
_("@info", "Cannot open '%(file)s' for reading.", file=filepath))
raise
try:
content = ifl.read()
except:
warning(
_("@info",
"Cannot read content of '%(file)s' using %(enc)s encoding.",
file=filepath,
enc=encoding))
raise
ifl.close()
lines = [x + "\n" for x in re.split(r"\r\n|\r|\n", content)]
# ...no file.readlines(), it treats some other characters as line breaks.
if lines[-1] == "\n":
# If the file ended properly in a line break, the last line will be
# phony, from the empty element splitted out by the last line break.
lines.pop()
return lines
def diff (self, path, rev1=None, rev2=None):
# Base override.
if rev1 is not None and rev2 is not None:
rspec = "-r %s:%s" % (rev1, rev2)
elif rev1 is not None:
rspec = "-r %s" % rev1
elif rev2 is not None:
raise PologyError(
_("@info \"Subversion\" is a version control system",
"Subversion cannot diff from working copy "
"to a named revision."))
else:
rspec = ""
res = collect_system(["svn", "diff", path, rspec], env=self._env)
if res[-1] != 0:
warning(_("@info",
"Subversion reports it cannot diff path '%(path)s':\n"
"%(msg)s",
path=path, msg=res[1]))
return []
udiff = []
nskip = 0
for line in res[0].split("\n"):
if nskip > 0:
nskip -= 1
continue
if line.startswith("Index:"):
udiff.append((":", line[line.find(":") + 1:].strip()))
nskip = 3
elif line.startswith("@@"):
m = re.search(r"-(\d+),(\d+) *\+(\d+),(\d+)", line)
spans = tuple(map(int, m.groups())) if m else (0, 0, 0, 0)
udiff.append(("@", spans))
elif line.startswith(" "):
udiff.append((" ", line[1:]))
elif line.startswith("-"):
udiff.append(("-", line[1:]))
elif line.startswith("+"):
udiff.append(("+", line[1:]))
return udiff
def _get_module(name, cmsg=None):
if name not in _modules_on_request:
try:
_modules_on_request[name] = __import__(name)
except:
if cmsg:
warning(
_("@info", "Cannot import module '%(mod)s'; consequence:\n"
"%(msg)s",
mod=name,
msg=cmsg))
else:
warning(_("@info", "Cannot import module '%(mod)s'.",
mod=name))
_modules_on_request[name] = None
return _modules_on_request[name]
def _limit_to_chset (text, chset, translit, cname):
ltext = []
for c in text:
if c in chset:
ltext.append(c)
continue
ct = translit.get(c) # must come before translit_ascii
if ct is not None:
ltext.append(ct)
continue
ct = translit_ascii.get(c)
if ct is not None:
ltext.append(ct)
continue
warning(_("@info",
"Character '%(char)s' (%(code)s) cannot be transliterated "
"into character set %(charset)s, removing it.",
char=c, code=("U+%X" % ord(c)), charset=cname))
ltext.append("?")
return "".join(ltext)
def warning_on_msg(text, msg, cat, subsrc=None, file=sys.stderr):
"""
Warning on a PO message.
Outputs the message reference (catalog name and the message position),
along with the warning text.
@param text: text to report
@type text: string
@param msg: the message for which the text is reported
@type msg: L{Message_base}
@param cat: the catalog where the message lives
@type cat: L{Catalog}
@param subsrc: more detailed source of the message
@type subsrc: C{None} or string
@param file: send output to this file descriptor
@type file: C{file}
"""
posinfo = _msg_pos_fmt(cat.filename, msg.refline, msg.refentry)
text = cinterp("%s: %s", posinfo, text)
warning(text, subsrc=subsrc, showcmd=False)
def read_entities_by_env (entpathenv, recurse=True, fcap=False):
"""
Read XML entity definitions from directory paths given by
an environment variable.
Directory paths given by environment variable are searched for files with
C{.entities} extension, and all found files are sent to L{read_entities}.
Search through directories can be recursive or non-recursive.
See L{fcap_entities} for use of C{fcap} parameter.
If the environment variable is not set, a warning is output and empty
collection of entities returned.
@param entpathenv: environment variable that holds directory paths
@type entpathenv: string
@param recurse: whether to search directories recursively
@type recurse: bool
@param fcap: whether to add paired first-caps entities
@type fcap: bool
@returns: (name, value) dictionary of parsed entities
@rtype: dict
"""
entities = {}
entpath = os.getenv(entpathenv)
if entpath is None:
warning(_("@info",
"Environment variable with paths to entity definitions "
"'%(envar)s' is not set.",
envar=entpathenv))
return entities
entfilepaths = collect_files_by_ext(entpath.split(":"), "entities")
entities.update(read_entities(entfilepaths, fcap))
return entities
def _hito_w_simple(text, tick, refmap, ijklen_min, ijklen_max, toijek, silent,
errspans):
segs = []
p = 0
while True:
pp = p
p = text.find(tick, p)
if p < 0:
segs.append(text[pp:])
break
segs.append(text[pp:p])
pp = p
p += len(tick)
if p >= len(text) or not text[p:p + 1].isalpha():
segs.append(tick)
continue
ijklen = ijklen_min
ekvfrm = None
while ijklen <= ijklen_max and ekvfrm is None:
ijkfrm = text[p:p + ijklen]
ekvfrm = refmap.get(ijkfrm)
ijklen += 1
if ekvfrm is not None:
segs.append(ekvfrm if not toijek else ijkfrm)
p += len(ijkfrm)
else:
segs.append(tick)
errmsg = _("@info \"jat\" is the name of an old Serbian letter",
"Unknown jat-reflex starting from '%(snippet)s'.",
snippet=text[pp:pp + 20])
if not silent:
warning(errmsg)
if errspans is not None:
errspans.append((pp, pp + ijklen_max, errmsg))
return "".join(segs)
def translate(self, texts):
# Serialize texts to send to Apertium in one go.
# Separate texts with an inplace tag followed by dot,
# to have each text interpreted as standalone sentence.
# FIXME: Any way to really translate each text in turn,
# without it being horribly slow?
sep0 = "<br class='..."
sep1 = "..."
sep2 = "'/>."
sep = None
nsep = 0
while not sep: # determine shortest acceptable separator
sep = sep0 + sep1 * nsep + sep2
for text in texts:
if sep in text:
sep = None
nsep += 1
break
stext = sep.join(texts)
# Translate empty string to test language pair.
# Otherwise, if a lot of text is sent and language pair not good,
# Apertium may just signal broken pipe.
res = collect_system(self.cmdline, instr="")
if res[2] != 0:
warning(
_("@info",
"Executing Apertium failed:\n%(output)s",
output=res[0]))
# ...really res[0], error is output to stdout. Tsk.
return None
res = collect_system(self.cmdline, instr=stext)
if res[2] != 0:
warning(
_("@info",
"Executing Apertium failed:\n%(output)s",
output=res[0]))
# ...really res[0], error is output to stdout. Tsk.
return None
texts_tr = res[0].split(sep)
if len(texts_tr) != len(texts):
warning(
_("@info", "Apertium reported wrong number of translations, "
"%(num1)d instead of %(num2)d.",
num1=len(texts_tr),
num2=len(texts)))
return None
texts_tr = [
resolve_entities_simple(x, self.htmlents) for x in texts_tr
]
return texts_tr
def commit (self, paths, message=None, msgfile=None, incparents=True):
# Base override.
if isinstance(paths, basestring):
paths = [paths]
if not paths:
return True
opaths = paths
root, paths = self._gitroot(paths)
# Check if all paths are versioned.
# Add to index any modified paths that have not been added.
for opath in opaths:
if not self.is_versioned(opath):
warning(_("@info"
"Git cannot commit non-versioned path '%(path)s'.",
path=opath))
return False
if os.path.exists(opath) and not self.add(opath):
warning(_("@info"
"Git cannot add path '%(path)s' to index.",
path=opath))
return False
# Reset all paths in index which have not been given to commit.
ipaths = self._paths_to_commit(root)
rpaths = list(set(ipaths).difference(paths))
if rpaths:
warning(_("@info",
"Git is resetting paths in index which are "
"not to be committed."))
cmdline = "git reset %s" % " ".join(rpaths)
system_wd(unicode_to_str(cmdline), root)
# ...seems to return != 0 even if it did what it was told to.
# Commit the index.
cmdline = ["git", "commit"]
if message is not None:
cmdline += ["-m", message]
elif msgfile is not None:
cmdline += ["-F", msgfile]
# Do not use collect_system(), user may need to input stuff.
cmdstr = " ".join(map(escape_sh, cmdline))
if system_wd(unicode_to_str(cmdstr), root) != 0:
return False
return True
def report_on_msg_hl(highlight,
msg,
cat,
fmsg=None,
subsrc=None,
file=sys.stdout):
"""
Report on parts of a PO message.
For each of the spans found in the L{highlight<report_msg_content>}
specification which have a note attached, outputs the position reference
(catalog name, message position, spanned segment) and the span note.
The highlight can be relative to a somewhat modified, filtered message
instead of the original one.
@param highlight: highlight specification
@type highlight: L{highlight<report_msg_content>}
@param msg: the message for which the text is reported
@type msg: L{Message_base}
@param cat: the catalog where the message lives
@type cat: L{Catalog}
@param fmsg: filtered message to which the highlight corresponds
@type fmsg: L{Message_base}
@param subsrc: more detailed source of the message
@type subsrc: C{None} or string
@param file: send output to this file descriptor
@type file: C{file}
"""
refpos = _msg_pos_fmt(cat.filename, msg.refline, msg.refentry)
if not fmsg: # use original message as filtered if not given
fmsg = msg
for hspec in highlight:
name, item, spans = hspec[:3]
if name == "msgctxt":
text = msg.msgctxt or u""
ftext = fmsg.msgctxt or u""
elif name == "msgid":
text = msg.msgid
ftext = fmsg.msgid
elif name == "msgid_plural":
text = msg.msgid_plural or u""
ftext = fmsg.msgid_plural or u""
elif name == "msgstr":
text = msg.msgstr[item]
ftext = fmsg.msgstr[item]
# TODO: Add more fields.
else:
warning(
_("@info", "Unknown field '%(field)s' "
"in highlighting specification.",
field=name))
continue
if len(hspec) > 3:
# Override filtered text from filtered message
# by filtered text from the highlight spec.
ftext = hspec[3]
spans = adapt_spans(text, ftext, spans, merge=False)
if msg.msgid_plural is not None and name == "msgstr":
name = "%s_%d" % (name, item)
for span in spans:
if len(span) < 3:
continue
start, end, snote = span
if isinstance(start, int) and isinstance(end, int):
seglen = end - start
if seglen > 0:
segtext = text[start:end]
if len(segtext) > 30:
segtext = segtext[:27] + "..."
posinfo = "%s:%d:\"%s\"" % (name, start, escape(segtext))
else:
posinfo = "%s:%d" % (name, start)
else:
posinfo = "%s" % name
posinfo = ColorString("<green>%s</green>") % posinfo
rtext = cinterp("%s[%s]: %s", refpos, posinfo, snote)
report(rtext, subsrc=subsrc, showcmd=False)
def report_msg_content(msg,
cat,
wrapf=None,
force=False,
note=None,
delim=None,
highlight=None,
showmsg=True,
fmsg=None,
showfmsg=False,
subsrc=None,
file=sys.stdout):
"""
Report the content of a PO message.
Provides the message reference, consisting of the catalog name and
the message position within it, the message contents,
and any notes on particular segments.
Parts of the message can be highlighted using colors.
Parameter C{highlight} provides the highlighting specification, as
list of tuples where each tuple consists of: name of the message element
to highlight, element index (used when the element is a list of values),
list of spans, and optionally the filtered text of the element value.
For example, to highlight spans C{(5, 10)} and C{(15, 25)} in the C{msgid},
and C{(30, 40)} in C{msgstr}, the highlighting specification would be::
[("msgid", 0, [(5, 10), (15, 25)]), ("msgstr", 0, [(30, 40)])]
Names of the elements that can presently be highlighted are: C{"msgctxt"},
C{"msgid"}, C{"msgid_plural"}, C{"msgstr"}, C{"manual_comment"},
C{"auto_comment"}, C{"source"}, C{"flag"}.
For unique fields the element index is not used, but 0 should be given
for consistency (may be enforced later).
Span tuples can have a third element, following the indices, which is
the note about why the particular span is highlighted;
there may be more elements after the note, and these are all ignored.
If start or end index of a span is not an integer,
then the note is taken as relating to the complete field.
Sometimes the match to which the spans correspond has been made on a
filtered value of the message field (e.g. after accelerator markers
or tags have been removed). In that case, the filtered text can be
given as the fourth element of the tuple, after the list of spans, and
the function will try to fit spans from filtered onto original text.
More globally, if the complete highlight is relative to a modified,
filtered version of the message, this message can be given as
C{fmsg} parameter.
The display of content can be controlled by C{showmsg} parameter;
if it is C{False}, only the message reference and span notes are shown.
Similarly for the C{showfmsg} parameter, which controls the display
of the content of filtered message (if given by C{fmsg}).
To show the filtered message may be useful for debugging filtering
in cases when it is not straightforward, or it is user-defined.
@param msg: the message to report the content for
@type msg: L{Message_base}
@param cat: the catalog where the message lives
@type cat: L{Catalog} or C{None}
@param wrapf:
the function used for wrapping message fields in output.
See L{to_lines()<message.Message_base.to_lines>} method
of message classes for details.
If not given, it will be taken from the catalog
(see L{Catalog.wrapf<catalog.Catalog.wrapf>}).
@type wrapf: (string)->[string...]
@param force: whether to force reformatting of cached message content
@type force: bool
@param note: note about why the content is being reported
@type note: string
@param delim: text to print on the line following the message
@type delim: C{None} or string
@param highlight: highlighting specification of message elements
@type highlight: (see description)
@param showmsg: show content of the message
@type showmsg: bool
@param fmsg: filtered message
@type fmsg: L{Message_base}
@param showfmsg: show content of the filtered message, if any
@type showfmsg: bool
@param subsrc: more detailed source of the message
@type subsrc: C{None} or string
@param file: output stream
@type file: file
"""
rsegs = []
wrapf = wrapf or cat.wrapf()
notes_data = []
if highlight:
msg = Message(msg) # must work on copy, highlight modifies it
ffmsg = fmsg or msg # use original message as filtered if not given
# Unify spans for same parts, to have single coloring pass per part
# (otherwise markup can get corrupted).
highlightd = {}
for hspec in highlight:
name, item, spans = hspec[:3]
pkey = (name, item)
phspec = highlightd.get(pkey)
if phspec is None:
# Make needed copies in order not to modify
# the original highlight when adding stuff later.
highlightd[pkey] = list(hspec)
highlightd[pkey][2] = list(spans)
else:
phspec[2].extend(spans)
# Take filtered text if available and not already taken.
if len(hspec) > 3 and len(phspec) <= 3:
phspec.append(hspec[3])
highlight = highlightd.values()
for hspec in highlight:
name, item, spans = hspec[:3]
def hl(text, ftext):
if len(hspec) > 3:
# Override filtered text from filtered message
# by filtered text from the highlight spec.
ftext = hspec[3]
aspans = adapt_spans(text, ftext, spans, merge=False)
notes_data.append((text, name, item, aspans))
text = _highlight_spans(text, spans, "red", ftext=ftext)
return text
if name == "msgctxt":
if msg.msgctxt or ffmsg.msgctxt:
msg.msgctxt = hl(msg.msgctxt or u"", ffmsg.msgctxt or u"")
elif name == "msgid":
msg.msgid = hl(msg.msgid, ffmsg.msgid)
elif name == "msgid_plural":
msg.msgid_plural = hl(msg.msgid_plural or u"",
ffmsg.msgid_plural or u"")
elif name == "msgstr":
msg.msgstr[item] = hl(msg.msgstr[item], ffmsg.msgstr[item])
elif name == "manual_comment":
msg.manual_comment[item] = hl(msg.manual_comment[item],
ffmsg.manual_comment[item])
elif name == "auto_comment":
msg.auto_comment[item] = hl(msg.auto_comment[item],
ffmsg.auto_comment[item])
elif name == "source":
msg.source[item] = Monpair(
(hl(msg.source[item][0],
ffmsg.source[item][0]), msg.source[item][1]))
elif name == "flag":
pass # FIXME: How to do this?
else:
warning(
_("@info", "Unknown field '%(field)s' "
"in highlighting specification.",
field=name))
# Report the message.
msegs = []
if cat is not None:
msegs += [_msg_pos_fmt(cat.filename, msg.refline, msg.refentry) + "\n"]
if showmsg:
msgstr = msg.to_string(wrapf=wrapf, force=force, colorize=1)
msegs += [msgstr.rstrip() + "\n"]
if msegs:
rsegs.append(cjoin(msegs).rstrip())
# Report notes.
if note is not None: # global
notestr = _("@info", "<bold>[note]</bold> %(msg)s", msg=note)
rsegs.append(notestr)
if notes_data: # span notes
note_ord = 1
for text, name, item, spans in notes_data:
if msg.msgid_plural is not None and name == "msgstr":
name = "%s_%d" % (name, item)
for span in spans:
if len(span) < 3:
continue
start, end, snote = span
if isinstance(start, int) and isinstance(end, int):
seglen = end - start
if seglen > 0:
segtext = text[start:end]
if len(segtext) > 30:
segtext = _("@item:intext shortened longer text",
"%(snippet)s...",
snippet=segtext[:27])
posinfo = "%s:%d:\"%s\"" % (name, start,
escape(segtext))
else:
posinfo = "%s:%d" % (name, start)
else:
posinfo = "%s" % name
posinfo = ColorString("<green>%s</green>") % posinfo
rsegs.append(
_("@info", "[%(pos)s]: %(msg)s", pos=posinfo, msg=snote))
note_ord += 1
# Report the filtered message, if given and requested.
if fmsg and showfmsg:
fmtnote = (ColorString("<green>%s</green>") %
_("@info", ">>> Filtered message was:"))
rsegs.append(fmtnote)
fmsgstr = fmsg.to_string(wrapf=wrapf, force=force, colorize=1)
mstr = fmsgstr.rstrip() + "\n"
rsegs.append(mstr.rstrip())
if delim:
rsegs.append(delim)
rtext = cjoin(rsegs, "\n").rstrip()
report(rtext, subsrc=subsrc, file=file)
def hybdl(path, path0, accnohyb=False):
cat = Catalog(path)
cat0 = Catalog(path0, monitored=False)
nhybridized = 0
nstopped = 0
for msg in cat:
if "no-hybdl" in manc_parse_flag_list(msg, "|"):
continue
# Unembed diff if message was diffed for review.
# Replace ediff with manual review flag.
diffed = False
for flag in msg.flag:
if flag.startswith("ediff"):
msg.flag.remove(flag)
diffed = True
if diffed:
msg_ediff_to_new(msg, msg)
msg.flag.add(u"reviewed")
# Fetch original message.
msg0 = cat0.get(msg)
if msg0 is None:
warning_on_msg(
_("@info", "Message does not exist in the original catalog."),
msg, cat)
nstopped += 1
continue
if len(msg.msgstr) != len(msg0.msgstr):
warning_on_msg(
_(
"@info", "Number of translations not same as in "
"the original message."), msg, cat)
nstopped += 1
continue
if msg.msgstr == msg0.msgstr:
# No changes, nothing new to hybridize.
continue
# Hybridize translation.
textsh = []
textshinv = []
for text0, text in zip(msg0.msgstr, msg.msgstr):
texth = tohi(text0, text, parthyb=True)
textsh.append(texth)
if not accnohyb:
texthinv = tohi(text, text0, parthyb=True)
textshinv.append(texthinv)
if accnohyb or textsh == textshinv:
for i, texth in zip(range(len(msg.msgstr)), textsh):
msg.msgstr[i] = texth
nhybridized += 1
else:
nstopped += 1
msgh = MessageUnsafe(msg)
msgh.msgstr = textsh
msghinv = MessageUnsafe(msg)
msghinv.msgstr = textshinv
msg_ediff(msghinv, msgh, emsg=msgh, colorize=True)
report_msg_content(msgh, cat, delim=("-" * 20))
if nstopped == 0:
if cat.sync():
report("! %s (%d)" % (path, nhybridized))
else:
warning(
n_("@info", "%(num)d message in '%(file)s' cannot be "
"cleanly hybridized.",
"%(num)d messages in '%(file)s' cannot be "
"cleanly hybridized.",
num=nstopped,
file=path))
nhybridized = 0
return nhybridized
def main():
locale.setlocale(locale.LC_ALL, "")
# Get defaults for command line options from global config.
cfgsec = pology_config.section("posieve")
def_do_skip = cfgsec.boolean("skip-on-error", True)
def_msgfmt_check = cfgsec.boolean("msgfmt-check", False)
def_skip_obsolete = cfgsec.boolean("skip-obsolete", False)
# Setup options and parse the command line.
usage = _("@info command usage",
"%(cmd)s [OPTIONS] SIEVE [POPATHS...]",
cmd="%prog")
desc = _(
"@info command description",
"Apply sieves to PO paths, which may be either single PO files or "
"directories to search recursively for PO files. "
"Some of the sieves only examine PO files, while others "
"modify them as well. "
"The first non-option argument is the sieve name; "
"a list of several comma-separated sieves can be given too.")
ver = _("@info command version", u"%(cmd)s (Pology) %(version)s\n"
u"Copyright © 2007, 2008, 2009, 2010 "
u"Chusslove Illich (Часлав Илић) <%(email)s>",
cmd="%prog",
version=version(),
email="*****@*****.**")
opars = ColorOptionParser(usage=usage, description=desc, version=ver)
opars.add_option(
"-a",
"--announce-entry",
action="store_true",
dest="announce_entry",
default=False,
help=_("@info command line option description",
"Announce that header or message is just about to be sieved."))
opars.add_option("-b",
"--skip-obsolete",
action="store_true",
dest="skip_obsolete",
default=def_skip_obsolete,
help=_("@info command line option description",
"Do not sieve obsolete messages."))
opars.add_option(
"-c",
"--msgfmt-check",
action="store_true",
dest="msgfmt_check",
default=def_msgfmt_check,
help=_("@info command line option description",
"Check catalogs by %(cmd)s and skip those which do not pass.",
cmd="msgfmt -c"))
opars.add_option("-u",
"--single-entry",
metavar=_("@info command line value placeholder",
"ENTRY_NUMBER"),
action="store",
dest="single_entry",
default=0,
help=_("@info command line option description",
"Only perform the check on this ENTRY_NUMBER."))
opars.add_option(
"--force-sync",
action="store_true",
dest="force_sync",
default=False,
help=_("@info command line option description",
"Force rewriting of all messages, whether modified or not."))
opars.add_option("-H",
"--help-sieves",
action="store_true",
dest="help_sieves",
default=False,
help=_("@info command line option description",
"Show help for applied sieves."))
opars.add_option("--issued-params",
action="store_true",
dest="issued_params",
default=False,
help=_(
"@info command line option description",
"Show all issued sieve parameters "
"(from command line and user configuration)."))
opars.add_option("-l",
"--list-sieves",
action="store_true",
dest="list_sieves",
default=False,
help=_("@info command line option description",
"List available internal sieves."))
opars.add_option("--list-options",
action="store_true",
dest="list_options",
default=False,
help=_("@info command line option description",
"List the names of available options."))
opars.add_option("--list-sieve-names",
action="store_true",
dest="list_sieve_names",
default=False,
help=_("@info command line option description",
"List the names of available internal sieves."))
opars.add_option("--list-sieve-params",
action="store_true",
dest="list_sieve_params",
default=False,
help=_("@info command line option description",
"List the parameters known to issued sieves."))
opars.add_option("-m",
"--output-modified",
metavar=_("@info command line value placeholder", "FILE"),
action="store",
dest="output_modified",
default=None,
help=_("@info command line option description",
"Output names of modified files into FILE."))
opars.add_option("--no-skip",
action="store_false",
dest="do_skip",
default=def_do_skip,
help=_(
"@info command line option description",
"Do not try to skip catalogs which signal errors."))
opars.add_option("--no-sync",
action="store_false",
dest="do_sync",
default=True,
help=_("@info command line option description",
"Do not write any modifications to catalogs."))
opars.add_option("-q",
"--quiet",
action="store_true",
dest="quiet",
default=False,
help=_(
"@info command line option description",
"Do not display any progress info "
"(does not influence sieves themselves)."))
opars.add_option("-s",
metavar=_("@info command line value placeholder",
"NAME[:VALUE]"),
action="append",
dest="sieve_params",
default=[],
help=_("@info command line option description",
"Pass a parameter to sieves."))
opars.add_option(
"-S",
metavar=_("@info command line value placeholder", "NAME[:VALUE]"),
action="append",
dest="sieve_no_params",
default=[],
help=_(
"@info command line option description",
"Remove a parameter to sieves "
"(e.g. if it was issued through user configuration)."))
opars.add_option("-v",
"--verbose",
action="store_true",
dest="verbose",
default=False,
help=_("@info command line option description",
"Output more detailed progress information."))
add_cmdopt_filesfrom(opars)
add_cmdopt_incexc(opars)
add_cmdopt_colors(opars)
(op, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
if op.list_options:
report(list_options(opars))
sys.exit(0)
if len(free_args) < 1 and not (op.list_sieves or op.list_sieve_names):
error(_("@info", "No sieve to apply given."))
op.raw_sieves = []
op.raw_paths = []
if len(free_args) > 2 and op.single_entry != 0:
error(
_("@info",
"With single entry mode, you can only give one input file."))
if len(free_args) >= 1:
op.raw_sieves = free_args[0]
op.raw_paths = free_args[1:]
# Could use some speedup.
try:
import psyco
psyco.full()
except ImportError:
pass
set_coloring_globals(ctype=op.coloring_type, outdep=(not op.raw_colors))
# Dummy-set all internal sieves as requested if sieve listing required.
sieves_requested = []
if op.list_sieves or op.list_sieve_names:
# Global sieves.
modpaths = glob.glob(os.path.join(datadir(), "sieve", "[a-z]*.py"))
modpaths.sort()
for modpath in modpaths:
sname = os.path.basename(modpath)[:-3] # minus .py
sname = sname.replace("_", "-")
sieves_requested.append(sname)
# Language-specific sieves.
modpaths = glob.glob(
os.path.join(datadir(), "lang", "*", "sieve", "[a-z]*.py"))
modpaths.sort()
for modpath in modpaths:
sname = os.path.basename(modpath)[:-3] # minus .py
sname = sname.replace("_", "-")
lang = os.path.basename(os.path.dirname(os.path.dirname(modpath)))
sieves_requested.append(lang + ":" + sname)
# No need to load and setup sieves if only listing sieve names requested.
if op.list_sieve_names:
report("\n".join(sieves_requested))
sys.exit(0)
# Load sieve modules from supplied names in the command line.
if not sieves_requested:
sieves_requested = op.raw_sieves.split(",")
sieve_modules = []
for sieve_name in sieves_requested:
# Resolve sieve file.
if not sieve_name.endswith(".py"):
# One of internal sieves.
if ":" in sieve_name:
# Language-specific internal sieve.
lang, name = sieve_name.split(":")
sieve_path_base = os.path.join("lang", lang, "sieve", name)
else:
sieve_path_base = os.path.join("sieve", sieve_name)
sieve_path_base = sieve_path_base.replace("-", "_") + ".py"
sieve_path = os.path.join(datadir(), sieve_path_base)
else:
# Sieve name is its path.
sieve_path = sieve_name
try:
sieve_file = open(unicode_to_str(sieve_path))
# ...unicode_to_str because of exec below.
except IOError:
error(_("@info", "Cannot load sieve '%(file)s'.", file=sieve_path))
# Load file into new module.
sieve_mod_name = "sieve_" + str(len(sieve_modules))
sieve_mod = imp.new_module(sieve_mod_name)
exec sieve_file in sieve_mod.__dict__
sieve_file.close()
sys.modules[sieve_mod_name] = sieve_mod # to avoid garbage collection
sieve_modules.append((sieve_name, sieve_mod))
if not hasattr(sieve_mod, "Sieve"):
error(
_("@info",
"Module '%(file)s' does not define %(classname)s class.",
file=sieve_path,
classname="Sieve"))
# Setup sieves (description, known parameters...)
pp = ParamParser()
snames = []
for name, mod in sieve_modules:
scview = pp.add_subcmd(name)
if hasattr(mod, "setup_sieve"):
mod.setup_sieve(scview)
snames.append(name)
# If info on sieves requested, report and exit.
if op.list_sieves:
report(_("@info", "Available internal sieves:"))
report(pp.listcmd(snames))
sys.exit(0)
elif op.list_sieve_params:
params = set()
for scview in pp.cmdviews():
params.update(scview.params(addcol=True))
report("\n".join(sorted(params)))
sys.exit(0)
elif op.help_sieves:
report(_("@info", "Help for sieves:"))
report("")
report(pp.help(snames))
sys.exit(0)
# Prepare sieve parameters for parsing.
sieve_params = list(op.sieve_params)
# - append paramaters according to configuration
sieve_params.extend(read_config_params(pp.cmdviews(), sieve_params))
# - remove paramaters according to command line
if op.sieve_no_params:
sieve_params_mod = []
for parspec in sieve_params:
if parspec.split(":", 1)[0] not in op.sieve_no_params:
sieve_params_mod.append(parspec)
sieve_params = sieve_params_mod
# If assembly of issued parameters requested, report and exit.
if op.issued_params:
escparams = []
for parspec in sieve_params:
if ":" in parspec:
param, value = parspec.split(":", 1)
escparam = "%s:%s" % (param, escape_sh(value))
else:
escparam = parspec
escparams.append(escparam)
fmtparams = " ".join(["-s%s" % x for x in sorted(escparams)])
if fmtparams:
report(fmtparams)
sys.exit(0)
# Parse sieve parameters.
sparams, nacc_params = pp.parse(sieve_params, snames)
if nacc_params:
error(
_("@info", "Parameters not accepted by any of issued subcommands: "
"%(paramlist)s.",
paramlist=format_item_list(nacc_params)))
# ========================================
# FIXME: Think of something less ugly.
# Add as special parameter to each sieve:
# - root paths from which the catalogs are collected
# - whether destination independent coloring is in effect
# - test function for catalog selection
root_paths = []
if op.raw_paths:
root_paths.extend(op.raw_paths)
if op.files_from:
for ffpath in op.files_from:
root_paths.extend(collect_paths_from_file(ffpath))
if not op.raw_paths and not op.files_from:
root_paths = ["."]
is_cat_included = build_path_selector(incnames=op.include_names,
incpaths=op.include_paths,
excnames=op.exclude_names,
excpaths=op.exclude_paths)
for p in sparams.values():
p.root_paths = root_paths
p.raw_colors = op.raw_colors
p.is_cat_included = is_cat_included
# ========================================
# Create sieves.
sieves = []
for name, mod in sieve_modules:
sieves.append(mod.Sieve(sparams[name]))
# Get the message monitoring indicator from the sieves.
# Monitor unless all sieves have requested otherwise.
use_monitored = False
for sieve in sieves:
if getattr(sieve, "caller_monitored", True):
use_monitored = True
break
if op.verbose and not use_monitored:
report(_("@info:progress", "--> Not monitoring messages."))
# Get the sync indicator from the sieves.
# Sync unless all sieves have requested otherwise,
# and unless syncing is disabled globally in command line.
do_sync = False
for sieve in sieves:
if getattr(sieve, "caller_sync", True):
do_sync = True
break
if not op.do_sync:
do_sync = False
if op.verbose and not do_sync:
report(_("@info:progress", "--> Not syncing after sieving."))
# Open in header-only mode if no sieve has message processor.
# Categorize sieves by the presence of message/header processors.
use_headonly = True
header_sieves = []
header_sieves_last = []
message_sieves = []
for sieve in sieves:
if hasattr(sieve, "process"):
use_headonly = False
message_sieves.append(sieve)
if hasattr(sieve, "process_header"):
header_sieves.append(sieve)
if hasattr(sieve, "process_header_last"):
header_sieves_last.append(sieve)
if op.verbose and use_headonly:
report(_("@info:progress",
"--> Opening catalogs in header-only mode."))
# Collect catalog paths.
fnames = collect_paths_cmdline(rawpaths=op.raw_paths,
incnames=op.include_names,
incpaths=op.include_paths,
excnames=op.exclude_names,
excpaths=op.exclude_paths,
filesfrom=op.files_from,
elsecwd=True,
respathf=collect_catalogs,
abort=True)
if op.do_skip:
errwarn = warning
errwarn_on_msg = warning_on_msg
else:
errwarn = error
errwarn_on_msg = error_on_msg
# Prepare inline progress indicator.
if not op.quiet:
update_progress = init_file_progress(fnames,
addfmt=t_("@info:progress",
"Sieving: %(file)s"))
# Sieve catalogs.
modified_files = []
for fname in fnames:
if op.verbose:
report(_("@info:progress", "Sieving %(file)s...", file=fname))
elif not op.quiet:
update_progress(fname)
if op.msgfmt_check:
d1, oerr, ret = collect_system(
["msgfmt", "-o", "/dev/null", "-c", fname])
if ret != 0:
oerr = oerr.strip()
errwarn(
_("@info:progress", "%(file)s: %(cmd)s check failed:\n"
"%(msg)s",
file=fname,
cmd="msgfmt -c",
msg=oerr))
warning(
_("@info:progress",
"Skipping catalog due to syntax check failure."))
continue
try:
cat = Catalog(fname,
monitored=use_monitored,
headonly=use_headonly,
single_entry=int(op.single_entry))
except CatalogSyntaxError, e:
errwarn(
_("@info:progress",
"%(file)s: Parsing failed: %(msg)s",
file=fname,
msg=e))
warning(
_("@info:progress",
"Skipping catalog due to parsing failure."))
continue
skip = False
# First run all header sieves.
if header_sieves and op.announce_entry:
report(
_("@info:progress",
"Sieving header of %(file)s...",
file=fname))
for sieve in header_sieves:
try:
ret = sieve.process_header(cat.header, cat)
except SieveCatalogError, e:
errwarn(
_("@info:progress",
"%(file)s:header: Sieving failed: %(msg)s",
file=fname,
msg=e))
skip = True
break
if ret not in (None, 0):
break
for sieve in header_sieves:
try:
ret = sieve.process_header(cat.header, cat)
except SieveCatalogError, e:
errwarn(
_("@info:progress",
"%(file)s:header: Sieving failed: %(msg)s",
file=fname,
msg=e))
skip = True
break
if ret not in (None, 0):
break
if skip:
warning(
_("@info:progress",
"Skipping catalog due to header sieving failure."))
continue
# Then run all message sieves on each message,
# unless processing only the header.
if not use_headonly:
for msg in cat:
if op.skip_obsolete and msg.obsolete:
continue
if not op.quiet:
update_progress(fname)
if op.announce_entry:
report(
def apply_ediff(op):
# Read the ediff PO.
dummy_stream_path = "<stdin>"
if op.input:
if not os.path.isfile(op.input):
error(
_("@info",
"Path '%(path)s' is not a file or does not exist.",
path=op.input))
edfpath = op.input
readfh = None
else:
edfpath = dummy_stream_path
readfh = sys.stdin
try:
ecat = Catalog(edfpath, monitored=False, readfh=readfh)
except:
error(
_("@info ediff is shorthand for \"embedded difference\"",
"Error reading ediff '%(file)s'.",
file=edfpath))
# Split ediff by diffed catalog into original and new file paths,
# header message, and ordinary messages.
hmsgctxt = ecat.header.get_field_value(EDST.hmsgctxt_field)
if hmsgctxt is None:
error(
_("@info",
"Header field '%(field)s' is missing in the ediff.",
field=EDST.hmsgctxt_field))
edsplits = []
cehmsg = None
smsgid = u"\x00"
ecat.add_last(MessageUnsafe(dict(msgctxt=hmsgctxt,
msgid=smsgid))) # sentry
for emsg in ecat:
if emsg.msgctxt == hmsgctxt:
if cehmsg:
# Record previous section.
edsplits.append((fpaths, cehmsg, cemsgs))
if emsg.msgid == smsgid: # end sentry, avoid parsing below
break
# Mine original and new file paths out of header.
fpaths = []
for fpath in emsg.msgid.split("\n")[:2]:
# Strip leading "+ "/"- "
fpath = fpath[2:]
# Convert to planform path separators.
fpath = re.sub(r"/+", os.path.sep, fpath)
# Remove revision indicator.
p = fpath.find(EDST.filerev_sep)
if p >= 0:
fpath = fpath[:p]
# Strip path and append directory as requested.
if op.strip:
preflen = int(op.strip)
lst = fpath.split(os.path.sep, preflen)
if preflen + 1 == len(lst):
fpath = lst[preflen]
else:
fpath = os.path.basename(fpath)
else:
fpath = os.path.basename(fpath)
if op.directory and fpath:
fpath = os.path.join(op.directory, fpath)
# All done.
fpaths.append(fpath)
cehmsg = emsg
cemsgs = []
else:
cemsgs.append(emsg)
# Prepare catalog for rejects and merges.
rcat = Catalog("", create=True, monitored=False, wrapping=ecat.wrapping())
init_ediff_header(rcat.header, hmsgctxt=hmsgctxt, extitle="rejects")
# Apply diff to catalogs.
for fpaths, ehmsg, emsgs in edsplits:
# Open catalog for patching.
fpath1, fpath2 = fpaths
if fpath1:
# Diff from an existing catalog, open it.
if not os.path.isfile(fpath1):
warning(
_("@info",
"Path '%(path)s' is not a file or does not exist, "
"skipping it.",
path=fpath1))
continue
try:
cat = Catalog(fpath1)
except:
warning(
_("@info",
"Error reading catalog '%(file)s', skipping it.",
file=fpath1))
continue
elif fpath2:
# New catalog added in diff, create it (or open if it exists).
try:
mkdirpath(os.path.dirname(fpath2))
cat = Catalog(fpath2, create=True)
if cat.created():
cat.set_wrapping(ecat.wrapping())
except:
if os.path.isfile(fpath2):
warning(
_("@info",
"Error reading catalog '%(file)s', skipping it.",
file=fpath1))
else:
warning(
_("@info",
"Cannot create catalog '%(file)s', skipping it.",
file=fpath2))
continue
else:
error(_("@info", "Both catalogs in ediff indicated not to exist."))
# Do not try to patch catalog with embedded differences
# (i.e. previously patched using -e).
if cat.header.get_field_value(EDST.hmsgctxt_field) is not None:
warning(
_("@info", "Catalog '%(file)s' already contains "
"embedded differences, skipping it.",
file=cat.filename))
continue
# Do not try to patch catalog if the patch contains
# unresolved split differences.
if reduce(lambda r, x: r or _flag_ediff_to_new in x.flag, emsgs,
False):
warning(
_("@info", "Patch for catalog '%(file)s' contains unresolved "
"split differences, skipping it.",
file=cat.filename))
continue
# Patch the catalog.
rejected_ehmsg = patch_header(cat, ehmsg, ecat, op)
rejected_emsgs_flags = patch_messages(cat, emsgs, ecat, op)
any_rejected = rejected_ehmsg or rejected_emsgs_flags
if fpath2 or any_rejected:
created = cat.created()
if cat.sync():
if not created:
if any_rejected and op.embed:
report(
_("@info:progress E is for \"with embedding\"",
"Partially patched (E): %(file)s",
file=cat.filename))
elif any_rejected:
report(
_("@info:progress",
"Partially patched: %(file)s",
file=cat.filename))
elif op.embed:
report(
_("@info:progress E is for \"with embedding\"",
"Patched (E): %(file)s",
file=cat.filename))
else:
report(
_("@info:progress",
"Patched: %(file)s",
file=cat.filename))
else:
if op.embed:
report(
_("@info:progress E is for \"with embedding\"",
"Created (E): %(file)s",
file=cat.filename))
else:
report(
_("@info:progress",
"Created: %(file)s",
file=cat.filename))
else:
pass #report("unchanged: %s" % cat.filename)
else:
os.unlink(fpath1)
report(_("@info:progress", "Removed: %(file)s", file=fpath1))
# If there were any rejects and reembedding is not in effect,
# record the necessary to present them.
if any_rejected and not op.embed:
if not rejected_ehmsg:
# Clean header diff.
ehmsg.manual_comment = ehmsg.manual_comment[:1]
ehmsg.msgstr[0] = u""
rcat.add_last(ehmsg)
for emsg, flag in rejected_emsgs_flags:
# Reembed to avoid any conflicts.
msg1, msg2, msg1_s, msg2_s = resolve_diff_pair(emsg)
emsg = msg_ediff(msg1_s,
msg2_s,
emsg=msg2_s,
ecat=rcat,
enoctxt=hmsgctxt)
if flag:
emsg.flag.add(flag)
rcat.add_last(emsg)
# If there were any rejects, write them out.
if len(rcat) > 0:
# Construct paths for embedded diffs of rejects.
rsuff = "rej"
if ecat.filename != dummy_stream_path:
rpath = ecat.filename
p = rpath.rfind(".")
if p < 0:
p = len(rpath)
rpath = rpath[:p] + (".%s" % rsuff) + rpath[p:]
else:
rpath = "stdin.%s.po" % rsuff
rcat.filename = rpath
rcat.sync(force=True, noobsend=True)
report(
_(
"@info:progress file to which rejected parts of the patch "
"have been written to",
"*** Rejects: %(file)s",
file=rcat.filename))
def main():
locale.setlocale(locale.LC_ALL, "")
# Get defaults for command line options from global config.
cfgsec = pology_config.section("poepatch")
def_do_merge = cfgsec.boolean("merge", True)
# Setup options and parse the command line.
usage = _("@info command usage", "%(cmd)s [OPTIONS] [OPTIONS] < EDIFF\n"
"%(cmd)s -u [OPTIONS] PATHS...",
cmd="%prog")
desc = _("@info command description",
"Apply embedded diff of PO files as patch.")
ver = _("@info command version", u"%(cmd)s (Pology) %(version)s\n"
u"Copyright © 2009, 2010 "
u"Chusslove Illich (Часлав Илић) <%(email)s>",
cmd="%prog",
version=version(),
email="*****@*****.**")
opars = ColorOptionParser(usage=usage, description=desc, version=ver)
opars.add_option(
"-a",
"--aggressive",
action="store_true",
dest="aggressive",
default=False,
help=_(
"@info command line option description",
"Apply every message to its paired message in the target file, "
"irrespective of whether its non-pairing parts match too."))
opars.add_option(
"-d",
"--directory",
metavar=_("@info command line value placeholder", "DIR"),
dest="directory",
help=_(
"@info command line option description",
"Prepend this directory path to any resolved target file path."))
opars.add_option(
"-e",
"--embed",
action="store_true",
dest="embed",
default=False,
help=_(
"@info command line option description",
"Instead of applying resolved newer version of the message, "
"add the full embedded diff into the target file."))
opars.add_option(
"-i",
"--input",
metavar=_("@info command line value placeholder", "FILE"),
dest="input",
help=_(
"@info command line option description",
"Read the patch from the given file instead of standard input."))
opars.add_option(
"-n",
"--no-merge",
action="store_false",
dest="do_merge",
default=def_do_merge,
help=_("@info command line option description",
"Do not try to indirectly pair messages by merging catalogs."))
opars.add_option(
"-p",
"--strip",
metavar=_("@info command line value placeholder", "NUM"),
dest="strip",
help=_(
"@info command line option description",
"Strip the smallest prefix containing NUM leading slashes from "
"each file name found in the ediff file (like in patch(1)). "
"If not given, only the base name of each file is taken."))
opars.add_option(
"-u",
"--unembed",
action="store_true",
dest="unembed",
default=False,
help=_(
"@info command line option description",
"Instead of applying a patch, resolve all embedded differences "
"in given paths to newer versions of messages."))
(op, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
# Could use some speedup.
try:
import psyco
psyco.full()
except ImportError:
pass
if not op.unembed:
if free_args:
error(
_("@info",
"Too many arguments in command line: %(argspec)s",
argspec=" ".join(free_args)))
if op.strip and not op.strip.isdigit():
error(
_("@info",
"Option %(opt)s expects a positive integer value.",
opt="--strip"))
apply_ediff(op)
else:
paths = []
for path in free_args:
if not os.path.exists(path):
warning(
_("@info", "Path '%(path)s' does not exist.", path=path))
if os.path.isdir(path):
paths.extend(collect_catalogs(path))
else:
paths.append(path)
for path in paths:
unembed_ediff(path)
if fmtexp not in reported_fmtexps:
if not esuff:
report(fmtexp)
reported_fmtexps.add(fmtexp)
else:
afmtexp = " @" + esuff + ": " + ", ".join(fmtprops)
report(afmtexp)
nproblems += cnproblems
tp.empty_pcache()
if unmatched_srcs:
fmtsrcs = format_item_list(sorted(getattr(x, "pattern", x)
for x in unmatched_srcs))
warning(_("@info",
"Sources requested by name not found: %(srclist)s.",
srclist=fmtsrcs))
if unmatched_keys:
fmtkeys = format_item_list(sorted(getattr(x, "pattern", x)
for x in unmatched_keys))
warning(_("@info",
"Derivations requested by key not found: %(keylist)s.",
keylist=fmtkeys))
return nproblems
class _Wre (object):
def __init__ (self, pattern):
def validate (tp, onlysrcs=None, onlykeys=None, demoexp=False, expwkeys=False):
needed_pkeys = set()
nom_pkeys = (
[u"н"],
[u"нм", u"нж", u"нс", u"ну"],
)
needed_pkeys.update(sum(nom_pkeys, []))
gender_pkey = u"_род"
needed_pkeys.add(gender_pkey)
known_genders = set((u"м", u"ж", u"с", u"у"))
known_genders.update(map(ctol, known_genders))
known_alts = [
("_s", u"сист"),
("_a", u"алт"),
("_a2", u"алт2"),
("_a3", u"алт3"),
]
base_envs = [u"", u"л", u"иј", u"ијл"]
all_envs = set(base_envs)
for aenv in [x[1] for x in known_alts]:
all_envs.update(x + aenv for x in base_envs)
if demoexp:
demoexp_pkeys = [u"н", u"г", u"д", u"а", u"в", u"и",
u"нк", u"гк", u"дк", u"ак", u"вк",
u"нм", u"нмп"]
needed_pkeys.update(demoexp_pkeys)
dkeys_by_rtkey = {}
# Sort keys such that derivations are checked by file and position.
dkeys = tp.dkeys(single=onlykeys is None)
def sortkey (x):
path, lno, cno = tp.source_pos(x)
return path.count(os.path.sep), path, lno, cno
dkeys = sorted(dkeys, key=sortkey)
nproblems = 0
unmatched_srcs = set(onlysrcs) if onlysrcs is not None else None
unmatched_keys = set(onlykeys) if onlykeys is not None else None
reported_fmtexps = set()
for dkey in dkeys:
srcname = tp.source_name(dkey)
path, lno, cno = tp.source_pos(dkey)
cnproblems = 0
if ( ( onlysrcs is not None
and not _match_text(srcname, onlysrcs, unmatched_srcs))
or ( onlykeys is not None
and not _match_text(dkey, onlykeys, unmatched_keys))
):
continue
try:
aprops = []
seenesuffs = set()
cenvs = tp.envs(dkey)
for cenv in cenvs:
if cenv != "":
envmatched = False
for ksuff, esuff in known_alts:
if cenv in all_envs and cenv.endswith(esuff):
envmatched = True
break
else:
envmatched = True
ksuff, esuff = "", ""
if envmatched and esuff not in seenesuffs:
dkeym = dkey + ksuff
props = dict([(x, tp.get2(dkeym, norm_pkey(x)))
for x in needed_pkeys])
aprops.append((esuff, props))
seenesuffs.add(esuff)
elif cenv not in all_envs:
warning(_("@info",
"Derivation at %(file)s:%(line)d:%(col)d "
"defines unknown environment '%(env)s'.",
file=path, line=lno, col=cno, env=cenv))
cnproblems += 1
except Exception, e:
warning(str_to_unicode(str(e)))
cnproblems += 1
continue
for esuff, props in aprops:
# Assure all nominative forms are unique.
for pkeys in nom_pkeys: # select first nominative set by priority
pvals = [props.get(x) for x in pkeys]
noms = filter(lambda x: x is not None, pvals)
if noms:
break
if noms:
rtkeys = map(norm_rtkey, noms)
for rtkey in rtkeys:
odkey = dkeys_by_rtkey.get(rtkey)
if odkey is not None and tp.props(dkey) != tp.props(odkey):
opath, olno, ocno = tp.source_pos(odkey)
warning(_("@info",
"Derivation at %(file1)s:%(line1)d:%(col1)d "
"has normalized nominative equal to "
"derivation at %(file2)s:%(line2)d:%(col2)d; "
"consider adding a disambiguation marker "
"(%(dchar)s).",
file1=path, line1=lno, col1=cno,
file2=opath, line2=olno, col2=ocno,
dchar=_disamb_marker))
cnproblems += 1
for rtkey in rtkeys: # must be in new loop
dkeys_by_rtkey[rtkey] = dkey
# Assure presence of gender on noun derivations.
if props.get(nom_pkeys[0][0]) is not None:
gender = props.get(gender_pkey)
if gender is None:
warning(_("@info",
"Derivation at %(file)s:%(line)d:%(col)d "
"does not define gender.",
file=path, line=lno, col=cno))
cnproblems += 1
else:
for gender in hictoall(gender):
if gender not in known_genders:
warning(_("@info",
"Derivation at %(file)s:%(line)d:%(col)d "
"defines unknown gender '%(gen)s'.",
file=path, line=lno, col=cno, gen=gender))
cnproblems += 1
# Show selection of expanded properties if requested.
if demoexp and not cnproblems:
demoprops = [(x, props.get(x)) for x in demoexp_pkeys]
demoprops = filter(lambda x: x[1] is not None, demoprops)
fmtprops = ["%s=%s" % (x[0], _escape_pval(x[1]))
for x in demoprops]
fmtsyns = ["%s" % _escape_syn(x) for x in tp.syns(dkey)]
fmtexp = ", ".join(fmtsyns) + ": " + ", ".join(fmtprops)
if expwkeys:
fmtdkeys = ", ".join(sorted(tp.altdkeys(dkey)))
fmtexp = "# " + fmtdkeys + "\n" + fmtexp
if fmtexp not in reported_fmtexps:
if not esuff:
report(fmtexp)
reported_fmtexps.add(fmtexp)
else:
afmtexp = " @" + esuff + ": " + ", ".join(fmtprops)
report(afmtexp)
nproblems += cnproblems
tp.empty_pcache()
def equip_header(hdr, cat):
"""
Add extra information to header [type F4B hook].
The following header fields are set:
- C{Language}: the language code of translation;
set only if the language can be determined
- C{X-Environment}: linguistic subset of the language of translation
(team choices on terminology, ortography...);
set to C{kde} if not existing, otherwise left untouched.
- C{X-Accelerator-Marker}: accelerator marker character which may
be encountered in text
- C{X-Text-Markup}: text markups (e.g. Qt rich text, Docbook...) which
may be encountered in text, as keywords
For the hook to function properly, the local checkout of language catalogs
must match the repository structure up to a certain level.
See the documentation on C{check-tp-kde} sieve for details.
TODO: Put that instruction here.
"""
cname = cat.name
csubdir = get_project_subdir(cat.filename)
if not csubdir:
warning(
_("@info TP stands for Translation Project",
"Cannot determine KDE TP subdirectory "
"of '%(file)s', skipping header updates.",
file=cat.filename))
return 1
pathels = os.path.abspath(cat.filename).split(os.path.sep)
lang_rx = re.compile(r"^[a-z]{2}(_[A-Z]{2}|@[a-z]+)?$")
lang = None
if len(pathels) >= 5 and pathels[-4] == "summit":
if lang_rx.search(pathels[-5]):
lang = pathels[-5]
elif len(pathels) >= 4:
if lang_rx.search(pathels[-4]):
lang = pathels[-4]
if is_txt_cat(cname, csubdir):
accmark = ""
mtypes = [""]
elif is_qt_cat(cname, csubdir):
accmark = "&"
mtypes = ["qtrich"]
elif is_docbook_cat(cname, csubdir):
accmark = ""
mtypes = ["docbook4"]
elif is_html_cat(cname, csubdir):
accmark = ""
mtypes = ["html"]
elif is_unknown_cat(cname, csubdir):
accmark = None
mtypes = None
else: # default to native KDE4 catalog
accmark = "&"
mtypes = ["kde4"]
fvs = []
fvs.append(("Language", lang, "Language-Team", False))
fvs.append(("X-Environment", u"kde", None, True))
if accmark is not None:
fvs.append(("X-Accelerator-Marker", accmark, None, False))
if mtypes is not None:
fvs.append(("X-Text-Markup", ", ".join(mtypes), None, False))
for fnam, fval, fnamaft, fkeep in fvs:
if fval is None:
continue
existing = hdr.select_fields(fnam)
if not (existing and fkeep):
if len(existing) > 1:
hdr.remove_field(fnam)
hdr.set_field(unicode(fnam), unicode(fval), after=fnamaft)
return 0
def unembed_ediff(path, all=False, old=False):
try:
cat = Catalog(path)
except:
warning(
_("@info",
"Error reading catalog '%(file)s', skipping it.",
file=path))
return
hmsgctxt = cat.header.get_field_value(EDST.hmsgctxt_field)
if hmsgctxt is not None:
cat.header.remove_field(EDST.hmsgctxt_field)
uehmsg = None
unembedded = {}
for msg in cat:
ediff_flag = None
for flag in _flags_all:
if flag in msg.flag:
ediff_flag = flag
msg.flag.remove(flag)
if not ediff_flag and not all:
continue
if ediff_flag in (_flag_ediff_no_match, _flag_ediff_to_new):
# Throw away fully rejected embeddings, i.e. reject the patch.
# For split-difference embeddings, throw away the current-to-new;
# this effectively rejects the patch, which is safest thing to do.
cat.remove_on_sync(msg)
elif hmsgctxt is not None and msg.msgctxt == hmsgctxt:
if uehmsg:
warning_on_msg(
_("@info", "Unembedding results in duplicate header, "
"previous header at %(line)d(#%(entry)d); "
"skipping it.",
line=uehmsg.refline,
entry=uehmsg.refentry), msg, cat)
return
msg_ediff_to_x = not old and msg_ediff_to_new or msg_ediff_to_old
hmsg = msg_ediff_to_x(clear_header_metadata(msg))
if hmsg.msgstr and hmsg.msgstr[0]:
cat.header = Header(hmsg)
cat.remove_on_sync(msg)
uehmsg = msg
else:
msg1, msg2, msg1_s, msg2_s = resolve_diff_pair(msg)
tmsg = (not old and (msg2, ) or (msg1, ))[0]
if tmsg is not None:
if tmsg.key in unembedded:
msg_p = unembedded[tmsg.key]
warning_on_msg(
_("@info", "Unembedding results in "
"duplicate message, previous message "
"at %(line)d(#%(entry)d); skipping it.",
line=msg_p.refline,
entry=msg_p.refentry), msg, cat)
return
msg.set(Message(msg2))
unembedded[tmsg.key] = msg
else:
cat.remove_on_sync(msg)
if cat.sync():
report(_("@info:progress", "Unembedded: %(file)s", file=cat.filename))
