def reject(dbh, entr, edtree, errs, rejcnt=None):
KW = jdb.KW
L('cgi.edsubmit.reject').debug("rejecting entry id %s, rejcnt=%s" %
(entr.dfrm, rejcnt))
# rejectable() returns a list entr rows on the path to the edit
# edit root, starting with the one closest to the root, and ending
# with our entry's parent, that can be rejected. If this is a new
# entry, 'rejs' will be set to [].
try:
rejs = rejectable(edtree, entr.dfrm)
except NonLeafError as e:
L('cgi.edsubmit.reject').debug("NonLeafError")
errs.append (jmcgi.Markup("Edits have been made to this entry. "\
"To reject entries, you must reject the version(s) most "
"recently edited, which are: %s"\
% ', '.join ("id="+url(x) for x in leafsn([e.args[0]]))))
return
except IsApprovedError as e:
L('cgi.edsubmit.reject').debug("IsApprovedrror")
errs.append("You can only reject unapproved entries.")
return
if not rejcnt or rejcnt > len(rejs): rejcnt = len(rejs)
chhead = (rejs[-rejcnt]).id if rejcnt else None
L('cgi.edsubmit.reject').debug("rejs=%r, rejcnt=%d, chhead=%s" %
([x.id for x in rejs], -rejcnt, chhead))
entr.stat = KW.STAT['R'].id
entr.dfrm = None
entr.unap = False
res = addentr(dbh, entr)
if chhead: delentr(dbh, chhead)
return res
def mkxref(v):
# If there is no tsens, generate an xref to only the first
# sense. Rationale: Revs prior to ~2018-06-07 we generated
# xrefs to all senses in this scenario. When there were a
# lot of reverse xrefs to a word from the Example sentences,
# every sense of the target word would have them all repeated.
# However unless there is only one target sense, we can be
# sure we are wrong: if the senses were so similar to be
# interchangable they wouldn't be separate senses. Since
# we'll be wrong either way and someone will need to manually
# correct it later, choose the way that produces the least
# amount of clutter in the entry. Also, in many cases the
# first *will* be the right sense.
nosens = False
if not v.tsens:
if v.nsens != 1:
L('multiple senses').warning("using sense 1: %s" % (fs(v)))
nosens = True
v.tsens = 1
if v.tsens > v.nsens:
L('sense number too big').error(fs(v))
return None
xref = jdb.Obj(entr=v.entr,
sens=v.sens,
xref=v.ord,
typ=v.typ,
xentr=v.targ,
xsens=v.tsens,
rdng=v.rdng,
kanj=v.kanj,
notes=v.notes,
nosens=nosens,
lowpri=not v.prio)
return xref
def del_xresolv(dbconn,
xref_src=[],
targ_src=None,
start=None,
stop=None,
active=False):
# CAUTION: we assume here that xref.xref has the same value
# as xresolv.ord and thus we can use xref.xref to delete
# the corresponding xresolv row. That is currently true but
# has not been true in past and could change in the future.
#FIXME: this replicates same WHERE logic used in resolve();
# should factor out to a single location, either in python
# code or in a database view.
c0 = "v.entr=x.entr AND v.sens=x.sens"\
" AND v.typ=x.typ AND v.ord=x.xref"
c1, args1 = src_clause(xref_src, targ_src)
c2, args2 = idrange_clause(start, stop)
c3 = "v.stat=2 AND NOT v.unapp" if active else ""
c4 = "tstat=2"
c5 = "" #"NOT tunap" if appr_targ else "")
whr = " AND ".join([c for c in [c0, c1, c2, c3, c4, c5] if c])
if whr: whr = "WHERE " + whr
args = args1 + args2
sql = "DELETE FROM xresolv v"\
" USING (SELECT x.entr, x.sens, x.xref, x.typ,"\
" ex.src, ex.stat, ex.unap,"\
" et.src as tsrc,et.stat AS tstat,et.unap AS tunap"\
" FROM xref x"\
" JOIN entr ex ON x.entr=ex.id"\
" JOIN entr et ON x.xentr=et.id) AS x"\
" %s" % whr
L('del_xresolv.sql').debug("sql: %s" % sql)
L('del_xresolv.sql').debug("args: %r" % (args, ))
cursor = db.ex(dbconn, sql, args)
return cursor.rowcount
def addentr (dbh, entr):
entr._hist[-1].unap = entr.unap
entr._hist[-1].stat = entr.stat
L('edsubmit').debug("addentr(): adding entry to database")
L('edsubmit').debug("addentr(): %d hists, last hist is %s [%s] %s" % (len(entr._hist),
entr._hist[-1].dt, entr._hist[-1].userid, entr._hist[-1].name))
res = jdb.addentr (dbh, entr)
L('edsubmit').debug("addentr(): entry id=%s, seq=%s, src=%s added to database" % res)
return res
def submit (dbh, entr, edtree, errs):
KW = jdb.KW
L('edsubmit').debug("submit(): submitting entry with parent id %s" % entr.dfrm)
if not entr.dfrm and entr.stat != KW.STAT['A'].id:
L('edsubmit').debug("submit(): bad url param exit")
errs.append ("Bad url parameter, no dfrm"); return
if entr.stat == jdb.KW.STAT['R'].id:
L('edsubmit').debug("submit(): bad stat=R exit")
errs.append ("Bad url parameter, stat=R"); return
res = addentr (dbh, entr)
return res
def main(args, opts):
jdb.reset_encoding(sys.stdout, 'utf-8')
errs = []
chklist = {}
try:
form, svc, dbg, cur, sid, sess, parms, cfg = jmcgi.parseform()
except Exception as e:
jmcgi.err_page([str(e)])
fv = form.getfirst
fl = form.getlist
if not sess or sess.priv != 'A': users = []
else:
sql = "SELECT * FROM users ORDER BY userid"
sesscur = jdb.dbOpenSvc(cfg, svc, session=True, nokw=True)
users = jdb.dbread(sesscur, sql)
L('cgi.users').debug('read %d rows from table "user"' % (len(users), ))
jmcgi.jinja_page("users.jinja",
users=users,
session=sess,
cfg=cfg,
parms=parms,
svc=svc,
dbg=dbg,
sid=sid,
this_page='user.py',
result=fv('result'))
def rejectable(edtree, id):
# edtree -- A (root,dict) 2-tuple as returned by get_subtree().
# id -- (int) An id number of an entr row in 'edtree'.
#
# Returns: A list of rows in 'edtree' starting at row 'id'
# and moving back towards the root via the 'dfrm' references
# until one of the following conditions obtain:
# * An entry is reached that is referenced by more than one
# other entry. (This is an entry that has two or more
# direct edits.)
# * An entry that is not "unapproved".
# The entry that terminated the search is not included
# in the list.
# If the entry row designated by 'id' is not a leaf node,
# or is approved, an error is raised.
if not edtree:
L('cgi.edsubmit.rejectable').debug("edtree is none, returning")
return []
root, d = edtree
erow = d[id]
if erow._dfrm: raise NonLeafError(erow)
if not erow.unap: raise IsApprovedError(erow)
erows = []
while 1:
erows.append(erow)
if not erow.dfrm: break
erow = d[erow.dfrm]
if not erow.unap or len(erow._dfrm) > 1: break
erows.reverse()
return erows
def resolve(dbconn,
tmpf,
xref_src,
targ_src,
start=None,
stop=None,
active=False):
# Call process_entr_cands() to get candidate entries for the
# unresolved xrefs limited by our function arguments, choose the
# optimal one for each unresolved xref, create a corresponding
# xref row and write to the temp file 'tmpf'.
entrcnt = unrefcnt = xrefcnt = 0
for rows in get_entr_cands(dbconn, xref_src, targ_src, start, stop,
active):
# 'rows' is a set of target candidate rows for all the
# unresolved xrefs for a single entry.
xrefs = process_entr_cands(rows)
L('resolve').debug("process_entr_cands returned %s xrefs" % len(xrefs))
for x in xrefs:
csv = '\t'.join([
str(x.entr),
str(x.sens),
str(x.xref),
str(x.typ),
str(x.xentr),
str(x.xsens),
str(x.rdng) if x.rdng else '\\N',
str(x.kanj) if x.kanj else '\\N', x.notes or '\\N',
"t" if x.nosens else "f", "t" if x.lowpri else "f"
]) + '\n'
tmpf.write(csv)
entrcnt += 1
unrefcnt += unrefcount(rows)
xrefcnt += len(xrefs)
return entrcnt, unrefcnt, xrefcnt
def main(args, opts):
jdb.reset_encoding(sys.stdout, 'utf-8')
errs = []
chklist = {}
try:
form, svc, dbg, cur, sid, sess, parms, cfg = jmcgi.parseform()
except Exception as e:
jmcgi.err_page([str(e)])
fv = form.getfirst
fl = form.getlist
L('cgi.user').debug("sess=%r" % sess)
L('cgi.user').debug("u=%r, new=%r" % (fv('u'), fv('new')))
# If 'sess' evaluates false, user is not logged in in which
# case we go directly to the "user" page which will say that
# one has to be logged in. We do this instead of using jmcgi.-
# err_page() because the notes on the users.jinja page inform
# the user about the requirements for access.
# If user is logged in but is not an Admin user, we ignore
# the "u=..." url parameter and show user info only for
# the logged in user.
# If the user is an Admin user, we show info for the user
# requested by the "u=..." parameter if any, or the logged
# in user if not.
# There may still be no user due to "u=..." for non-existant
# or some corner-case but that's ok, page will show a "no user
# found" message.
user = userid = new = None
if sess: # I.e., if logged in...
if sess.priv == 'A':
userid, new = fv('u'), fv('new')
if not new and not userid: userid = sess.userid
L('cgi.user').debug("userid=%r, new=%r" % (userid, new))
if userid:
user = jmcgi.get_user(userid, svc, cfg)
L('cgi.user').debug("read user data: %s" % (sanitize_o(user), ))
L('cgi.user').debug("rendering template, new=%r" % new)
jmcgi.jinja_page("user.jinja",
user=user,
result=fv('result'),
session=sess,
cfg=cfg,
parms=parms,
new=new,
svc=svc,
sid=sid,
dbg=dbg,
this_page='user.py')
def delentr(dbh, id):
# Delete entry 'id' (and by cascade, any edited entries
# based on this one). This function deletes the entire
# entry, including history. To delete the entry contents
# but leaving the entr and hist records, use database
# function delentr. 'dbh' is an open dbapi cursor object.
L('cgi.edsubmit.delentr').debug("deleting entry id %s from database" % id)
sql = "DELETE FROM entr WHERE id=%s"
dbh.execute(sql, (id, ))
def approve(dbh, entr, edtree, errs):
KW = jdb.KW
L('cgi.edsubmit.approve').debug("approving entr id %s" % entr.dfrm)
# Check stat. May be A or D, but not R.
if entr.stat == KW.STAT['R'].id:
L('cgi.edsubmit.approve').debug("stat=R exit")
errs.append("Bad url parameter, stat=R")
return
dfrmid = entr.dfrm
if dfrmid:
# Since 'dfrmid' has a value, this is an edit of an
# existing entry. Check that there is a single edit
# chain back to the root entry.
try:
approve_ok(edtree, dfrmid)
except NonLeafError as e:
L('cgi.edsubmit.approve').debug("NonLeafError")
errs.append (jmcgi.Markup("Edits have been made to this "\
"entry. You need to reject those edits before you can "\
"approve this entry. The id numbers are: %s"\
% ', '.join ("id="+url(x) for x in leafsn([e.args[0]]))))
return
except BranchesError as e:
L('cgi.edsubmit.approve').debug("BranchesError")
errs.append (jmcgi.Markup("There are other edits pending on "\
"some of the predecessor entries of this one, and this "\
"entry cannot be approved until those are rejected. "\
"The id numbers are: %s"\
% ', '.join ("id="+url(x) for x in leafsn(e.args[0]))))
return
# Write the approved entry to the database...
entr.dfrm = None
entr.unap = False
res = addentr(dbh, entr)
# ...and delete the old root if any. Because the dfrm foreign
# key is specified with "on delete cascade", deleting the root
# entry will also delete all it's children. edtree[0].id is
# the id number of the edit root.
if edtree: delentr(dbh, edtree[0].id)
return res
def dblogin(cur, userid, password):
# Login by authenticating the userid/password pair.
# and getting a session record which is returned with
# the session id. If 'userid' has an active session
# already, it (the most recent one if more than one)
# is returned. If not, a new session is created.
# Reusing existing sessions help prevent the proliferation
# of sessions that was occuring previously.
# Check userid, password validity.
sql = "SELECT userid FROM users "\
"WHERE userid=%s AND pw=crypt(%s, pw) AND NOT disabled"
rs = jdb.dbread(cur, sql, (userid, password))
if not rs:
L('cgi.jmcgi').debug("login: pw fail for %s" % userid)
time.sleep(1)
return '', None
# Look for an existing session (the most recent if more than one).
sql = "SELECT s.id,s.userid,s.ts,u.fullname,u.email,u.priv" \
" FROM sessions s JOIN users u ON u.userid=s.userid" \
" WHERE u.userid=%%s AND NOT u.disabled" \
" AND (NOW()-ts)<'%s'::INTERVAL" \
" ORDER BY ts DESC LIMIT 1" % SESSION_TIMEOUT
rs = jdb.dbread(cur, sql, (userid, ))
L('cgi.jmcgi').debug("login: %s: %s sessions found" % (userid, len(rs)))
if len(rs) == 1:
sid = rs[0][0]
# Update the session timestamp to 'now'.
db_update_sid_ts(cur, sid)
L('cgi.jmcgi').debug("login: %s: using session: %s" % (userid, sid))
return sid, rs[0]
# No existing session found, create a new session.
sql = "INSERT INTO sessions(userid) VALUES(%s) RETURNING(id)"
cur.execute(sql, (userid, ))
sid = cur.fetchone()[0]
cur.connection.commit()
L('cgi.jmcgi').debug("login: %s: new session %s" % (userid, sid))
sess = db_validate_sid(cur, sid)
return sid, sess
def db_validate_sid(cur, sid):
# Check that 'sid' is an existing session and if so
# return a session record. Otherwise return None.
sql = "SELECT s.id,s.userid,s.ts,u.fullname,u.email,u.priv" \
" FROM sessions s JOIN users u ON u.userid=s.userid" \
" WHERE id=%%s AND NOT u.disabled" \
" AND (NOW()-ts)<'%s'::INTERVAL" \
% SESSION_TIMEOUT
rs = jdb.dbread(cur, sql, (sid, ))
L('cgi.jmcgi').debug("login: validating sid %s, result=%s" %
(sid, len(rs)))
if len(rs) == 0: return None
return rs[0]
def choose_candidate(rows):
L('choose_candidate').debug("received %d rows" % len(rows))
if len(rows) == 1:
# There is only one candidates row based on which we can decide
# unambiguously among three possibilities: 1) there was no match,
# 2) there were multiple matches, or 3) we found exactly one
# match.
v = rows[0]
if v.tsrc is None:
L('xref target not found').error(fs(v))
return None
if v.nentr == 1:
L('xref resolved (1)').info(fs(v))
return v
if rows[0].nentr > 1:
L('multiple candidates').error(fs(v))
return None
# If there were multiple candidates rows we need to look through
# them all to find the best choice. We will never choose a
# candidates row that has .nentr>1 since that is definitionally
# a match to multiple targets. Also note the candidates must
# already match the unresolved xref's reading or kanji or both
# (if the xref specifies both).
for v in rows:
# If looking for a reading match, first choice is a match
# to the first reading of an entry that has no kanji.
if v.nentr == 1 and not v.ktxt and v.rdng == 1 and v.nokanji:
L('xref resolved (2)').info(fs(v))
return v
for v in rows:
# Otherwise look for a candidate that matches on either the
# first reading or first kanji.
first = (v.rtxt and v.rdng == 1) or (v.ktxt and v.kanj == 1)
if v.nentr == 1 and first:
L('xref resolved (3)').info(fs(v))
return v
for v in rows:
# And as last choice accept a candidate with a matching
# reading and/or kanji regardless of the position of the
# matches.
if v.nentr == 1:
L('xref resolved (4)').info(fs(v))
return v
L('multiple candidates').error(fs(v))
return None
def approve_ok(edtree, id):
# edtree -- A (root,dict) 2-tuple as returned by get_subtree().
# id -- (int) An id number of an entr row in 'edtree'.
# Returns: None
#
# An entry is approvable if:
# 1. It is a leaf in the edit tree.
# We can't approve a non-leaf node because that would
# require changing the dfrm links of following nodes
# (rule is that we only add or erase entries; we don't
# change existing ones) and even if we did that, the
# history records of the following nodes would no
# longer be correct.
# 2. There are no edit branches on the path back to the
# root node, i.e. we have a single linear string of
# edits. All other edits in the tree are erased when
# we approve this one. The approved entry carries
# the history of all earlier entries on its path, but
# not any history from other branches. We require
# those branches be explicitly rejected (removing
# them from our tree) first so that history is not
# unknowingly discarded and to be sure approver is
# aware of them.
#
# If the given entry is not approvable, an error is raised.
if not edtree:
L('cgi.edsubmit.approve_ok').debug("edtree is none, returning")
return
root, d = edtree
erow = d[id]
branches = []
if erow._dfrm: raise NonLeafError(erow)
while erow != root:
last = erow
erow = d[erow.dfrm]
if len(erow._dfrm) > 1:
branches.extend([x for x in erow._dfrm if x != last])
if branches: raise BranchesError(branches)
return
def initcgi(cfgfilename):
"""
Does three things:
1. Fixes any url command line argument so that it is usable by
the cgi module (see the args() function).
Currently the only argument accepted is a url used when this
script run interactively for debugging. In the future it may
also accept an option giving the location of a configuration
file.
2. Reads the configuration file (location currently hardwired.)
3. Initializes the Python logging system so log messages will have
a consistent format. Callers of this function should not use
the logger.L() function until after this function is called.
"""
# Adjust any url argument so that the cgi module can use it.
args()
# If the .ini filename below has no directory separator in it,
# it is looked for in a directory on sys.path. If it does have
# a separator in it it is treated as a normal relative or
# absolute path.
cfg = jdb.cfgOpen(cfgfilename)
logfname = cfg.get('web', 'LOG_FILENAME', fallback='jmdictdb.log')
loglevel = cfg.get('web', 'LOG_LEVEL', fallback='debug')
# Additional logging configuration. If there is a [logging]
# section in the config file, it should contain key=value
# pairs where each key is the name of a logger used in the
# jmdictdb code and the corresponding value is a logging
# level that the logger will be set to. The level may be
# numeric or a logging module keyword ("info", "error", etc)
# in either case. This is to allow fine tuning exactly
# what logging messages will be output.
logger.log_config(level=loglevel, filename=logfname)
if cfg.has_section('logging'):
for k, v in cfg.items('logging'):
L(k).setLevel(logger.levelnum(v))
return cfg
def err_page(errs):
L('cgi.edsubmit.err_page').debug("going to error page. Errors:\n%s" %
'\n'.join(errs))
jmcgi.err_page(errs)
def main(cmdln_args):
args = parse_cmdline(cmdln_args)
dburi = args.database
try:
dbconn = db.connect(dburi)
except jdb.dbapi.OperationalError as e:
sys.exit("Error, unable to connect to database: %s" % str(e))
KW = jdb.Kwds(dbconn.cursor())
# Monkey patch jdb module until migration to db module is complete
# because many lib functions still expect KW as a jdb global.
jdb.KW = KW
level = loglevel(args.level)
logger.log_config(level=1 if args.messages else level,
filename=args.logfile)
if args.messages:
try:
filter = parse_mlist(args.messages, level)
except ValueError as e:
sys.exit("Bad -m option: '%s'" % str(e))
L().handlers[0].addFilter(filter)
try:
xref_src = get_src_ids(args.source_corpus)
except KeyError:
L('unknown corpus').warning(args.source_corpus)
sys.exit(0)
try:
targ_src = get_src_ids(args.target_corpus)
except KeyError:
L('unknown corpus').warning(args.target_corpus)
sys.exit(0)
# Read the unresolved xrefs and their candidate targets
# from the database, choose the optimal candidates and write
# the xrefs generated from them to file 'tmpf'.
tmpf = io.StringIO()
entrcnt, unrefcnt, xrefcnt \
= resolve (dbconn, tmpf, xref_src, targ_src,
start=args.start, stop=args.stop,
active=not args.ignore_nonactive)
# Write the generated xrefs in file 'tmpf' back to the
# database using Postgresql's COPY command (via psycopg2's
# 'copy_from()' function).
cursor = dbconn.cursor()
tmpf.seek(0)
#FIXME: wrap following in try/except
cursor.copy_from(tmpf, 'xref')
# Now delete any unresolved xrefs that have corresponding
# resolved xrefs. The delete is restricted to the same
# xref and target .src values and start and stop entry id's
# that were used in selecting the unresolved xrefs. However
# we still delete *any* matching unresolved xref whether
# the matched xref was created by us or not.
delcnt = 0
if not args.keep:
delcnt = del_xresolv(dbconn,
xref_src,
targ_src,
start=args.start,
stop=args.stop,
active=not args.ignore_nonactive)
L().info("processed %d unresolved, %d xrefs generated, "
"%d xresolvs deleted" % (unrefcnt, xrefcnt, delcnt))
if args.noaction:
L().info("rollback due to --noaction")
dbconn.rollback()
else:
dbconn.commit()
num_ecc_hidden_size = 64
num_ecc_out_size = 2
num_fc_hidden_size = 64
num_edge_feature = 1
batch_size = 100
test_size = 1000
lr = 0.001
dropout = True
seed = 0
logger = te.Logger('./logs/gcn_predictor/missing_entity=' +
str(missing_entity) + '_num_fc_hidden_size=' +
str(num_fc_hidden_size) + 'num_ecc_hidden_size=' +
str(num_ecc_hidden_size) + '_lr=' + str(lr) + '_dropout=' +
str(dropout) + '/seed=' + str(seed))
mylogger = L('./mylogs/gcn_predictor/missing_entity=' + str(missing_entity) +
'_num_fc_hidden_size=' + str(num_fc_hidden_size) +
'num_ecc_hidden_size=' + str(num_ecc_hidden_size) + '_lr=' +
str(lr) + '_dropout=' + str(dropout))
set_seed(seed)
class EdgeNet(nn.Module):
def __init__(self, input_size, output_size):
super(EdgeNet, self).__init__()
self.fc1 = nn.Sequential(nn.Linear(input_size, num_fc_hidden_size),
nn.ReLU(True))
self.fc2 = nn.Sequential(nn.Linear(num_fc_hidden_size, output_size))
def forward(self, x):
h1 = self.fc1(x)
out = self.fc2(h1)
return out
def submission(dbh, entr, disp, errs, is_editor=False, userid=None):
# Add a changed entry, 'entr', to the jmdictdb database accessed
# by the open DBAPI cursor, 'dbh'.
#
# dbh -- An open DBAPI cursor
# entr -- A populated Entr object that defines the entry to
# be added. See below for description of how some of its
# attributes affect the submission.
# disp -- Disposition, one of three string values:
# '' -- Submit as normal user.
# 'a' -- Approve this submission.
# 'r' -- Reject this submission.
# errs -- A list to which an error messages will be appended.
# Note that if the error message contains html it should be
# wrapped in jmcgi.Markup() to prevent it from being escaped
# in the template. Conversely, error messages that contain
# text from user input should NOT be so wrapped since they
# must be escaped in the template.
# is_editor -- True is this submission is being performed by
# a logged in editor. Approved or Rejected dispositions will
# fail if this is false. Its value may be conveniently
# obtained from jmcgi.is_editor(). False if a normal user.
# userid -- The userid if submitter is logged in editor or
# None if not.
#
# Note that we never modify existing database entries other
# than to sometimes completetly erase them. Submissions
# of all three types (submit, approve, reject) *always*
# result in the creation of a new entry object in the database.
# The new entry will be created by writing 'entr' to the
# database. The following attributes in 'entr' are relevant:
#
# entr.dfrm -- If None, this is a new submission. Otherwise,
# it must be the id number of the entry this submission
# is an edit of.
# entr.stat -- Must be consistent with changes requested. In
# particular, if it is 4 (Delete), changes made in 'entr'
# will be ignored, and a copy of the parent entry will be
# submitted with stat D.
# entr.src -- Required to be set, new entry will copy.
# # FIXME: prohibit non-editors from making src
# # different than parent?
# entr.seq -- If set, will be copied. If not set, submission
# will get a new seq number but this untested and very
# likely to break something.
# # FIXME: prohibit non-editors from making seq number
# # different than parent, or non-null if no parent?
# entr.hist -- The last hist item on the entry will supply
# the comment, email and name fields to newly constructed
# comment that will replace it in the database. The time-
# stamp and diff are regenerated and the userid field is
# set from our userid parameter.
# # FIXME: should pass history record explicity so that
# # we can be sure if the caller is or is not supplying
# # one. That will make it easier to use this function
# # from other programs.
# The following entry attributes need not be set:
# entr.id -- Ignored (reset to None).
# entr.unap -- Ignored (reset based on 'disp').
# Additionally, if 'is_editor' is false, the rdng._freq and
# kanj._freq items will be copied from the parent entr rather
# than using the ones supplied on 'entr'. See jdb.copy_freqs()
# for details about how the copy works when the rdng's or kanj's
# differ between the parent and 'entr'.
KW = jdb.KW
L('cgi.edsubmit.submission').info(
("disp=%s, is_editor=%s, userid=%s, entry id=%s,\n" + " " * 36 +
"parent=%s, stat=%s, unap=%s, seq=%s, src=%s") %
(disp, is_editor, userid, entr.id, entr.dfrm, entr.stat, entr.unap,
entr.seq, entr.src))
L('cgi.edsubmit.submission').info("entry text: %s %s" %
((';'.join(k.txt for k in entr._kanj)),
(';'.join(r.txt for r in entr._rdng))))
L('cgi.edsubmit.submission').debug("seqset: %s" %
logseq(dbh, entr.seq, entr.src))
oldid = entr.id
entr.id = None # Submissions, approvals and rejections will
entr.unap = not disp # always produce a new db entry object so
merge_rev = False # nuke any id number.
if not entr.dfrm: # This is a submission of a new entry.
entr.stat = KW.STAT['A'].id
entr.seq = None # Force addentr() to assign seq number.
pentr = None # No parent entr.
edtree = None
else: # Modification of existing entry.
edroot = get_edroot(dbh, entr.dfrm)
edtree = get_subtree(dbh, edroot)
# Get the parent entry and augment the xrefs so when hist diffs
# are generated, they will show xref details.
L('cgi.edsubmit.submission').debug("reading parent entry %d" %
entr.dfrm)
pentr, raw = jdb.entrList(dbh, None, [entr.dfrm], ret_tuple=True)
if len(pentr) != 1:
L('cgi.edsubmit.submission').debug("missing parent %d" % entr.dfrm)
# The editset may have changed between the time our user
# displayed the Confirmation screen and they clicked the
# Submit button. Changes involving unapproved edits result
# in the addition of entries and don't alter the preexisting
# tree shape. Approvals of edits, deletes or rejects may
# affect our subtree and if so will always manifest themselves
# as the disappearance of our parent entry.
errs.append(
"The entry you are editing no loger exists because it "
"was approved, deleted or rejected. "
"Please search for entry '%s' seq# %s and reenter your changes "
"if they are still applicable." %
(KW.SRC[ent.src].kw, entr.seq))
return
pentr = pentr[0]
jdb.augment_xrefs(dbh, raw['xref'])
if entr.stat == KW.STAT['D'].id:
# If this is a deletion, set $merge_rev. When passed
# to function merge_hist() it will tell it to return the
# edited entry's parent, rather than the edited entry
# itself. The reason is that if we are doing a delete,
# we do not want to make any changes to the entry, even
# if the submitter has done so.
merge_rev = True
# Merge_hist() will combine the history entry in the submitted
# entry with the all the previous history records in the
# parent entry, so the the new entry will have a continuous
# history. In the process it checks that the parent entry
# exists -- it might not if someone else has approved a
# different edit in the meantime.
# merge_hist also returns an entry. If 'merge_rev' is false,
# the entry returned is 'entr'. If 'merge_rev' is true,
# the entry returned is the entr pointed to by 'entr.dfrm'
# (i.e. the original entry that the submitter edited.)
# This is done when a delete is requested and we want to
# ignore any edits the submitter may have made (which 'entr'
# will contain.)
# Before calling merge_hist() check for a condition that would
# cause merge_hist() to fail.
if entr.stat == KW.STAT['D'].id and not getattr(entr, 'dfrm', None):
L('cgi.edsubmit.submission').debug("delete of new entry error")
errs.append("Delete requested but this is a new entry.")
if disp == 'a' and has_xrslv(entr) and entr.stat == KW.STAT['A'].id:
L('cgi.edsubmit.submission').debug("unresolved xrefs error")
errs.append("Can't approve because entry has unresolved xrefs")
if not errs:
# If this is a submission by a non-editor, restore the
# original entry's freq items which non-editors are not
# allowed to change.
if not is_editor:
if pentr:
L('cgi.edsubmit.submission').debug("copying freqs from parent")
jdb.copy_freqs(pentr, entr)
# Note that non-editors can provide freq items on new
# entries. We expect an editor to vet this when approving.
# Entr contains the hist record generate by the edconf.py
# but it is not trustworthy since it could be modified or
# created from scratch before we get it. So we extract
# the unvalidated info from it (name, email, notes, refs)
# and recreate it.
h = entr._hist[-1]
# When we get here, if merge_rev is true, pentr will also be
# true. If we are wrong, add_hist() will throw an exception
# but will never return a None, so no need to check return val.
L('cgi.edsubmit.submission').debug("adding hist for '%s', merge=%s" %
(h.name, merge_rev))
entr = jdb.add_hist(entr, pentr, userid, h.name, h.email, h.notes,
h.refs, merge_rev)
if not errs:
# Occasionally, often from copy-pasting, a unicode BOM
# character finds its way into one of an entry's text
# strings. We quietly remove any here.
n = jdb.bom_fixall(entr)
if n > 0:
L('cgi.edsubmit.submission').debug("removed %s BOM character(s)" %
n)
if not errs:
if not disp:
added = submit(dbh, entr, edtree, errs)
elif disp == "a":
added = approve(dbh, entr, edtree, errs)
elif disp == "r":
added = reject(dbh, entr, edtree, errs, None)
else:
L('cgi.edsubmit.submission').debug("bad url parameter (disp=%s)" %
disp)
errs.append("Bad url parameter (disp=%s)" % disp)
L('cgi.edsubmit.submission').debug("seqset: %s" %
logseq(dbh, entr.seq, entr.src))
if not errs: return added
# Note that changes have not been committed yet, caller is
# expected to do that.
return None
def main(args, opts):
global Svc, Sid
jdb.reset_encoding(sys.stdout, 'utf-8')
errs = []
dbh = svc = None
try:
form, svc, dbg, dbh, sid, sess, parms, cfg = jmcgi.parseform()
except ValueError as e:
jmcgi.err_page([str(e)])
# Svc and Sid are used in function url() and are global in
# in order to avoid having to pass them through several layers
# of function calls.
Svc, Sid = svc, sid
L('cgi.edsubmit').debug("started: userid=%s, sid=%s" %
(sess and sess.userid, sess and sess.id))
fv = form.getfirst
# disp values: '': User submission, 'a': Approve. 'r': Reject;
disp = fv('disp') or ''
if not sess and disp:
errs.append("Only registered editors can approve or reject entries")
if errs: jmcgi.err_page(errs)
try:
entrs = serialize.unserialize(fv("entr"))
except Exception:
jmcgi.err_page(["Bad 'entr' parameter, unable to unserialize."])
added = []
# Clear any possible transactions begun elsewhere (e.g. by the
# keyword table read in jdb.dbOpen()). Failure to do this will
# cause the following START TRANSACTON command to fail with:
# InternalError: SET TRANSACTION ISOLATION LEVEL must be
# called before any query
L('cgi.edsubmit.main').debug("starting transaction")
dbh.connection.rollback()
dbh.execute("START TRANSACTION ISOLATION LEVEL SERIALIZABLE")
# FIXME: we unserialize the entr's xref's as they were resolved
# by the edconf.py page. Should we check them again here?
# If target entry was deleted in meantime, attempt to add
# our entr to db will fail with obscure foreign key error.
# Alternatively an edited version of target may have been
# created which wont have our xref pointing to it as it should.
for entr in entrs:
# FIXME: submission() can raise a psycopg2
# TransactionRollbackError if there is a serialization
# error resulting from a concurrent update. Detecting
# such a condition is why run with serializable isolation
# level. We need to trap it and present some sensible
# error message.
e = submission(dbh, entr, disp, errs, jmcgi.is_editor(sess),
sess.userid if sess else None)
# The value returned by submission() is a 3-tuple consisting
# of (id, seq, src) for the added entry.
if e: added.append(e)
if errs:
L('cgi.edsubmit.main').info("rolling back transaction due to errors")
dbh.connection.rollback()
jmcgi.err_page(errs)
else:
L('cgi.edsubmit.main').info("doing commit")
dbh.connection.commit()
jmcgi.jinja_page("submitted.jinja",
added=added,
parms=parms,
svc=svc,
dbg=dbg,
sid=sid,
session=sess,
cfg=cfg,
this_page='edsubmit.py')
L('cgi.edsubmit.main').debug("thank you page sent, exiting normally")
def main(args, opts):
jdb.reset_encoding(sys.stdout, 'utf-8')
errs = []
so = None
stats = {}
try:
form, svc, dbg, cur, sid, sess, parms, cfg = jmcgi.parseform()
except Exception as e:
jmcgi.err_page([str(e)])
fv = form.getfirst
fl = form.getlist
# The form values "userid", "fullname", "email", "priv",
# "disabled", come from editable fields in the user.py
# page.
# The values "new" (i.e. create) and "delete" are also user
# boolean input from user.py indicating what the user wants
# to do. Neither of them set indicates an update action on
# an existing account. Both of them set is an error.
# The form value "subjid" identifies the user whose data was
# originally loaded into the user.py form and is the user any
# update or delete will be performed on. If it differs from
# sess.userid it indcates the action is to e done on the
# account of someone other than the logged user themself
# and is prohibited unless the logged in user has "admin"
# privilege. For a new (create) action, "subjid" is ignored
# and the new user is created with the id given in "userid".
# Set 'action' to "n" (new), "u" (update), "d" (delete) or
# "x" (invalid) according to the values of fv('new') and
# fv('delete') that were received from as url parameters
# from the User form.
action = {(0,0):'u', (0,1):'d', (1,0):'n', (1,1):'x'}\
[(bool(fv('new')),bool(fv('delete')))]
L('cgi.userupd').debug(
"new=%r, delete=%r, action=%r, subjid=%r, userid=%r" %
(fv('new'), fv('delete'), action, fv('subjid'), fv('userid')))
# NOTE: The jmcgi.err_page() calls below do not return,
# jmcgi.err_page() calls sys.exit().
if not sess:
jmcgi.err_page([],
prolog="<p>You must login before you can change your "
"user settings.</p>")
if fv('subjid') != sess.userid and sess.priv != 'A':
jmcgi.err_page(
[],
prolog="<p>You do not have sufficient privilege to alter "
"settings for anyone other than yourself.</p>")
if action in ('nd') and sess.priv != 'A':
jmcgi.err_page(
[],
prolog="<p>You do not have sufficient privilege to create "
"or delete users.</p>")
if action == 'x':
jmcgi.err_page(
[], prolog="<p>\"New user\" and \"Delete\" are incompatible.</p>")
errors = []
# Get the id of the user we will be updating. If creating a
# new user, 'subjid' should not exist and thus 'subj' will be
# None which has the beneficial effect of causing gen_sql_-
# params() to generate change parameters for every form value
# which is what we want when creating a user.
subj = jmcgi.get_user(fv('subjid'), svc, cfg)
if action in 'nu': # "new" or "update" action...
if action == 'u':
L('cgi.userupd').debug("update user %r" % sanitize_o(subj))
else:
L('cgi.userupd').debug("create user %r" % fv('userid'))
if action == 'n' and \
(subj or fv('userid')==sess.userid
or jmcgi.get_user(fv('userid'), svc, cfg)):
# This is the creation of a new user (fv('userid')).
# The userid must not already exist. The tests for
# subj and sess.userid are simply to avoid an expensive
# get_user() call when we already know the user exists.
errors.append("Account name %s is already in use." % fv('userid'))
if action == 'u' and fv('userid')!=subj.userid \
and (fv('userid')==sess.userid \
or jmcgi.get_user(fv('userid'), svc, cfg)):
# This is an update of an existing user.
# If the new userid (fv('userid')) is the same as the
# subj.userid it's not being changed and is ok. If
# different then it must not be the same as an exiting
# userid. The test for sess.userid is simply to avoid
# an expensive get_user() call when we already know
# that user exists.
errors.append("Account name %s is already in use." % fv('userid'))
# Get the parameters we'll need for the sql statement used
# to update the user/sessions database.
collist, values, err \
= gen_sql_params (sess.priv=='A', subj, fv('pw1'), fv('pw2'),
fv('userid'), fv('fullname'), fv('email'),
fv('priv'), fv('disabled'))
errors.extend(err)
L('cgi.userupd').debug("collist: %r" % collist)
L('cgi.userupd').debug("values: %r" % sanitize_v(values, collist))
else: # "delete" action...
# We ignore changes made to the form fields since we
# are going to delete the user, they are irrelevant.
# Except for one: the "userid" field. If that was
# changed we treat it as an error due to the risk that
# the user thought the changed userid will be deleted
# which is not what will happen (we delete the "subjid"
# user.)
values = []
if fv('userid') != fv('subjid'):
errors.append("Can't delete user when userid has been changed.")
if not subj:
errors.append("User '%s' doesn't exist." % fv('subjid'))
if errors:
jmcgi.err_page(
errs=errors,
prolog="The following errors were found in the changes "
"you requested. Please use your browser's Back button "
"to return to the user page, correct them, and resubmit "
"your changes.")
update_session = None
result = None
summary = 'not available'
if action == 'n': # Create new user...
cols = ','.join(c for c, p in collist)
pmarks = ','.join(p for c, p in collist)
sql = "INSERT INTO users(%s) VALUES (%s)" % (cols, pmarks)
values_sani = sanitize_v(values, collist)
summary = "added user \"%s\"" \
% ''.join(p for c,p in collist if c=='userid')
elif action == 'd': # Delete existing user...
sql = "DELETE FROM users WHERE userid=%s"
values.append(fv('subjid'))
values_sani = values
summary = "deleted user \"%s\"" % fv('subjid')
else: # Update existing user...
if not collist: result = 'nochange'
else:
if subj and subj.userid == sess.userid \
and fv('userid') and fv('userid'):
update_session = fv('userid')
updclause = ','.join(("%s=%s" % (c, p)) for c, p in collist)
sql = "UPDATE users SET %s WHERE userid=%%s" % updclause
values.append(fv('subjid'))
values_sani = sanitize_v(values, collist)
summary = "updated user %s (%s)" \
% (fv('subjid'), ','.join([c for c,p in collist]))
if result != 'nochange':
sesscur = jdb.dbOpenSvc(cfg, svc, session=True, nokw=True)
L('cgi.userupd.db').debug("sql: %r" % sql)
# 'values_sani' should be the same as values but with any
# password text masked out.
L('cgi.userupd.db').debug("args: %r" % values_sani)
sesscur.execute(sql, values)
sesscur.connection.commit()
#FIXME: trap db errors and try to figure out what went
# wrong in terms that a user can remediate.
if update_session:
L('cgi.userupd').debug("update sess.userid: %r->%r" %
(sess.userid, update_session))
sess.userid = update_session
L('cgi.userupd').info(summary)
result = 'success'
# If the user is not an admin we send them back to their
# settings page (using 'userid' since it they may have changed
# it.) For admin users it's more complcatd because they might
# have deleted the user. Easiest for us to send him/her back
# to the user list page which we know always exists.
if sess.priv == 'A': return_to = 'users.py?'
else:
return_to = 'user.py?u=%s' % fv('userid')
jmcgi.redirect(urlbase() + "%s&svc=%s&sid=%s&dbg=%s&result=%s" %
(return_to, svc, sid, dbg, result))
def gen_sql_params(is_admin, subj, pw1, pw2, userid, fullname, email, priv,
disabled):
"""
Compares the form values received from the User form
with the values in 'subj' to determine which have been
changed. For user creation ("new" action) there is no
'subj' so any non-null form values will be seen as changes.
If the action is delete, this function need not even be
called since the only info needed for deletion is the
user id.
Return values:
collist -- A list of 2-tuples each consisting of
column name -- Name of the changed field/
parameter -- This is either a psycopg2 paramater marker
("%s") or a Postgresql expression containing the same.
values -- A list of the same length as 'collist' containing
the values the will be substituted for the parameter markers
in collist be Postgresql.
errors -- A list of errors discovered when generating 'collist'
and 'values'. If this list in non-empty, 'collist' and
'values' should be ignored and the errors should be displayed
to the user.
"""
new = not subj
# Convert from the "priv" strings we receive from the user.jinja
# form to the single letter keys used in the user database.
priv = {'admin': 'A', 'editor': 'E', 'user': None}.get(priv)
collist = []
values = []
errors = []
if pw1 or pw2:
if pw1 != pw2:
errors.append("Password mismatch. Please reenter "
"your passwords.")
else:
collist.append(('pw', "crypt(%s, gen_salt('bf'))"))
values.append(pw1)
else:
if not subj: # 'not subj' is a proxy for action=='n'
errors.append("Password is required for new users.")
if new or userid != subj.userid:
L('cgi.userupd').debug("userid change: %r" % userid)
# Max length of 16 is enforced in database DDL.
if not re.match(r'[a-zA-Z][a-zA-Z0-9]{2,15}$', userid or ''):
errors.append('Invalid "userid" value')
else:
collist.append(('userid', "%s"))
values.append(userid)
if new or fullname != subj.fullname:
L('cgi.userupd').debug("fullname change: %r" % fullname)
if fullname and len(fullname) > 120:
# This limit is not enforced in database DDL but is
# applied here as a sanity/maleficence check.
errors.append("Full name is too long, max of 120 "
"characters please.")
collist.append(('fullname', "%s"))
values.append(fullname)
if new or email != subj.email:
L('cgi.userupd').debug("email change: %r" % email)
if email and '@' not in (email):
errors.append('Email addresses must include an "@" ' 'character.')
elif email and len(email) > 120:
# This limit is not enforced in database DDL but is
# applied here as a sanity/maleficence check.
errors.append("Email address too long, max of 120 "
"characters please.")
else:
collist.append(('email', "%s"))
values.append(email)
if is_admin:
# Only if the script executor is an admin user do we
# allow changing the account's priv or disabled status.
if new or priv != subj.priv:
L('cgi.userupd').debug("priv change: %r" % priv)
collist.append(('priv', "%s"))
values.append(priv)
if new or bool(disabled) != subj.disabled:
L('cgi.userupd').debug("disabled change: %r" % bool(disabled))
collist.append(('disabled', "%s"))
values.append(bool(disabled))
return collist, values, errors
def get_entr_cands(dbconn,
xref_src,
targ_src,
start=None,
stop=None,
active=False):
# Yield sets of candidates rows for all unresolved xrefs
# for a single entry.
#
# dbconn -- An open dbapi connection to a jmdictdb database.
# xref_src -- A 2-tuple consiting of:
# 0: a sequence of numbers identifying the entr.src numbers
# to be included or excluded when searching for xrefs to
# resolve.
# 1: (bool) if false the values in item 0 specify src numbers
# to include; if false they are numbers to exclude.
# targ_src -- Same format as 'xref_src' but for target entry
# .src numbers.
# start -- Lowest entry id number to process. If None of not
# given default is 1.
# stop -- Process entries up to but not including this id number.
# active -- (bool) If true do not process any unresolved xrefs
# that are attached to deleted, rejected, or unapproved entries
# (i.e. restrict processing to active, approved entries).
# If false, such unresolved xrefs will be processed along with
# the active, approved ones.
## Following parameter is currently ignored, see comments below.
## appr_targ -- (bool) if true restrict target candidate entries
## to those that are approved. If false, unapproved entries
## will also be target candidates.
L().info("reading candidates from database, may take a while...")
#FIXME: this replicates same WHERE logic used in del_xresolv();
# should factor out to a single location, either in python
# code or in a database view.
c1, args1 = src_clause(xref_src, targ_src)
c2, args2 = idrange_clause(start, stop)
#FIXME: should not hardwire 'stat' value.
c3 = "v.stat=2 AND NOT v.unapp" if active else ""
# Disallow resolution to deleted or rejected entries.
#FIXME: should not hardwire 'stat' value.
c4 = "tstat=2"
#FIXME: following condition controls whether or not unapproved
# xrefs are considered as target candidates. It is disabled
# because it is probably the wrong thing to do right now. If an
# xref is resolved to an unapproved entry, the target unapproved
# entry will not get the reverse xref and if it is approved the
# resulting new active entry will not have the reverse xref.
# We probably need to generate xrefs to all the unapproved entries
# as well as the approved active one but that means restoring a
# flavor of the "one unresolved -> multiple xrefs" capability
# that was removed in hg-180615-3e9e9c.
# Without this condition unapproved edits will result in
# resolution failing due to multiple targets forcing manual
# intervention which seems the best option at the moment.
c5 = "" #"NOT tunap" if appr_targ else "")
whr = " AND ".join([c for c in [c1, c2, c3, c4, c5] if c])
if whr: whr = "WHERE " + whr
args = tuple(args1 + args2)
sql = "SELECT * FROM rslv v %s"\
" ORDER BY entr,sens,typ,ord,targ" % whr
L().debug("sql: %s" % sql)
L().debug("args: %r" % (args, ))
rs = db.query(dbconn, sql, args)
L().info("read %d candidates from database" % len(rs))
key = lambda x: (x.entr, x.sens, x.typ, x.ord)
key = lambda x: x.entr
for _, rows in itertools.groupby(rs, key=key):
yield list(rows)
def parseform(readonly=False):
"""\
Do some routine tasks that are needed for (most) every page,
specifically:
* Call cgi.FieldStorage to parse parameters.
* Extract the svc parameter, validate it, and open the requested database.
* Get session id and handle log or logout requests.
Return an 8-tuple of:
form (cgi.FieldStorage obj)
svc (string) -- Checked svc value.
dbg (int) -- true if "dbg" url param has true value, else 0.
cur (dbapi cursor) -- Open cursor for database defined by 'svc'.
sid (string) -- session.id in hexidecimal form or "".
sess (Session inst.) -- Session for sid or None.
params (dict) -- Received and decoded form parameters.
cfg (Config inst.) -- Config object from reading config.ini.
"""
cfg = initcgi("config.ini") # Read config, initialize logging.
L('cgi.jmcgi').debug("parseform: called in %s" %
sys.modules['__main__'].__file__)
errs = []
sess = None
sid = ''
cur = None
svc = None
def_svc = cfg['web'].get('DEFAULT_SVC', 'jmdict')
if def_svc.startswith('db_'): def_svc = def_svc[3:]
check_server_status(cfg.get('web', 'STATUS_DIR'))
form = cgi.FieldStorage()
L('cgi.jmcgi').debug("query_string: %s" % os.environ.get('QUERY_STRING'))
for k in form.keys():
v = ['***']*len(form.getlist(k)) if k in ('password','pw1','pw2') \
else form.getlist(k)
L('cgi.jmcgi').debug("parseform: form key %s=%r" % (k, v))
dbg = int(form.getfirst('dbg') or '0')
svc = form.getfirst('svc') or def_svc
#L('cgi.jmcgi').debug("parseform: svc=%s" % svc)
usid = form.getfirst('sid') or '' # No SID is "", not None.
try:
svc = safe(svc)
except ValueError:
errs.append('svc=' + svc)
if not errs:
try:
cur = jdb.dbOpenSvc(cfg, svc)
except KeyError as e:
errs.append('svc=' + str(e))
if errs: raise ValueError(';'.join(errs))
# Login, logout, and session identification...
scur = jdb.dbOpenSvc(cfg, svc, session=True, nokw=True)
action = form.getfirst('loginout') # Will be None, "login" or "logout"
sid = get_sid_from_cookie() or ''
sid_from_cookie = bool(sid)
if usid: sid = usid # Use sid from url if available.
L('cgi.jmcgi').debug("parseform: sid=%s, from_cookie=%s, action=%s" %
(sid, sid_from_cookie, action))
uname = form.getfirst('username') or ''
pw = form.getfirst('password') or ''
sid, sess = get_session(scur, action, sid, uname, pw)
L('cgi.jmcgi').debug("parseform: %s session, sid=%s" %
("got" if sess else "no", sid))
if sid: set_sid_cookie(sid, delete=(action == "logout"))
if sid_from_cookie: sid = ''
scur.connection.close()
# Collect the form parameters. Caller is expected to pass
# them to the page template which will use them in the login
# section as hidden parms so the page can be recreated after
# a login. We leave out any param name "result" since that
# is used as a confirmation message by the userupd.py page
# and best to see it only once.
parms = [(k, v) for k in list(form.keys())
if k not in ('loginout', 'username', 'password', 'result')
for v in form.getlist(k)]
return form, svc, dbg, cur, sid, sess, parms, cfg
