def fitX(func, target, *funcArgs, **funcKwargs):
"""
e.g. fitX(func=kt2e0, target=77.338912, ax=ax, bx=bx) == 5.
"""
def curryFuncWithTarget(func, target, *funcArgs, **funcKwargs):
argsP = funcArgs
kwargsP = funcKwargs
def f(x):
return (func(x, *argsP, **kwargsP) - target)**2
return f
cf = curryFuncWithTarget(func, target, *funcArgs, **funcKwargs)
(out, infodict, ier, mesg) = [None] * 4
try:
(out, infodict, ier,
mesg) = scipy.optimize.fsolve(func=cf,
x0=((2 * S.rand()) - 0.5),
full_output=1,
xtol=1.0e-04)
if ier == 1:
lcfitlogger.debug("Successful fsolve\n")
else:
lcfitlogger.warning("Problem with fsolve. ier = %s, mesg = \"%s.\"\n %s\n" % \
(ier, mesg.replace('\n', ' '), infodict))
except Exception, e:
lcfitlogger.error(
"Exception with fsolve: \"%s\". Catching and ignoring..." % e)
def __call__(self, req):
req.session = Session(req)
if req.session.has_key(LCFIT_SESSION_KEY):
self.lcdb.logout(req.session[LCFIT_SESSION_KEY])
lcfitlogger.debug( 'LcLogout: __call__. Logging out session key: %s' % req.session[LCFIT_SESSION_KEY])
req.session.invalidate()
lcfitlogger.warning( 'LcLogout: __call__. Logging out unkown session.')
util.redirect(req, self.redirectTarget)
def __call__(self, req):
req.session = Session(req)
if req.session.has_key(LCFIT_SESSION_KEY):
self.lcdb.logout(req.session[LCFIT_SESSION_KEY])
lcfitlogger.debug(
'LcLogout: __call__. Logging out session key: %s' %
req.session[LCFIT_SESSION_KEY])
req.session.invalidate()
lcfitlogger.warning(
'LcLogout: __call__. Logging out unkown session.')
util.redirect(req, self.redirectTarget)
def try_execute(dbcon, sql, data=None, fetch_N='all', tries_N=3): """ Do the execute, trying to reconnect a few times. Pass a db connection back, either the old one or a new reconnected ref.""" ## TODO incorporate lists of SQL statements and data in order to ## transactionalize them. ... hm ... But what if you want ## intermediate data ... hmm. Maybe a reconnecting cursor is what ## I want? So then I don't have a bunch of duplicated connection ## lines. But what happens if we disconnect or have to roll ## back.... Hmm... I am leaving this as-is, because it should be ## fine for the little application that is LCFIT, but the ## interface of multiple SQL statements to the DB, handling ## commits and rollbacks, seems to be a thing I haven't completely ## figured out. Perhaps the best approach is to have complex ## updates/ inserts/ deletions as a server side function, with ## pure non-transactionally complex selects left explicit. tries_comp = 0 while tries_N >= 0: try: curs = dbcon.cursor() #raise Exception(sql + ' xxx ' + repr(data)) curs.execute(sql, data) if fetch_N == 'all': res = curs.fetchall() elif fetch_N == 'one': res = curs.fetchone() elif fetch_N == 'none': res = None else: raise Exception, "Bad value for fetch_N: %s" % fetch_N curs.close() dbcon.commit() return (res,dbcon) except (psycopg2.OperationalError, psycopg2.InterfaceError, psycopg2.InternalError), e: lcfitlogger.error( "Error trying to execute query, reconnecting: \"%s\", \"%s\"." % (sql, e)) tries_N -= 1 time.sleep(2**tries_comp) tries_comp += 1 try: dbcon = psycopg2.connect(dbcon.dsn) except Exception, e: raise LcException, "Exception trying to connect: \"%s\"." % str(e) lcfitlogger.warning( "Successfully reconnected, re-executed cursor.")
def fitX(func, target, *funcArgs, **funcKwargs):
"""
e.g. fitX(func=kt2e0, target=77.338912, ax=ax, bx=bx) == 5.
"""
def curryFuncWithTarget(func, target, *funcArgs, **funcKwargs):
argsP = funcArgs
kwargsP = funcKwargs
def f(x):
return (func(x, *argsP, **kwargsP) - target)**2
return f
cf = curryFuncWithTarget(func, target, *funcArgs, **funcKwargs)
(out, infodict, ier, mesg) = [None]*4
try:
(out, infodict, ier, mesg) = scipy.optimize.fsolve(
func=cf, x0=((2*S.rand())-0.5), full_output=1, xtol=1.0e-04)
if ier == 1:
lcfitlogger.debug( "Successful fsolve\n")
else:
lcfitlogger.warning("Problem with fsolve. ier = %s, mesg = \"%s.\"\n %s\n" % \
(ier, mesg.replace('\n', ' '), infodict))
except Exception, e:
lcfitlogger.error( "Exception with fsolve: \"%s\". Catching and ignoring..." % e)
def __call__(self, session=None, form=None):
"""Handle the page"""
if self._hasNecessaryData(session, form):
# ... if authorized, use the returned ID, set session, and redirect
authId = self._hasValidAuth(session, form)
if type(authId) is types.IntType:
if session is None:
session = Cookie.SimpleCookie()
session[LCFIT_SESSION_KEY] = authId
headers = "Status: 303\nLocation: %s\n\n" % self.redirectTarget
sys.stdout.write(session.output() + "\n")
sys.stdout.write(headers)
sys.stdout.flush()
lcfitlogger.info( 'LcLoginProcess: __call__. Good auth. Auth id: %s, username: %s.' \
% (authId, form.getvalue(LCFIT_USERNAME_KEY, 'XX')))
return
else:
lcfitlogger.warning('LcLoginProcess: __call__. Bad auth.')
raise LcException, "Could not authorize. Go \"back\" and try again."
return (0)
else:
lcfitlogger.warning('LcLoginProcess: __call__. Incomplete data.')
raise LcException, "Incomplete Login Information. Go \"back\" and try again."
return (0)
def __call__(self, session=None, form=None):
"""Handle the page"""
if self._hasNecessaryData(session, form):
# ... if authorized, use the returned ID, set session, and redirect
authId = self._hasValidAuth(session, form)
if type(authId) is types.IntType:
if session is None:
session = Cookie.SimpleCookie()
session[LCFIT_SESSION_KEY] = authId
headers = "Status: 303\nLocation: %s\n\n" % self.redirectTarget
sys.stdout.write(session.output() + "\n")
sys.stdout.write(headers)
sys.stdout.flush()
lcfitlogger.info( 'LcLoginProcess: __call__. Good auth. Auth id: %s, username: %s.' \
% (authId, form.getvalue(LCFIT_USERNAME_KEY, 'XX')))
return
else:
lcfitlogger.warning( 'LcLoginProcess: __call__. Bad auth.')
raise LcException, "Could not authorize. Go \"back\" and try again."
return(0)
else:
lcfitlogger.warning( 'LcLoginProcess: __call__. Incomplete data.')
raise LcException, "Incomplete Login Information. Go \"back\" and try again."
return(0)
def adjustAx(mx, gender='combined'):
numAgeWidths = LCFIT_DEFAULT_NO_AGEWIDTHS
# Going to let qx[2] float as 2.6 since can't go to a previous x
ax_new = N.array([0.0] * numAgeWidths, N.float64)
ax_old = N.array([0.0] * numAgeWidths, N.float64)
qx = N.array([0.0] * numAgeWidths, N.float64)
dx = N.array([0.0] * numAgeWidths, N.float64)
lx = N.array([0.0] * numAgeWidths, N.float64)
dx_old = N.array([0.0] * numAgeWidths, N.float64)
dx = N.array([0.0] * numAgeWidths, N.float64)
# estimate ax as 2.5, store as ax_old
if gender == 'combined':
ax_new[0] = (0.045 + 0.053) / 2 + (
(2.684 + 2.800) /
2) * mx[0] # punt -- average m, f from "Demography" p48
ax_new[1] = (1.651 + 1.522) / 2 + (
(2.816 + 1.518) / 2) * mx[0] # punt -- ditto...
elif gender == 'male':
ax_new[0] = 0.045 + 2.684 * mx[0]
ax_new[1] = 1.651 + 2.816 * mx[0]
elif gender == 'female':
ax_new[0] = 0.053 + 2.800 * mx[0]
ax_new[1] = 1.522 + 1.518 * mx[0]
ax_new[2:17] = 5.0 * N.array([
.46, .54, .57, .49, .50, .52, .54, .54, .54, .53, .52, .52, .52, .51,
.51
])
ax_new[18:-2] = N.array(
[2.27, 2.19, 2.08,
1.95]) # Australia 1927. 1966: 2.31 2.23 1.88 1.63 1.41 1.35
ax_new[
-2] = 1.6 # Averaging and eyeballing Rus, Jap, US begining and end of data
ax_new[-1] = 1.6 # ditto
#lcfitlogger.debug( "%s" % ax_new)
"""
AgeF-USM-USF-JM-JF-RusM-Rus
start 105 1.60 1.54 1.24 1.21 1.81 1.79 avg=1.531666667
end 105 1.63 1.65 1.61 1.52 1.36 1.41 avg=1.53
start 110 1.69 1.61 1.20 1.17 2.21 2.21 avg=1.681666667
end 110 1.61 1.70 1.53 1.47 1.29 1.38 avg=1.496666667
From HMD
"""
iter_repeats = 1
while iter_repeats <= 10:
# ** calculate qx from mx and ax
qx[0] = mx[0] / (1 + (1 - ax_old[0]) * mx[0])
qx[1] = mx[1] / (1 + (4 - ax_old[1]) * mx[1])
qx[2:-1] = (5 * mx[2:-1]) / (1 + (5 - ax_old[2:-1]) * mx[2:-1])
qx[-1] = 1.0
qx[qx >= 1.0] = 1.0 # Sanity
qx[qx <= 0.0] = 0.0 # Sanity
# calculate lx from qx
lx[0] = 1
lx[1:] = N.cumprod(1.0 - qx[0:-1])
# estimate dx
dx = -N.diff(N.concatenate((lx, [0.0])))
dx[dx <= 0.0] = 0.00001 # XXX Otherwise we get inf's
# use polynomial thing to estimate ax_new
ax_old[...] = ax_new[...]
for i in range(2, len(ax_old) - 2):
ax_new[i] = ((-(5.0 / 24.0) * dx[i - 1]) + (2.5 * dx[i]) +
((5.0 / 24.0) * dx[i + 1])) / dx[i]
pass
# find difference between ax_new and ax_old
ax_diff = ax_old - ax_new
ax_distance = N.sum(ax_diff**2)
# if difference is small return ax_new
if ax_distance < 0.1:
if True:
pass
#lcfitlogger.debug( "\n\nSuccess!\nax iteration (%s): %s\n" % (iter_repeats, ax_new))
return (ax_new, qx, dx, lx)
else:
# How many times around?
iter_repeats += 1
ax_old[...] = ax_new[...]
dx_old[...] = dx[...]
# check for reasonable ax
if (ax_new < 0.001).any(
): # Seems that this should never be true yet it works without it....
#raise Exception("ax lt 0.001. iter = %s <br> \nmx: %s<br>\nax: %s" % (iter_repeats, mx, ax_new))
pass
pass
# All done, looping again
pass
"""
# OLD CODE. spine, CHECKING FOR between 0 and 1 and adjusting if not
nqx[0] = nmx[0] / (1 + (qx0_intercept - qx0_slope * nmx[0]) * nmx[0]) # 0-1 age .93, 1.7. based on lfexpt.m
nqx[1] = 4*nmx[1]/(1+(LCFIT_DEFAULT_AGEWIDTH/2)*nmx[1]) # 1-4
"""
lcfitlogger.warning(
"\n\nFailure!\nax iteration (%s): \n%s\n" %
(iter_repeats, pprint.pformat((ax_new, qx, mx, lx), width=10)))
raise LcException('ax not settling down. <br>ax_distance: %s, iter_repeats: %s<br>ax_dif: %s<br>new: %s<br>old: %s <br>lx: %s <br>qx: %s<br>mx: %s<br>' % \
(ax_distance, iter_repeats, ax_diff, ax_new, ax_old, lx, qx, mx))
def adjustAx(mx, gender='combined'):
numAgeWidths = LCFIT_DEFAULT_NO_AGEWIDTHS
# Going to let qx[2] float as 2.6 since can't go to a previous x
ax_new = N.array([0.0] * numAgeWidths, N.float64)
ax_old = N.array([0.0] * numAgeWidths, N.float64)
qx = N.array([0.0] * numAgeWidths, N.float64)
dx = N.array([0.0] * numAgeWidths, N.float64)
lx = N.array([0.0] * numAgeWidths, N.float64)
dx_old = N.array([0.0] * numAgeWidths, N.float64)
dx = N.array([0.0] * numAgeWidths, N.float64)
# estimate ax as 2.5, store as ax_old
if gender == 'combined':
ax_new[0] = (0.045 + 0.053)/2 + ((2.684 + 2.800)/2)*mx[0] # punt -- average m, f from "Demography" p48
ax_new[1] = (1.651 + 1.522)/2 + ((2.816 + 1.518)/2)*mx[0] # punt -- ditto...
elif gender == 'male':
ax_new[0] = 0.045 + 2.684 * mx[0]
ax_new[1] = 1.651 + 2.816 * mx[0]
elif gender == 'female':
ax_new[0] = 0.053 + 2.800 * mx[0]
ax_new[1] = 1.522 + 1.518 * mx[0]
ax_new[2:17] = 5.0 * N.array([.46, .54, .57, .49, .50, .52, .54, .54, .54, .53, .52, .52, .52, .51, .51])
ax_new[18:-2] = N.array([2.27, 2.19, 2.08, 1.95]) # Australia 1927. 1966: 2.31 2.23 1.88 1.63 1.41 1.35
ax_new[-2] = 1.6 # Averaging and eyeballing Rus, Jap, US begining and end of data
ax_new[-1] = 1.6 # ditto
#lcfitlogger.debug( "%s" % ax_new)
"""
AgeF-USM-USF-JM-JF-RusM-Rus
start 105 1.60 1.54 1.24 1.21 1.81 1.79 avg=1.531666667
end 105 1.63 1.65 1.61 1.52 1.36 1.41 avg=1.53
start 110 1.69 1.61 1.20 1.17 2.21 2.21 avg=1.681666667
end 110 1.61 1.70 1.53 1.47 1.29 1.38 avg=1.496666667
From HMD
"""
iter_repeats = 1
while iter_repeats <= 10:
# ** calculate qx from mx and ax
qx[0] = mx[0] / (1+(1-ax_old[0])*mx[0])
qx[1] = mx[1] / (1+(4-ax_old[1])*mx[1])
qx[2:-1] = (5*mx[2:-1]) / (1 + (5-ax_old[2:-1])*mx[2:-1])
qx[-1] = 1.0
qx[qx >= 1.0] = 1.0 # Sanity
qx[qx <= 0.0] = 0.0 # Sanity
# calculate lx from qx
lx[0] = 1
lx[1:] = N.cumprod(1.0-qx[0:-1])
# estimate dx
dx = -N.diff(N.concatenate((lx, [0.0])))
dx[dx<=0.0] = 0.00001 # XXX Otherwise we get inf's
# use polynomial thing to estimate ax_new
ax_old[...] = ax_new[...]
for i in range(2, len(ax_old)-2):
ax_new[i] = ( (-(5.0/24.0)*dx[i-1]) + (2.5*dx[i]) + ((5.0/24.0)*dx[i+1]) )/dx[i]
pass
# find difference between ax_new and ax_old
ax_diff = ax_old - ax_new
ax_distance = N.sum(ax_diff**2)
# if difference is small return ax_new
if ax_distance < 0.1:
if True:
pass
#lcfitlogger.debug( "\n\nSuccess!\nax iteration (%s): %s\n" % (iter_repeats, ax_new))
return (ax_new, qx, dx, lx)
else:
# How many times around?
iter_repeats += 1
ax_old[...] = ax_new[...]
dx_old[...] = dx[...]
# check for reasonable ax
if (ax_new < 0.001).any(): # Seems that this should never be true yet it works without it....
#raise Exception("ax lt 0.001. iter = %s <br> \nmx: %s<br>\nax: %s" % (iter_repeats, mx, ax_new))
pass
pass
# All done, looping again
pass
"""
# OLD CODE. spine, CHECKING FOR between 0 and 1 and adjusting if not
nqx[0] = nmx[0] / (1 + (qx0_intercept - qx0_slope * nmx[0]) * nmx[0]) # 0-1 age .93, 1.7. based on lfexpt.m
nqx[1] = 4*nmx[1]/(1+(LCFIT_DEFAULT_AGEWIDTH/2)*nmx[1]) # 1-4
"""
lcfitlogger.warning("\n\nFailure!\nax iteration (%s): \n%s\n" % (iter_repeats, pprint.pformat((ax_new, qx, mx, lx), width=10)))
raise LcException('ax not settling down. <br>ax_distance: %s, iter_repeats: %s<br>ax_dif: %s<br>new: %s<br>old: %s <br>lx: %s <br>qx: %s<br>mx: %s<br>' % \
(ax_distance, iter_repeats, ax_diff, ax_new, ax_old, lx, qx, mx))
