def optimal_random_effect(fixed_effect):
#
# set start_var value for the fixed effects
var_id = node_name2var_id['world']
sql_cmd = 'UPDATE start_var SET start_var_value = ' + str(fixed_effect)
sql_cmd += ' WHERE start_var_id == ' + str(var_id)
new = False
connection = dismod_at.create_connection(file_name, new)
dismod_at.sql_command(connection, sql_cmd)
connection.close()
#
# optimize the random effects
dismod_at.system_command_prc([program, file_name, 'fit', 'random'])
#
# retrieve the optimal random effects
new = False
connection = dismod_at.create_connection(file_name, new)
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
#
var_id = node_name2var_id['child_0']
uhat_0 = fit_var_table[var_id]['fit_var_value']
#
var_id = node_name2var_id['child_1']
uhat_1 = fit_var_table[var_id]['fit_var_value']
#
uhat = numpy.array([uhat_0, uhat_1])
return uhat
def get_table_dict():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the covariate table
col_name = ['covariate_name', 'reference']
col_type = ['text', 'real']
row_list = [['sex', 0.0], ['income', 2000.0]]
tbl_name = 'covariate'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
n_row = len(row_list)
#
table_dict = dismod_at.get_table_dict(connection, tbl_name)
assert len(table_dict) == n_row
for i in range(n_row):
assert len(table_dict[i]) == 2
assert table_dict[i]['covariate_name'] == row_list[i][0]
assert table_dict[i]['reference'] == row_list[i][1]
#
connection.close()
print('get_table_dict: OK')
def perturb_command(database, tbl_name, sigma_str):
assert type(database) == str
assert type(tbl_name) == str
assert type(sigma_str) == str
#
# key
if tbl_name == 'scale_var':
key = 'scale_var_value'
elif tbl_name == 'start_var':
key = 'start_var_value'
else:
msg = f'perturb command: tbl_name = {tbl_name} is not '
msg += 'start_var or scale_var'
assert False, msg
#
# sigma
sigma = float(sigma_str)
if sigma <= 0:
msg = f'perturb command: sigma = {sigma} is less than or equal zero'
assert False, msg
#
# limit_var_table
limit_var_table = get_limit_var_table(database)
#
# connection
new = False
connection = dismod_at.create_connection(database, new)
#
# table
table = dismod_at.get_table_dict(connection, tbl_name)
#
for (var_id, limit_row) in enumerate(limit_var_table):
#
# mul
log_multiplier = random.gauss(0, sigma)
multiplier = math.exp(log_multiplier)
#
# value
value = table[var_id][key]
value = multiplier * value
if limit_row['lower'] is not None:
value = max(value, limit_row['lower'])
if limit_row['upper'] is not None:
value = min(value, limit_row['upper'])
#
# table
table[var_id][key] = value
#
# replace table
dismod_at.replace_table(connection, tbl_name, table)
#
return
def updateMeasNoiseDensity(self, density_name: str, params: Dict[str,
Any]):
connection = dismod_at.create_connection(self.path, False)
density_table = dismod_at.get_table_dict(connection, 'density')
command = 'UPDATE data SET density_id = ' \
+ str(self.density_dict[density_name])
print(command)
dismod_at.sql_command(connection, command)
for k, v in params.items():
command = 'UPDATE data SET ' + k + ' = ' + str(v)
dismod_at.sql_command(connection, command)
print(command)
connection.close()
def replace_table():
import dismod_at
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create my table
col_name = ['int_name', 'real_name', 'text_name']
col_type = ['integer', 'real', 'text']
row_list = [[1, 2.0, 'three']]
tbl_name = 'my'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
# original table
my_table = dismod_at.get_table_dict(connection, tbl_name)
#
# original values
assert len(my_table) == 1
row = my_table[0]
assert row['int_name'] == 1
assert row['real_name'] == 2.0
assert row['text_name'] == 'three'
#
# new row in the table
row = {'int_name': 2, 'real_name': 3.0, 'text_name': 'four'}
my_table.append(row)
dismod_at.replace_table(connection, tbl_name, my_table)
#
# check the new table
new_table = dismod_at.get_table_dict(connection, 'my')
assert new_table == my_table
#
connection.close()
print('get_name_type: OK')
def create_truth_var_table():
new = False
connection = dismod_at.create_connection(file_name, new)
var_table = dismod_at.get_table_dict(connection, 'var')
rate_table = dismod_at.get_table_dict(connection, 'rate')
covariate_table = dismod_at.get_table_dict(connection, 'covariate')
integrand_table = dismod_at.get_table_dict(connection, 'integrand')
node_table = dismod_at.get_table_dict(connection, 'node')
time_table = dismod_at.get_table_dict(connection, 'time')
age_table = dismod_at.get_table_dict(connection, 'age')
# -------------------------------------------------------------------------
# create truth table
tbl_name = 'truth_var'
col_name = ['truth_var_value']
col_type = ['real']
row_list = list()
for var_id in range(len(var_table)):
value = None
#
row = var_table[var_id]
var_type = row['var_type']
age = age_table[row['age_id']]['age']
time = time_table[row['time_id']]['time']
if var_type.startswith('mulcov_'):
covariate = covariate_table[row['covariate_id']]['covariate_name']
value = mulcov_dict[covariate]
elif var_type == 'rate':
node = node_table[row['node_id']]['node_name']
rate = rate_table[row['rate_id']]['rate_name']
value = true_rate(node, rate, age, time)
else:
assert False
#
row_list.append([value])
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
connection.close()
return
# Create database file_name = 'example.db' example_db(file_name) program = '../../devel/dismod_at' # # fit fixed dismod_at.system_command_prc([ program, file_name, 'init' ]) dismod_at.system_command_prc([ program, file_name, 'set', 'truth_var', 'prior_mean' ]) dismod_at.system_command_prc([ program, file_name, 'predict', 'truth_var' ]) # # new = False connection = dismod_at.create_connection(file_name, new) data_table = dismod_at.get_table_dict(connection, 'data') avgint_table = dismod_at.get_table_dict(connection, 'avgint') predict_table = dismod_at.get_table_dict(connection, 'predict') # n_data = len(data_table) assert len(avgint_table) == n_data assert len(predict_table) == n_data for data_id in range( n_data ) : data_row = data_table[data_id] avgint_id = predict_table[data_id]['avgint_id'] avgint_row = avgint_table[avgint_id] for key in avgint_row : assert avgint_row[key] == data_row[key] # avg_integrand = predict_table[data_id]['avg_integrand'] meas_value = data_table[data_id]['meas_value']
option_table ) # ---------------------------------------------------------------------- return # =========================================================================== # Run the init command to create the var table file_name = 'example.db' example_db(file_name) # program = '../../devel/dismod_at' dismod_at.system_command_prc([ program, file_name, 'init' ]) # ----------------------------------------------------------------------- # read database new = False connection = dismod_at.create_connection(file_name, new) var_table = dismod_at.get_table_dict(connection, 'var') rate_table = dismod_at.get_table_dict(connection, 'rate') covariate_table= dismod_at.get_table_dict(connection, 'covariate') # ----------------------------------------------------------------------- # truth table: tbl_name = 'truth_var' col_name = [ 'truth_var_value' ] col_type = [ 'real' ] row_list = list() var_id2true = list() for var_id in range( len(var_table) ) : var_info = var_table[var_id] truth_var_value = None var_type = var_info['var_type'] if var_type == 'mulcov_rate_value' : rate_id = var_info['rate_id']
mulcov_table, option_table ) # ---------------------------------------------------------------------- return # =========================================================================== # Run the init command to create the var table file_name = 'example.db' example_db(file_name) program = '../../devel/dismod_at' system_command([ program, file_name, 'init' ]) # ----------------------------------------------------------------------- # read database new = False connection = dismod_at.create_connection(file_name, new) var_table = dismod_at.get_table_dict(connection, 'var') rate_table = dismod_at.get_table_dict(connection, 'rate') covariate_table= dismod_at.get_table_dict(connection, 'covariate') # ----------------------------------------------------------------------- # truth table: tbl_name = 'truth_var' col_name = [ 'truth_var_value' ] col_type = [ 'real' ] row_list = list() var_id2true = list() for var_id in range( len(var_table) ) : var_info = var_table[var_id] truth_var_value = None var_type = var_info['var_type'] if var_type == 'mulcov_rate_value' : rate_id = var_info['rate_id']
check += 'init_ldlt_ran_hes_done_\n'
check += 'init_hes_cross_done_\n'
check += 'init_fix_like_done_\n'
check += 'init_fix_con_done_\n'
check += 'End cppad_mixed::initialize\n'
check += 'End dismod_at: fit_model constructor\n'
check += 'Begin cppad_mixed::init_laplace_obj\n'
check += 'init_laplace_obj_fun_done_\n'
check += 'init_laplace_obj_hes_done_\n'
check += 'End cppad_mixed::init_laplace_obj\n'
assert stdout == check
# -----------------------------------------------------------------------
# read database
new = False
connection = dismod_at.create_connection(file_name, new)
node_table = dismod_at.get_table_dict(connection, 'node')
age_table = dismod_at.get_table_dict(connection, 'age')
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
connection.close()
#
# There are four values for iota in this model
assert len(var_table) == 4
assert len(fit_var_table) == 4
#
# check that the fit is accurate
for var_id in range(len(var_table)):
age_id = var_table[var_id]['age_id']
age = age_table[age_id]['age']
node_id = var_table[var_id]['node_id']
node_name = node_table[node_id]['node_name']
#
# run init and fit
program = '../../devel/dismod_at'
for command in [ 'init', 'fit' ] :
cmd = [ program, file_name, command ]
if command == 'fit' :
variables = 'both'
cmd.append(variables)
print( ' '.join(cmd) )
flag = subprocess.call( cmd )
if flag != 0 :
sys.exit('The dismod_at ' + command + ' command failed')
#
# get variable and fit_var tables
connection = dismod_at.create_connection(file_name, new)
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
fit_data_subset_dict = dismod_at.get_table_dict(connection, 'fit_data_subset')
# -----------------------------------------------------------------------------
count = 0
n_var = len(var_table)
for var_id in range(n_var) :
var = var_table[var_id]
age_id = var['age_id']
age = age_list[age_id]
if age == 40 :
fit = fit_var_table[var_id]
iota_at_40 = fit['fit_var_value']
data = fit_data_subset_dict[0]
avgint_at_40 = data['avg_integrand']
assert abs( avgint_at_40 / iota_at_40 - 1.0 ) < 1e-10
example_db(file_name)
#
# initialize the database
program = '../../devel/dismod_at'
dismod_at.system_command_prc([program, file_name, 'init'])
#
# fit both the fixed and random effects
dismod_at.system_command_prc([program, file_name, 'fit', 'both'])
# ---------------------------------------------------------------------------
# Compare Fit and Truth
# ---------------------------------------------------------------------------
#
# read var table and supporting information including the fit
new = False
connection = dismod_at.create_connection(file_name, new)
subgroup_table = dismod_at.get_table_dict(connection, 'subgroup')
node_table = dismod_at.get_table_dict(connection, 'node')
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
n_var = len(var_table)
#
# check fit results
for var_id in range(n_var):
fit_var_value = fit_var_table[var_id]['fit_var_value']
var_type = var_table[var_id]['var_type']
name = None
if var_type == 'rate':
node_id = var_table[var_id]['node_id']
name = node_table[node_id]['node_name']
else:
assert var_type == 'mulcov_rate_value'
# import get_started_db example
sys.path.append(os.getcwd() + '/example/get_started')
import get_started_db
#
# change into the test/user directory
distutils.dir_util.mkpath('build/test/user')
os.chdir('build/test/user')
# ---------------------------------------------------------------------------
# create get_started.db
get_started_db.get_started_db()
# -----------------------------------------------------------------------
# In option table add row specifying parent_node_id
new = False
file_name = 'get_started.db'
connection = dismod_at.create_connection(file_name, new)
option_table = dismod_at.get_table_dict(connection, 'option')
node_table = dismod_at.get_table_dict(connection, 'node')
parent_node_id = None
for row in option_table:
if row['option_name'] == 'parent_node_name':
name = row['option_value']
for node_id in range(len(node_table)):
if node_table[node_id]['node_name'] == name:
assert parent_node_id == None
parent_node_id = str(node_id)
assert parent_node_id != None
cmd = "INSERT INTO option ('option_name', 'option_value')"
cmd += " VALUES('parent_node_id', '" + parent_node_id + "')"
dismod_at.sql_command(connection, cmd)
connection.close()
# -----------------------------------------------------------------------
def get_limit_var_table(database):
#
# table
new = False
connection = dismod_at.create_connection(database, new)
table = dict()
for tbl_name in [
'node',
'option',
'prior',
'smooth',
'smooth_grid',
'var',
]:
table[tbl_name] = dismod_at.get_table_dict(connection, tbl_name)
connection.close()
#
# parent_node_id
parent_node_id = None
parent_node_name = None
for row in table['option']:
if row['option_name'] == 'parent_node_id':
parent_node_id = int(row['option_value'])
if row['option_name'] == 'parent_node_name':
parent_node_name = row['option_value']
assert not (parent_node_id is None and parent_node_name is None)
if parent_node_id is None:
for (node_id, row) in enumerate(table['node']):
if row['node_name'] == parent_node_name:
parent_node_id = node_id
if parent_node_id is None:
msg = f'parent_node_name = {parent_node_name} is not in node table'
assert False, msg
#
# random_seed
random_seed = 0
for row in table['option']:
if row['option_name'] == 'random_seed':
random_seed = int(row['option_value'])
if random_seed != 0:
random.seed(random_seed)
#
# result
limit_var_table = list()
#
# var_id, var_row
for (var_id, var_row) in enumerate(table['var']):
#
# var_type, node_id, subgroup_id, smooth_id
var_type = var_row['var_type']
node_id = var_row['node_id']
subgoup_id = var_row['subgroup_id']
group_id = var_row['group_id']
subgroup_id = var_row['subgroup_id']
smooth_id = var_row['smooth_id']
assert (subgroup_id is None) or (group_id is None)
#
# smooth_row
smooth_row = table['smooth'][smooth_id]
#
# value_prior_id, const_value
value_prior_id = None
const_value = None
if var_type.startswith('mulstd_'):
value_prior_id = smooth_row['mulstd_value_prior_id']
else:
# age_id, time_id
age_id = var_row['age_id']
time_id = var_row['time_id']
#
# grid_row
for grid_row in table['smooth_grid']:
#
# match
# does grid_row match this variable
match = True
match = match and grid_row['smooth_id'] == smooth_id
match = match and grid_row['age_id'] == age_id
match = match and grid_row['time_id'] == time_id
if match:
#
# value_prior_id, const_value
value_prior_id = grid_row['value_prior_id']
const_value = grid_row['const_value']
#
# lower, upper, mean
if value_prior_id is None:
assert const_value is not None
lower = const_value
upper = const_value
mean = const_value
else:
assert const_value is None
prior_row = table['prior'][value_prior_id]
lower = prior_row['lower']
upper = prior_row['upper']
mean = prior_row['mean']
#
row = {'lower': lower, 'mean': mean, 'upper': upper}
limit_var_table.append(row)
#
return limit_var_table
command = [program, database] + cmd.split()
print(' '.join(command))
flag = subprocess.call(command)
if flag != 0:
sys.exit('The dismod_at init command failed')
#
exec_shell_cmd('init')
#
# connect to database
new = False
connection = dismod_at.create_connection(database, new)
#
# get variable and fit_var tables
var_table = dismod_at.get_table_dict(connection, 'var')
rate_table = dismod_at.get_table_dict(connection, 'rate')
integrand_table = dismod_at.get_table_dict(connection, 'integrand')
#
assert len(var_table) == 2
for i in range(2):
row = var_table[i]
assert row['var_type'] == 'rate'
assert row['age_id'] == i
assert row['time_id'] == 0
assert row['node_id'] == 0
assert row['integrand_id'] == None
assert row['covariate_id'] == None
rate_row = rate_table[row['rate_id']]
assert rate_row['rate_name'] == 'omega'
# ============================================================================
# ----------------------------------------------------------------------
# ===========================================================================
# Fit to determine nonzero covariate multipliers
file_name = 'example.db'
example_db(file_name)
#
program = '../../devel/dismod_at'
dismod_at.system_command_prc([program, file_name, 'init'])
dismod_at.system_command_prc([program, file_name, 'fit', 'fixed'])
#
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
density_table = dismod_at.get_table_dict(connection, 'density')
#
# density_id for a uniform distribution
uniform_id = None
for density_id in range(len(density_table)):
if density_table[density_id]['density_name'] == 'uniform':
uniform_id = density_id
#
# check covariate multiplier values
nonzero_mulcov = list()
for var_id in range(len(var_table)):
row = var_table[var_id]
match = row['var_type'] == 'mulcov_rate_value'
if match:
# ----------------------------------------------------------------------
return
# ===========================================================================
# Run the init command to create the var table
file_name = 'example.db'
example_db(file_name)
#
program = '../../devel/dismod_at'
dismod_at.system_command_prc([program, file_name, 'init'])
# -----------------------------------------------------------------------
# read database
new = False
connection = dismod_at.create_connection(file_name, new)
var_table = dismod_at.get_table_dict(connection, 'var')
rate_table = dismod_at.get_table_dict(connection, 'rate')
integrand_table = dismod_at.get_table_dict(connection, 'integrand')
covariate_table = dismod_at.get_table_dict(connection, 'covariate')
node_table = dismod_at.get_table_dict(connection, 'node')
# -----------------------------------------------------------------------
# truth table:
tbl_name = 'truth_var'
col_name = ['truth_var_value']
col_type = ['real']
row_list = list()
var_id2true = list()
for var_id in range(len(var_table)):
var_info = var_table[var_id]
truth_var_value = None
var_type = var_info['var_type']
integrand_name = 'Sincidence' density_name = 'students' eta_str = 'null' nu_str = str(nu) dismod_at.system_command_prc([ program, file_name, 'data_density', integrand_name, density_name, eta_str, nu_str ]) # # fit with Students-t (now that we have a better starting point) dismod_at.system_command_prc([ program, file_name, 'fit', 'fixed' ]) # # get second fit information new = False connection = dismod_at.create_connection(file_name, new) node_table = dismod_at.get_table_dict(connection, 'node') rate_table = dismod_at.get_table_dict(connection, 'rate') var_table = dismod_at.get_table_dict(connection, 'var') fit_var_table = dismod_at.get_table_dict(connection, 'fit_var') fit_data_subset = dismod_at.get_table_dict(connection, 'fit_data_subset') data_table = dismod_at.get_table_dict(connection, 'data') # n_var = len( fit_var_table ) assert n_var == 1 for var_id in range( n_var ) : var_type = var_table[var_id]['var_type'] rate_id = var_table[var_id]['rate_id'] node_id = var_table[var_id]['node_id'] # assert( var_type == 'rate' ) assert( rate_table[rate_id]['rate_name'] == 'iota' )
example_db(file_name)
#
from dismod_at import system_command_prc
program = '../../devel/dismod_at'
system_command_prc([program, file_name, 'init'])
system_command_prc([program, file_name, 'fit', 'fixed'])
system_command_prc([program, file_name, 'fit', 'both'])
# -----------------------------------------------------------------------
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
# -----------------------------------------------------------------------
# check the zero random effects solution
#
# get variable and fit_var tables
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
rate_table = dismod_at.get_table_dict(connection, 'rate')
node_table = dismod_at.get_table_dict(connection, 'node')
time_table = dismod_at.get_table_dict(connection, 'time')
subgroup_table = dismod_at.get_table_dict(connection, 'subgroup')
integrand_table = dismod_at.get_table_dict(connection, 'integrand')
#
# For each rate (iota and rho) there are two fixed effects.
# In addition, for each rate and each subgroup there are two random effects.
# Thus there are four fixed effects and 8 random effects.
n_var = len(var_table)
assert n_var == 12
#
# initialize sum of random effects for each rate and time
sum_random = {'Sincidence': [0.0, 0.0], 'remission': [0.0, 0.0]}
# Create database and run init, start, fit with just fixed effects
file_name = 'example.db'
example_db(file_name)
#
program = '../../devel/dismod_at'
dismod_at.system_command_prc([program, file_name, 'init'])
dismod_at.system_command_prc([program, file_name, 'fit', 'random'])
# -----------------------------------------------------------------------
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
# -----------------------------------------------------------------------
# check the zero random effects solution
#
# get variable and fit_var tables
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
rate_table = dismod_at.get_table_dict(connection, 'rate')
node_table = dismod_at.get_table_dict(connection, 'node')
#
# one age and two times for each of north_america, canada, unites_states
n_var = len(var_table)
assert n_var == 6
#
# check of values uses the fact that the data density is Gaussian
for var_id in range(n_var):
var_type = var_table[var_id]['var_type']
assert (var_type == 'rate')
#
rate_id = var_table[var_id]['rate_id']
assert (rate_table[rate_id]['rate_name'] == 'iota')
# connection
new = False
connection = dismod_at.create_connection(database_file_arg, new)
# ---------------------------------------------------------------------------
# create log table if it does not yet exist
cmd = 'create table if not exists log('
cmd += ' log_id integer primary key,'
cmd += ' message_type text ,'
cmd += ' table_name text ,'
cmd += ' row_id integer ,'
cmd += ' unix_time integer ,'
cmd += ' message text )'
dismod_at.sql_command(connection, cmd)
# ---------------------------------------------------------------------------
# write begin for this command
log_table = dismod_at.get_table_dict(connection, 'log')
log_id = len(log_table)
#
# message
message = 'begin'
for i in range(len(sys.argv) - 2):
message += ' ' + sys.argv[i + 2]
#
cmd = 'insert into log'
cmd += ' (log_id,message_type,table_name,row_id,unix_time,message) values('
cmd += str(log_id) + ',' # log_id
cmd += '"command",' # message_type
cmd += 'null,' # table_name
cmd += 'null,' # row_id
cmd += str(begin_time) + ',' # unix_time
cmd += '"' + message + '")' # message
print(' '.join(cmd))
flag = subprocess.call(cmd)
if flag != 0:
sys.exit('The dismod_at init command failed')
#
# fit
cmd = [program, file_name, 'fit', 'fixed']
print(' '.join(cmd))
flag = subprocess.call(cmd)
if flag != 0:
sys.exit('The dismod_at ' + command + ' command failed')
#
# check fit results
new = False
connection = dismod_at.create_connection(file_name, new)
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
connection.close()
#
max_error = 0.0
for var_id in range(len(var_table)):
row = fit_var_table[var_id]
var_type = var_table[var_id]['var_type']
fit_value = row['fit_var_value']
true_value = None
if var_type == 'mulcov_meas_noise':
true_value = gamma_true
if var_type == 'rate':
true_value = iota_true
assert (true_value != None)
# remove # at start of next line to see relative error values
# simulate
dismod_at.system_command_prc(
[program, file_name, 'simulate',
str(number_sample)])
#
# sample simulate
dismod_at.system_command_prc(
[program, file_name, 'sample', 'simulate', 'both',
str(number_sample)])
# -----------------------------------------------------------------------
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
#
# read tables
var_table = dismod_at.get_table_dict(connection, 'var')
sample_table = dismod_at.get_table_dict(connection, 'sample')
connection.close()
#
# check that there are only two variables (omega_0 and omega_1)
n_var = len(var_table)
assert n_var == 2
#
# check have proper number of rows in sample table
assert len(sample_table) == number_sample * n_var
#
# true hessian
hessian = numpy.array([[2., -1.], [-1., 2.]])
hessian = hessian / (omega_world_mean * omega_world_mean)
#
# true variance of estimate
# create the var table
program = '../../devel/dismod_at'
file_name = 'get_started.db'
command = 'init'
cmd = [program, file_name, command]
print(' '.join(cmd))
flag = subprocess.call(cmd)
if flag != 0:
sys.exit('The dismod_at init command failed')
# -----------------------------------------------------------------------
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
# -----------------------------------------------------------------------
# get the variable information
var_table = dismod_at.get_table_dict(connection, 'var')
# -----------------------------------------------------------------------
# create a truth_var table with variables values to use during simulation
tbl_name = 'truth_var'
col_name = ['truth_var_value']
col_type = ['real']
row_list = list()
omega_world = 2e-2
income_multiplier = -1e-3
for var_id in range(len(var_table)):
var_row = var_table[var_id]
var_type = var_row['var_type']
if var_type == 'mulcov_rate_value':
truth_var_value = income_multiplier
elif var_type == 'rate':
truth_var_value = omega_world
[program, file_name, 'set', 'option', 'max_num_iter_fixed', '1'])
dismod_at.system_command_prc([program, file_name, 'fit', 'both'])
dismod_at.system_command_prc(
[program, file_name, 'set', 'scale_var', 'fit_var'])
dismod_at.system_command_prc(
[program, file_name, 'set', 'option', 'max_num_iter_fixed', '30'])
dismod_at.system_command_prc(
[program, file_name, 'set', 'option', 'warn_on_stderr', 'true'])
dismod_at.system_command_prc([program, file_name, 'fit', 'both'])
# -----------------------------------------------------------------------
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
#
# get tables
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
age_table = dismod_at.get_table_dict(connection, "age")
log_table = dismod_at.get_table_dict(connection, "log")
#
# check that convergence was detected during final fit by making
# sure there are no warnings during the fit
fit_log_id = None
for log_id in range(len(log_table)):
if log_table[log_id]['message'] == 'begin fit both':
fit_log_id = log_id
assert log_table[fit_log_id + 1]['message'] == 'end fit'
#
# rate variables
assert len(age_table) == 3
iota_optimal = 1e-4 * (50.0 + age_table[1]['age'])
#
# fit fixed
dismod_at.system_command_prc([program, database, 'fit', 'fixed'])
# -----------------------------------------------------------------------------
# read tables
#
new = False
connection = dismod_at.create_connection(database, new)
#
name_list = [
'fit_var',
'data',
]
table_list = dict()
for table_name in name_list:
table_list[table_name] = dismod_at.get_table_dict(connection, table_name)
#
connection.close()
# ----------------------------------------------------------------------------
#
# iota_hat
table = table_list['fit_var']
assert len(table) == 1
iota_hat = table[0]['fit_var_value']
#
# y, sigma
table = table_list['data']
assert len(table) == 2
y = numpy.array([table[0]['meas_value'], table[1]['meas_value']])
sigma = numpy.array([table[0]['meas_std'], table[1]['meas_std']])
#
example_db(file_name)
#
# initialize the database
program = '../../devel/dismod_at'
dismod_at.system_command_prc([program, file_name, 'init'])
#
# fit both the fixed and random effects
dismod_at.system_command_prc([program, file_name, 'fit', 'both'])
# ---------------------------------------------------------------------------
# Compare Fit and Truth
# ---------------------------------------------------------------------------
#
# read var table and supporting information including the fit
new = False
connection = dismod_at.create_connection(file_name, new)
subgroup_table = dismod_at.get_table_dict(connection, 'subgroup')
node_table = dismod_at.get_table_dict(connection, 'node')
age_table = dismod_at.get_table_dict(connection, 'age')
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
n_var = len(var_table)
#
# check fit results and create the truth_var table
tbl_name = 'truth_var'
col_name = ['truth_var_value']
col_type = ['real']
row_list = list()
var_id2node_name = list()
for var_id in range(n_var):
var_type = var_table[var_id]['var_type']
age_id = var_table[var_id]['age_id']
# ----------------------------------------------------------------------------
# Fit to estimate iota from censored data
# ----------------------------------------------------------------------------
#
file_name = 'example.db'
example_db(file_name)
#
# init and fit fixed
program = '../../devel/dismod_at'
system_command([ program, file_name, 'init' ])
system_command([ program, file_name, 'fit', 'fixed' ])
#
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
var_table = dismod_at.get_table_dict(connection, 'var')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
#
# check result of the fit fixed
assert len(var_table) == 1
first_estimate = fit_var_table[0]['fit_var_value']
if abs( 1.0 - first_estimate / iota_true ) > 1e-1 :
print("random_seed = ", random_seed)
print("first_estimate = ", first_estimate)
print(1.0 - first_estimate / iota_true)
print(first_estimate, iota_true)
assert False
# -----------------------------------------------------------------------------
# Simulate a censored data set and fit it
# ----------------------------------------------------------------------------
# set truth_var table to contain iota_true
rate_table, mulcov_table, option_table)
# ===========================================================================
file_name = 'example.db'
example_db(file_name)
#
program = '../../devel/dismod_at'
dismod_at.system_command_prc([program, file_name, 'init'])
dismod_at.system_command_prc([program, file_name, 'fit', 'both'])
dismod_at.system_command_prc([program, file_name, 'predict', 'fit_var'])
# -----------------------------------------------------------------------
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
predict_table = dismod_at.get_table_dict(connection, 'predict')
avgint_table = dismod_at.get_table_dict(connection, 'avgint')
node_table = dismod_at.get_table_dict(connection, 'node')
#
# check that all the avgint_table values were predicted (no subsetting)
assert len(predict_table) == 3
#
# S(a) = exp( - iota * a )
iota_canada = math.exp(canada_effect) * iota_north_america
iota_united_states = math.exp(united_states_effect) * iota_north_america
S_north_america = math.exp(-iota_north_america * 50.0)
S_canada = math.exp(-iota_canada * 50.0)
S_united_states = math.exp(-iota_united_states * 50.0)
truth = {
'north_america': S_north_america,
'canada': S_canada,
# ===========================================================================
file_name = 'example.db'
example_db(file_name)
#
program = '../../devel/dismod_at'
dismod_at.system_command_prc([program, file_name, 'init'])
dismod_at.system_command_prc([program, file_name, 'fit', 'both'])
dismod_at.system_command_prc(
[program, file_name, 'sample', 'asymptotic', 'both',
str(number_sample)])
# -----------------------------------------------------------------------
# get tables
new = False
connection = dismod_at.create_connection(file_name, new)
var_table = dismod_at.get_table_dict(connection, 'var')
node_table = dismod_at.get_table_dict(connection, 'node')
rate_table = dismod_at.get_table_dict(connection, 'rate')
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
sample_table = dismod_at.get_table_dict(connection, 'sample')
hes_random_table = dismod_at.get_table_dict(connection, 'hes_random')
connection.close()
dismod_at.db2csv_command(file_name)
# -----------------------------------------------------------------------
# var_id2name
var_id2node_name = dict()
assert len(var_table) == 3
for var_id in range(len(var_table)):
assert var_id < 3
row = var_table[var_id]
assert row['var_type'] == 'rate'