def optimal_random_effect(fixed_effect):
#
# set start_var value for the fixed effects
var_id = node_name2var_id['world']
sql_cmd = 'UPDATE start_var SET start_var_value = ' + str(fixed_effect)
sql_cmd += ' WHERE start_var_id == ' + str(var_id)
new = False
connection = dismod_at.create_connection(file_name, new)
dismod_at.sql_command(connection, sql_cmd)
connection.close()
#
# optimize the random effects
dismod_at.system_command_prc([program, file_name, 'fit', 'random'])
#
# retrieve the optimal random effects
new = False
connection = dismod_at.create_connection(file_name, new)
fit_var_table = dismod_at.get_table_dict(connection, 'fit_var')
#
var_id = node_name2var_id['child_0']
uhat_0 = fit_var_table[var_id]['fit_var_value']
#
var_id = node_name2var_id['child_1']
uhat_1 = fit_var_table[var_id]['fit_var_value']
#
uhat = numpy.array([uhat_0, uhat_1])
return uhat
def rate_table():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the rate table
col_name = [
'rate_name', 'parent_smooth_id', 'child_smooth_id', 'child_nslist_id'
]
col_type = ['text', 'integer', 'integer', 'integer']
row_list = [['pini', 0, 1, None], ['iota', 2, 3, None],
['rho', 2, 3, None], ['chi', 2, 3, None], ['omega', 2, 3, 0]]
tbl_name = 'rate'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('rate_table: OK')
def nslist_table():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the nslist table
col_name = ['nslist_name']
col_type = ['test']
row_list = [['first_list'], ['second_list']]
tbl_name = 'nslist'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('nslist_table: OK')
def age_table():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the age table
col_name = ['age']
col_type = ['real']
row_list = [[0.0], [20.0], [40.0], [60.0], [80.0], [100.0]]
tbl_name = 'age'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('age_table: OK')
def integrand_table():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the integrand table
col_name = ['integrand_name', 'eta']
col_type = ['text', 'real']
row_list = [['Tincidence', 1e-6], ['remission', 1e-6], ['mtall', 1e-6],
['mulcov_1', 1e-6]]
tbl_name = 'integrand'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('integrand_table: OK')
def density_table() :
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the density table
col_name = [ 'density_name' ]
col_type = [ 'text' ]
row_list = [
['uniform'],
['gaussian'],
['laplace'],
['log_gaussian'],
['log_laplace']
]
tbl_name = 'density'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [ tbl_name + '_id' ] + col_name
check_list = list()
for i in range( len(row_list) ) :
check_list.append( [i] + row_list[i] )
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('density_table: OK')
def get_name_type():
import dismod_at
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create temp table
col_name = ['int_name', 'real_name', 'text_name']
col_type = ['integer', 'real', 'text']
row_list = [[1, 2.0, 'three']]
tbl_name = 'temp'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
check_name = ['temp_id'] + col_name
check_type = ['integer primary key'] + col_type
#
# get the column names and corresponding types
(col_name, col_type) = dismod_at.get_name_type(connection, tbl_name)
#
assert col_name == check_name
assert col_type == check_type
#
connection.close()
print('get_name_type: OK')
def get_row_list():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the covariate table
col_name = ['covariate_name', 'reference']
col_type = ['text', 'real']
row_list = [['sex', 0.0], ['income', 2000.0]]
tbl_name = 'covariate'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
n_row = len(row_list)
#
# reverse the order of the columns
col_name = ['reference', 'covariate_name']
n_col = len(col_name)
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
#
assert len(row_list) == n_row
for i in range(n_row):
assert len(row_list[i]) == n_col
assert isinstance(row_list[i][0], float)
assert isinstance(row_list[i][1], str)
assert row_list[0][0] == 0.0
assert row_list[0][1] == 'sex'
assert row_list[1][0] == 2000.0
assert row_list[1][1] == 'income'
#
connection.close()
print('get_row_list: OK')
def get_table_dict():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the covariate table
col_name = ['covariate_name', 'reference']
col_type = ['text', 'real']
row_list = [['sex', 0.0], ['income', 2000.0]]
tbl_name = 'covariate'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
n_row = len(row_list)
#
table_dict = dismod_at.get_table_dict(connection, tbl_name)
assert len(table_dict) == n_row
for i in range(n_row):
assert len(table_dict[i]) == 2
assert table_dict[i]['covariate_name'] == row_list[i][0]
assert table_dict[i]['reference'] == row_list[i][1]
#
connection.close()
print('get_table_dict: OK')
def node_table():
import dismod_at
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the node table
col_name = ['node_name', 'parent']
col_type = ['text', 'integer']
row_list = [['world', None], ['north_america', 0], ['united_states', 1],
['canada', 1]]
tbl_name = 'node'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('node_table: OK')
def covariate_table():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the covariate table
col_name = ['covariate_name', 'reference', 'max_difference']
col_type = ['text', 'real', 'real']
row_list = [['sex', 0.0, 0.6], ['income', 2000.0, None]]
tbl_name = 'covariate'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('covariate_table: OK')
def prior_table():
import dismod_at
import copy
import collections
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create the prior table
ptype = 'integer primary key'
col_name2type = collections.OrderedDict([('prior_name', 'text'),
('density_id', 'integer'),
('lower', 'real'),
('upper', 'real'),
('mean', 'real'), ('std', 'real'),
('eta', 'real')])
col_name = list(col_name2type.keys())
col_type = list(col_name2type.values())
uniform_density_id = 0
row_list = [
[
'none', # prior_name
uniform_density_id, # density_id
None, # lower
None, # upper
0, # mean
None, # std
None # eta
],
[
'rate', # prior_name
uniform_density_id, # density_id
0.0, # lower
1.0, # upper
0.1, # mean
None, # std
None # eta
]
]
tbl_name = 'prior'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('prior_table: OK')
def weight_grid_table():
import dismod_at
import copy
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create weight table
col_name = ['weight_name', 'n_age', 'ntime']
col_type = ['text', 'integer', 'integer']
row_list = [['constant', 1, 1], ['age_linear', 2, 1], ['bilinear', 2, 2]]
tbl_name = 'weight'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
# create weight_grid table
col_name = ['weight_id', 'age_id', 'time_id', 'weight']
col_type = ['integer', 'integer', 'integer', 'real']
row_list = [
# constant
[0, 1, 1, 1.0],
# age_linear
[1, 0, 1, 0.5],
[1, 2, 1, 1.5],
# bilinear
[2, 0, 0, 0.5],
[2, 2, 0, 1.0],
[2, 0, 2, 1.0],
[2, 2, 2, 1.5]
]
tbl_name = 'weight_grid'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
# check values in the bilinear weight table
columns = ','.join(col_name)
columns = 'weight_grid_id,' + columns
cmd = 'SELECT ' + columns + ' FROM weight_grid'
cmd += ' INNER JOIN weight USING (weight_id)'
cmd += ' WHERE weight_name = "bilinear"'
#
count = 3
cursor = connection.cursor()
for row in cursor.execute(cmd):
check = copy.copy(row_list[count])
check.insert(0, count)
assert len(row) == len(check)
for j in range(len(row)):
assert row[j] == check[j]
count += 1
assert count == len(row_list)
#
connection.close()
print('weight_grid_table: OK')
def avgint_table():
import dismod_at
import copy
import collections
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
#
col_name2type = collections.OrderedDict([
# required columns
('integrand_id', 'integer'),
('density_id', 'integer'),
('node_id', 'integer'),
('weight_id', 'integer'),
('age_lower', 'real'),
('age_upper', 'real'),
('time_lower', 'real'),
('time_upper', 'real'),
# covariates
('x_sex', 'real'),
('x_income', 'real'),
])
col_name = list(col_name2type.keys())
col_type = list(col_name2type.values())
row_list = [[
1, # integrand_id
0, # density_id
3, # node_id
4, # weight_id
10.0, # age_lower
90.0, # age_upper
2000., # time_lower
2005., # time_upper
0.5, # x_sex
1000. # x_income
]]
# create the avgint table
tbl_name = 'avgint'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('avgint_table: OK')
def perturb_command(database, tbl_name, sigma_str):
assert type(database) == str
assert type(tbl_name) == str
assert type(sigma_str) == str
#
# key
if tbl_name == 'scale_var':
key = 'scale_var_value'
elif tbl_name == 'start_var':
key = 'start_var_value'
else:
msg = f'perturb command: tbl_name = {tbl_name} is not '
msg += 'start_var or scale_var'
assert False, msg
#
# sigma
sigma = float(sigma_str)
if sigma <= 0:
msg = f'perturb command: sigma = {sigma} is less than or equal zero'
assert False, msg
#
# limit_var_table
limit_var_table = get_limit_var_table(database)
#
# connection
new = False
connection = dismod_at.create_connection(database, new)
#
# table
table = dismod_at.get_table_dict(connection, tbl_name)
#
for (var_id, limit_row) in enumerate(limit_var_table):
#
# mul
log_multiplier = random.gauss(0, sigma)
multiplier = math.exp(log_multiplier)
#
# value
value = table[var_id][key]
value = multiplier * value
if limit_row['lower'] is not None:
value = max(value, limit_row['lower'])
if limit_row['upper'] is not None:
value = min(value, limit_row['upper'])
#
# table
table[var_id][key] = value
#
# replace table
dismod_at.replace_table(connection, tbl_name, table)
#
return
def updateMeasNoiseDensity(self, density_name: str, params: Dict[str,
Any]):
connection = dismod_at.create_connection(self.path, False)
density_table = dismod_at.get_table_dict(connection, 'density')
command = 'UPDATE data SET density_id = ' \
+ str(self.density_dict[density_name])
print(command)
dismod_at.sql_command(connection, command)
for k, v in params.items():
command = 'UPDATE data SET ' + k + ' = ' + str(v)
dismod_at.sql_command(connection, command)
print(command)
connection.close()
def mulcov_table():
import dismod_at
import copy
import collections
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create a mulcov table
col_name2type = collections.OrderedDict([('mulcov_type', 'text'),
('rate_id', 'integer'),
('integrand_id', 'integer'),
('covariate_id', 'integer'),
('smooth_id', 'integer')])
col_name = list(col_name2type.keys())
col_type = list(col_name2type.values())
row_list = [
[
'meas_value', # muitiplier_type
None, # rate_id is null becasue measurement covariate
2, # integrand_id
1, # covariate_id
2 # smooth_id
],
[
'rate_value', # muitiplier_type
1, # rate_id
None, # integrand_id is null because a rate covariate
2, # covariate_id
2 # smooth_id
]
]
tbl_name = 'mulcov'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('mulcov_table: OK')
def nslist_pair_table() :
import dismod_at
import copy
import collections
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create nslist_pair table column names and types
col_name2type = collections.OrderedDict( [
('nslist_id', 'integer' ),
('node_id', 'integer' ),
('smooth_id', 'integer' )
] )
col_name = list(col_name2type.keys())
col_type = list(col_name2type.values())
# two lists with different smoothing for each node
row_list = [
# nslist_id, node_id, smooth_id
[ 0, 0, 0 ],
[ 0, 1, 1 ],
[ 0, 2, 2 ],
[ 1, 0, 3 ],
[ 1, 1, 4 ],
[ 1, 2, 5 ]
]
tbl_name = 'nslist_pair'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [ tbl_name + '_id' ] + col_name
check_list = list()
for i in range( len(row_list) ) :
check_list.append( [i] + row_list[i] )
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('nslist_pair_table: OK')
def create_table() :
import dismod_at
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
inverted_exclamation = chr( 10 * 16 + 1 ) # 00a1
cent_sign = chr( 10 * 16 + 2 ) # 00a2
pound_sign = chr( 10 * 16 + 3 ) # 00a3
#
# create table
col_name = [ 'temp_name' ]
col_type = [ 'text' ]
row_list = [
[ inverted_exclamation ],
[ cent_sign ],
[ pound_sign ]
]
tbl_name = 'temp'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
# check values in table
row_list = list()
cmd = 'SELECT temp_id, temp_name FROM temp'
for row in cursor.execute(cmd) :
row_list.append(row)
#
for i in range( len(row_list) ) :
assert row_list[i][0] == i
#
assert row_list[0][1] == inverted_exclamation
assert row_list[1][1] == cent_sign
assert row_list[2][1] == pound_sign
#
connection.close()
print('create_table: OK')
def create_truth_var_table():
new = False
connection = dismod_at.create_connection(file_name, new)
var_table = dismod_at.get_table_dict(connection, 'var')
rate_table = dismod_at.get_table_dict(connection, 'rate')
covariate_table = dismod_at.get_table_dict(connection, 'covariate')
integrand_table = dismod_at.get_table_dict(connection, 'integrand')
node_table = dismod_at.get_table_dict(connection, 'node')
time_table = dismod_at.get_table_dict(connection, 'time')
age_table = dismod_at.get_table_dict(connection, 'age')
# -------------------------------------------------------------------------
# create truth table
tbl_name = 'truth_var'
col_name = ['truth_var_value']
col_type = ['real']
row_list = list()
for var_id in range(len(var_table)):
value = None
#
row = var_table[var_id]
var_type = row['var_type']
age = age_table[row['age_id']]['age']
time = time_table[row['time_id']]['time']
if var_type.startswith('mulcov_'):
covariate = covariate_table[row['covariate_id']]['covariate_name']
value = mulcov_dict[covariate]
elif var_type == 'rate':
node = node_table[row['node_id']]['node_name']
rate = rate_table[row['rate_id']]['rate_name']
value = true_rate(node, rate, age, time)
else:
assert False
#
row_list.append([value])
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
connection.close()
return
def replace_table():
import dismod_at
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create my table
col_name = ['int_name', 'real_name', 'text_name']
col_type = ['integer', 'real', 'text']
row_list = [[1, 2.0, 'three']]
tbl_name = 'my'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
# original table
my_table = dismod_at.get_table_dict(connection, tbl_name)
#
# original values
assert len(my_table) == 1
row = my_table[0]
assert row['int_name'] == 1
assert row['real_name'] == 2.0
assert row['text_name'] == 'three'
#
# new row in the table
row = {'int_name': 2, 'real_name': 3.0, 'text_name': 'four'}
my_table.append(row)
dismod_at.replace_table(connection, tbl_name, my_table)
#
# check the new table
new_table = dismod_at.get_table_dict(connection, 'my')
assert new_table == my_table
#
connection.close()
print('get_name_type: OK')
def create_database(file_name, age_list, time_list, integrand_table,
node_table, subgroup_table, weight_table, covariate_table,
avgint_table, data_table, prior_table, smooth_table,
nslist_table, rate_table, mulcov_table, option_table):
import sys
import dismod_at
# ----------------------------------------------------------------------
# avgint_extra_columns, data_extra_columns
avgint_extra_columns = list()
data_extra_columns = list()
for row in option_table:
if row['name'] == 'avgint_extra_columns':
avgint_extra_columns = row['value'].split()
if row['name'] == 'data_extra_columns':
data_extra_columns = row['value'].split()
# ----------------------------------------------------------------------
# create database
new = True
connection = dismod_at.create_connection(file_name, new)
# ----------------------------------------------------------------------
# create age table
col_name = ['age']
col_type = ['real']
row_list = []
for age in age_list:
row_list.append([age])
tbl_name = 'age'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create time table
col_name = ['time']
col_type = ['real']
row_list = []
for time in time_list:
row_list.append([time])
tbl_name = 'time'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create integrand table
col_name = ['integrand_name', 'minimum_meas_cv']
col_type = ['text', 'real']
row_list = []
for i in range(len(integrand_table)):
minimum_meas_cv = 0.0
if 'minimum_meas_cv' in integrand_table[i]:
minimum_meas_cv = integrand_table[i]['minimum_meas_cv']
row = [integrand_table[i]['name'], minimum_meas_cv]
row_list.append(row)
tbl_name = 'integrand'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
global_integrand_name2id = {}
for i in range(len(row_list)):
global_integrand_name2id[row_list[i][0]] = i
# ----------------------------------------------------------------------
# create density table
col_name = ['density_name']
col_type = ['text']
row_list = [
['uniform'],
['gaussian'],
['laplace'],
['students'],
['log_gaussian'],
['log_laplace'],
['log_students'],
['cen_gaussian'],
['cen_laplace'],
['cen_log_gaussian'],
['cen_log_laplace'],
]
tbl_name = 'density'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
global_density_name2id = {}
for i in range(len(row_list)):
global_density_name2id[row_list[i][0]] = i
# ----------------------------------------------------------------------
# create covariate table
col_name = ['covariate_name', 'reference', 'max_difference']
col_type = ['text', 'real', 'real']
row_list = []
for i in range(len(covariate_table)):
max_difference = None
if 'max_difference' in covariate_table[i]:
max_difference = covariate_table[i]['max_difference']
row = [
covariate_table[i]['name'], covariate_table[i]['reference'],
max_difference
]
row_list.append(row)
tbl_name = 'covariate'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
global_covariate_name2id = {}
for i in range(len(covariate_table)):
global_covariate_name2id[covariate_table[i]['name']] = i
# ----------------------------------------------------------------------
# create node table
global_node_name2id = {}
for i in range(len(node_table)):
global_node_name2id[node_table[i]['name']] = i
#
col_name = ['node_name', 'parent']
col_type = ['text', 'integer']
row_list = []
for i in range(len(node_table)):
node = node_table[i]
name = node['name']
parent = node['parent']
if parent == '':
parent = None
else:
parent = global_node_name2id[parent]
row_list.append([name, parent])
tbl_name = 'node'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create subgroup table
global_subgroup_name2id = {}
global_group_name2id = {}
group_id = 0
group_name = subgroup_table[0]['group']
global_group_name2id[group_name] = group_id
for i in range(len(subgroup_table)):
global_subgroup_name2id[subgroup_table[i]['subgroup']] = i
if subgroup_table[i]['group'] != group_name:
group_id = group_id + 1
group_name = subgroup_table[i]['group']
global_group_name2id[group_name] = group_id
#
col_name = ['subgroup_name', 'group_id', 'group_name']
col_type = ['text', 'integer', 'text']
row_list = []
for i in range(len(subgroup_table)):
if i == 0:
group_id = 0
group_name = subgroup_table[0]['group']
elif subgroup_table[i]['group'] != group_name:
group_id = group_id + 1
group_name = subgroup_table[i]['group']
subgroup_name = subgroup_table[i]['subgroup']
row_list.append([subgroup_name, group_id, group_name])
tbl_name = 'subgroup'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create prior table
col_name = [
'prior_name', 'lower', 'upper', 'mean', 'std', 'density_id', 'eta',
'nu'
]
col_type = [
'text', 'real', 'real', 'real', 'real', 'integer', 'real', 'real'
]
row_list = []
for i in range(len(prior_table)):
prior = prior_table[i]
density_id = global_density_name2id[prior['density']]
#
# columns that have null for default value
for key in ['lower', 'upper', 'std', 'eta', 'nu']:
if not key in prior:
prior[key] = None
#
row = [
prior['name'],
prior['lower'],
prior['upper'],
prior['mean'],
prior['std'],
density_id,
prior['eta'],
prior['nu'],
]
row_list.append(row)
tbl_name = 'prior'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
global_prior_name2id = {}
for i in range(len(row_list)):
global_prior_name2id[row_list[i][0]] = i
# ----------------------------------------------------------------------
# create weight table
col_name = ['weight_name', 'n_age', 'n_time']
col_type = ['text', 'integer', 'integer']
row_list = []
for i in range(len(weight_table)):
weight = weight_table[i]
name = weight['name']
n_age = len(weight['age_id'])
n_time = len(weight['time_id'])
row_list.append([name, n_age, n_time])
tbl_name = 'weight'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
global_weight_name2id = {}
for i in range(len(weight_table)):
global_weight_name2id[weight_table[i]['name']] = i
# null is used for constant weighting
global_weight_name2id[''] = None
# ----------------------------------------------------------------------
# create weight_grid table
col_name = ['weight_id', 'age_id', 'time_id', 'weight']
col_type = ['integer', 'integer', 'integer', 'real']
row_list = []
for i in range(len(weight_table)):
weight = weight_table[i]
age_id = weight['age_id']
time_id = weight['time_id']
fun = weight['fun']
for j in age_id:
for k in time_id:
w = fun(age_list[j], time_list[k])
row_list.append([i, j, k, w])
tbl_name = 'weight_grid'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create smooth table
col_name = [
'smooth_name', 'n_age', 'n_time', 'mulstd_value_prior_id',
'mulstd_dage_prior_id', 'mulstd_dtime_prior_id'
]
col_type = ['text', 'integer', 'integer', 'integer', 'integer', 'integer']
row_list = []
for i in range(len(smooth_table)):
smooth = smooth_table[i]
name = smooth['name']
n_age = len(smooth['age_id'])
n_time = len(smooth['time_id'])
#
prior_id = dict()
for key in ['value', 'dage', 'dtime']:
prior_id[key] = None
mulstd_key = 'mulstd_' + key + '_prior_name'
if mulstd_key in smooth:
prior_name = smooth[mulstd_key]
if prior_name != None:
prior_id[key] = global_prior_name2id[prior_name]
#
row_list.append([
name,
n_age,
n_time,
prior_id['value'],
prior_id['dage'],
prior_id['dtime'],
])
tbl_name = 'smooth'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
global_smooth_name2id = {}
for i in range(len(smooth_table)):
global_smooth_name2id[smooth_table[i]['name']] = i
# ----------------------------------------------------------------------
# create smooth_grid table
col_name = [
'smooth_id',
'age_id',
'time_id',
'value_prior_id',
'dage_prior_id',
'dtime_prior_id',
'const_value',
]
col_type = [
'integer', # smooth_id
'integer', # age_id
'integer', # time_id
'integer', # value_prior_id
'integer', # dage_prior_id
'integer', # dtime_prior_id
'real', # const_value
]
row_list = []
for i in range(len(smooth_table)):
smooth = smooth_table[i]
age_id = smooth['age_id']
time_id = smooth['time_id']
fun = smooth['fun']
max_j = 0
for j in age_id:
if age_list[j] > age_list[max_j]:
max_j = j
max_k = 0
for k in time_id:
if time_list[k] > time_list[max_k]:
max_k = k
for j in age_id:
for k in time_id:
(v, da, dt) = fun(age_list[j], time_list[k])
#
if j == max_j:
da = None
elif da != None:
da = global_prior_name2id[da]
#
if k == max_k:
dt = None
elif dt != None:
dt = global_prior_name2id[dt]
#
const_value = None
if isinstance(v, float):
const_value = v
v = None
elif v != None:
v = global_prior_name2id[v]
row_list.append([i, j, k, v, da, dt, const_value])
tbl_name = 'smooth_grid'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create nslist table
col_name = ['nslist_name']
col_type = ['text']
row_list = list()
for nslist_name in nslist_table:
row_list.append([nslist_name])
tbl_name = 'nslist'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
global_nslist_name2id = dict()
for i in range(len(row_list)):
global_nslist_name2id[row_list[i][0]] = i
# ----------------------------------------------------------------------
# create nslist_pair table
col_name = ['nslist_id', 'node_id', 'smooth_id']
col_type = ['integer', 'integer', 'integer']
row_list = list()
tbl_name = 'nslist_pair'
for key in nslist_table:
pair_list = nslist_table[key]
nslist_id = global_nslist_name2id[key]
for pair in pair_list:
node_id = global_node_name2id[pair[0]]
smooth_id = global_smooth_name2id[pair[1]]
row_list.append([nslist_id, node_id, smooth_id])
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create rate table
col_name = [
'rate_name', 'parent_smooth_id', 'child_smooth_id', 'child_nslist_id'
]
col_type = ['text', 'integer', 'integer', 'integer']
row_list = list()
for rate_name in ['pini', 'iota', 'rho', 'chi', 'omega']:
row = [rate_name, None, None, None]
for i in range(len(rate_table)):
rate = rate_table[i]
if rate['name'] == rate_name:
row = [rate_name]
for key in ['parent_smooth', 'child_smooth', 'child_nslist']:
entry = None
if key in rate:
entry = rate[key]
if entry != None:
if key == 'child_nslist':
entry = global_nslist_name2id[entry]
else:
entry = global_smooth_name2id[entry]
row.append(entry)
row_list.append(row)
tbl_name = 'rate'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
global_rate_name2id = {}
for i in range(len(row_list)):
global_rate_name2id[row_list[i][0]] = i
# ----------------------------------------------------------------------
# create mulcov table
col_name = [
'mulcov_type',
'rate_id',
'integrand_id',
'covariate_id',
'group_id',
'group_smooth_id',
'subgroup_smooth_id',
]
col_type = [
'text', # mulcov_type
'integer', # rate_id
'integer', # integrand_id
'integer', # covariate_id
'integer', # group_id
'integer', # group_smooth_id
'integer', # subgroup_smooth_id
]
row_list = []
warning_printed = False
for i in range(len(mulcov_table)):
mulcov = mulcov_table[i]
mulcov_type = mulcov['type']
effected = mulcov['effected']
covariate_id = global_covariate_name2id[mulcov['covariate']]
#
# rate_id and integrand_id
if mulcov_type == 'rate_value':
rate_id = global_rate_name2id[effected]
integrand_id = None
else:
integrand_id = global_integrand_name2id[effected]
rate_id = None
#
# group_id
if 'group' in mulcov:
group_id = global_group_name2id[mulcov['group']]
else:
group_id = 0
if not warning_printed:
msg = 'create_database Warning: '
msg += 'group key missing in mulcov table,\n'
msg += 'using default value; i.e., first group '
msg += '(you should fix this).'
print(msg)
warning_printed = True
#
# group_smooth_id
if mulcov['smooth'] == None:
group_smooth_id = None
else:
group_smooth_id = global_smooth_name2id[mulcov['smooth']]
#
# subgroup_smooth_id
if not 'subsmooth' in mulcov:
subgroup_smooth_id = None
elif mulcov['subsmooth'] == None:
subgroup_smooth_id = None
else:
subgroup_smooth_id = global_smooth_name2id[mulcov['subsmooth']]
#
row_list.append([
mulcov_type,
rate_id,
integrand_id,
covariate_id,
group_id,
group_smooth_id,
subgroup_smooth_id,
])
tbl_name = 'mulcov'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# avgint table
#
# extra_name, extra_type
extra_name = []
extra_type = []
if (len(avgint_table) > 0):
extra_name = avgint_extra_columns
row = avgint_table[0]
for key in extra_name:
if isinstance(row[key], str):
extra_type.append('text')
elif isinstance(row[key], int):
extra_type.append('integer')
elif isinstance(row[key], float):
extra_type.append('real')
else:
msg = 'db2csv_command: avgint_extra_columns: type error:'
msg += '\nThe type for column ' + key
msg += ' is not str, int, or float'
assert False, msg
#
# col_name
col_name = extra_name + [
'integrand_id', 'node_id', 'subgroup_id', 'weight_id', 'age_lower',
'age_upper', 'time_lower', 'time_upper'
]
for j in range(len(covariate_table)):
col_name.append('x_%s' % j)
#
# col_type
col_type = extra_type + [
'integer', # integrand_id
'integer', # node_id
'integer', # subgroup_id
'integer', # weight_id
'real', # age_lower
'real', # age_upper
'real', # time_lower
'real' # time_upper
]
for j in range(len(covariate_table)):
col_type.append('real')
#
# row_list
row_list = []
warning_printed = False
for i in range(len(avgint_table)):
avgint = avgint_table[i]
#
# subgroup column has a default value
if 'subgroup' not in avgint:
avgint['subgroup'] = subgroup_table[0]['subgroup']
if not warning_printed:
msg = 'create_database Warning: '
msg += 'subgroup key missing in avgint table,\n'
msg += 'using default value; i.e., first subgroup '
msg += '(you should fix this).'
print(msg)
warning_printed = True
#
# extra columns first
row = list()
for name in extra_name:
row.append(avgint[name])
#
avgint_id = i
integrand_id = global_integrand_name2id[avgint['integrand']]
node_id = global_node_name2id[avgint['node']]
subgroup_id = global_subgroup_name2id[avgint['subgroup']]
weight_id = global_weight_name2id[avgint['weight']]
row = row + [
integrand_id, node_id, subgroup_id, weight_id, avgint['age_lower'],
avgint['age_upper'], avgint['time_lower'], avgint['time_upper']
]
for j in range(len(covariate_table)):
row.append(avgint[covariate_table[j]['name']])
row_list.append(row)
tbl_name = 'avgint'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create data table
#
#
# extra_name, extra_type
extra_name = []
extra_type = []
if (len(data_table) > 0):
extra_name = data_extra_columns
row = data_table[0]
for key in extra_name:
if isinstance(row[key], str):
extra_type.append('text')
elif isinstance(row[key], int):
extra_type.append('integer')
elif isinstance(row[key], float):
extra_type.append('real')
else:
msg = 'db2csv_command: data_extra_columns: type error'
msg += '\nThe type for column ' + key
msg += ' is not str, int, or float'
assert False, msg
#
# col_name
col_name = extra_name + [
'integrand_id',
'node_id',
'subgroup_id',
'weight_id',
'age_lower',
'age_upper',
'time_lower',
'time_upper',
'hold_out',
'density_id',
'meas_value',
'meas_std',
'eta',
'nu',
]
for j in range(len(covariate_table)):
col_name.append('x_%s' % j)
#
# col_type
col_type = extra_type + [
'integer', # integrand_id
'integer', # node_id
'integer', # subgroup_id
'integer', # weight_id
'real', # age_lower
'real', # age_upper
'real', # time_lower
'real', # time_upper
'integer', # hold_out
'integer', # density_id
'real', # meas_value
'real', # meas_std
'real', # eta
'real', # nu
]
for j in range(len(covariate_table)):
col_type.append('real')
row_list = []
warning_printed = False
for i in range(len(data_table)):
data = data_table[i]
#
# extra columns first
row = list()
for name in extra_name:
row.append(data[name])
#
# columns that have null for default value
for key in ['meas_std', 'eta', 'nu']:
if not key in data:
data[key] = None
#
# subgroup column has a default value
if not 'subgroup' in data:
data['subgroup'] = subgroup_table[0]['subgroup']
if not warning_printed:
msg = 'create_database Warning: '
msg += 'subgroup key missing in data table,\n'
msg += 'using default value; i.e., first subgroup '
msg += '(you should fix this).'
print(msg)
warning_printed = True
#
integrand_id = global_integrand_name2id[data['integrand']]
density_id = global_density_name2id[data['density']]
node_id = global_node_name2id[data['node']]
subgroup_id = global_subgroup_name2id[data['subgroup']]
weight_id = global_weight_name2id[data['weight']]
hold_out = int(data['hold_out'])
row = row + [
integrand_id, node_id, subgroup_id, weight_id, data['age_lower'],
data['age_upper'], data['time_lower'], data['time_upper'],
hold_out, density_id, data['meas_value'], data['meas_std'],
data['eta'], data['nu']
]
for j in range(len(covariate_table)):
row.append(data[covariate_table[j]['name']])
row_list.append(row)
tbl_name = 'data'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# create option table
col_name = ['option_name', 'option_value']
col_type = ['text unique', 'text']
row_list = []
for row in option_table:
name = row['name']
value = row['value']
row_list.append([name, value])
tbl_name = 'option'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# close the connection
connection.close()
return
# ---------------------------------------------------------------------------
# begin_time
begin_time = int(time.time())
# ---------------------------------------------------------------------------
# deprecated
# special case where we execute the command first to create the database
if command_arg == 'csv2db':
if len(sys.argv) != 5:
sys.exit(usage)
configure_csv = sys.argv[3]
measure_csv = sys.argv[4]
dismod_at.csv2db_command(database_file_arg, configure_csv, measure_csv)
# ---------------------------------------------------------------------------
# connection
new = False
connection = dismod_at.create_connection(database_file_arg, new)
# ---------------------------------------------------------------------------
# create log table if it does not yet exist
cmd = 'create table if not exists log('
cmd += ' log_id integer primary key,'
cmd += ' message_type text ,'
cmd += ' table_name text ,'
cmd += ' row_id integer ,'
cmd += ' unix_time integer ,'
cmd += ' message text )'
dismod_at.sql_command(connection, cmd)
# ---------------------------------------------------------------------------
# write begin for this command
log_table = dismod_at.get_table_dict(connection, 'log')
log_id = len(log_table)
#
def create_database():
import dismod_at
import copy
#
# file_name, age_list, time_list, integrand_table
file_name = 'example.db'
age_list = [50.0]
time_list = [2000.0]
integrand_table = [{'name': 'mtother', 'minimum_meas_cv': 0.0}]
#
# node_table
node_table = [{
'name': 'world',
'parent': ''
}, {
'name': 'north_america',
'parent': 'world'
}, {
'name': 'united_states',
'parent': 'north_america'
}, {
'name': 'canada',
'parent': 'north_america'
}]
# weight_table
fun = constant_weight_fun
weight_table = [{
'name': 'constant',
'age_id': [0],
'time_id': [0],
'fun': fun
}]
# covariate_table
covariate_table = [{
'name': 'sex',
'reference': 0.0,
'max_difference': 0.6
}]
# data_table
data_table = []
row = {
'integrand': 'mtother',
'density': 'log_gaussian',
'weight': 'constant',
'hold_out': False,
'meas_std': 1e-5,
'age_lower': 0.0,
'age_upper': 100.0,
'time_lower': 1990.0,
'time_upper': 2010.0,
'sex': 0.5,
'subgroup': 'world',
}
row['data_name'] = 'one'
row['node'] = 'north_america'
row['meas_value'] = 1.0e-5
data_table.append(copy.copy(row))
row['data_name'] = 'two'
row['node'] = 'united_states'
row['meas_value'] = 1.5e-5
data_table.append(copy.copy(row))
row['data_name'] = 'three'
row['node'] = 'canada'
row['meas_value'] = 0.5e-5
data_table.append(copy.copy(row))
#
# prior_table
prior_table = [{
'name': 'zero',
'density': 'uniform',
'lower': 0.0,
'upper': 0.0,
'mean': 0.0,
}, {
'name': 'one',
'density': 'uniform',
'lower': 1.0,
'upper': 1.0,
'mean': 1.0,
}, {
'name': 'uniform_01',
'density': 'uniform',
'lower': 0.0,
'upper': 1.0,
'mean': 0.1,
}, {
'name': 'gaussian_01',
'density': 'gaussian',
'mean': 0.0,
'std': 1.0,
}, {
'name': 'log_gaussian',
'density': 'log_gaussian',
'mean': 0.0,
'eta': 1e-6
}]
#
# smooth list
smooth_table = [{
'name': 'uniform_01_constant',
'age_id': [0],
'time_id': [0],
'mulstd_value_prior_name': None,
'mulstd_dage_prior_name': None,
'mulstd_dtime_prior_name': None,
'fun': smooth_uniform_01_fun
}, {
'name': 'gaussian_01_constant',
'age_id': [0],
'time_id': [0],
'mulstd_value_prior_name': None,
'mulstd_dage_prior_name': None,
'mulstd_dtime_prior_name': None,
'fun': smooth_gaussian_01_fun
}]
#
# rate_table
rate_table = [{
'name': 'pini',
'parent_smooth': 'uniform_01_constant',
'child_smooth': 'gaussian_01_constant',
'child_nslist': None
}, {
'name': 'iota',
'parent_smooth': 'uniform_01_constant',
'child_smooth': 'gaussian_01_constant',
'child_nslist': None
}, {
'name': 'rho',
'parent_smooth': 'uniform_01_constant',
'child_smooth': 'gaussian_01_constant',
'child_nslist': None
}, {
'name': 'chi',
'parent_smooth': 'uniform_01_constant',
'child_smooth': 'gaussian_01_constant',
'child_nslist': None
}, {
'name': 'omega',
'parent_smooth': 'uniform_01_constant',
'child_smooth': 'gaussian_01_constant',
'child_nslist': None
}]
#
# mulcov_table
mulcov_table = [{
'covariate': 'sex',
'type': 'rate_value',
'effected': 'omega',
'group': 'world',
'smooth': 'uniform_01_constant'
}]
#
# option_table
option_table = [{
'name': 'parent_node_name',
'value': 'world'
}, {
'name': 'ode_step_size',
'value': '10.0'
}, {
'name': 'random_seed',
'value': '0'
}, {
'name': 'rate_case',
'value': 'iota_pos_rho_pos'
}, {
'name': 'tolerance',
'value': '1e-8'
}, {
'name': 'max_num_iter',
'value': '100'
}, {
'name': 'print_level',
'value': '0'
}, {
'name': 'derivative_test',
'value': 'second-order'
}]
# avgint_table
avgint_table = []
row = {
'integrand': 'mtother',
'weight': 'constant',
'age_lower': 0.0,
'age_upper': 100.0,
'time_lower': 1990.0,
'time_upper': 2010.0,
'sex': 0.5,
'subgroup': 'world',
}
row['node'] = 'north_america'
avgint_table.append(copy.copy(row))
row['node'] = 'united_states'
avgint_table.append(copy.copy(row))
row['node'] = 'canada'
row['meas_value'] = 0.5e-5
avgint_table.append(copy.copy(row))
# nslist_table
nslist_table = dict()
#
# ----------------------------------------------------------------------
# subgroup_table
subgroup_table = [{'subgroup': 'world', 'group': 'world'}]
# ----------------------------------------------------------------------
dismod_at.create_database(file_name, age_list, time_list, integrand_table,
node_table, subgroup_table, weight_table,
covariate_table, avgint_table, data_table,
prior_table, smooth_table, nslist_table,
rate_table, mulcov_table, option_table)
# ----------------------------------------------------------------------
# Check database
# ----------------------------------------------------------------------
#
# connection
new = False
connection = dismod_at.create_connection(file_name, new)
#
# age_table
tbl_name = 'age'
col_name = ['age']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
col_list = column_as_list(row_list, 0)
assert col_list == age_list
#
# time_table
tbl_name = 'time'
col_name = ['time']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
col_list = column_as_list(row_list, 0)
assert col_list == time_list
#
# intergrand_table
tbl_name = 'integrand'
col_name = ['integrand_name']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['mtother']]
assert row_list == check_list
#
# weight_table
tbl_name = 'weight'
col_name = ['weight_name', 'n_age', 'n_time']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['constant', 1, 1]]
assert row_list == check_list
#
# weight_grid_table
tbl_name = 'weight_grid'
col_name = ['weight_id', 'age_id', 'time_id', 'weight']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [[0, 0, 0, 1.0]]
assert row_list == check_list
#
# covariate_table
tbl_name = 'covariate'
col_name = ['covariate_name', 'reference']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['sex', 0.0]]
assert row_list == check_list
#
# node_table
tbl_name = 'node'
col_name = ['node_name', 'parent']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['world', None], ['north_america', 0], ['united_states', 1],
['canada', 1]]
assert row_list == check_list
#
# data_table
tbl_name = 'data'
col_name = ['integrand_id', 'node_id', 'meas_value', 'meas_std']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [[0, 1, 1.0e-5, 1e-5], [0, 2, 1.5e-5, 1e-5],
[0, 3, 0.5e-5, 1e-5]]
assert row_list == check_list
#
# prior_table
tbl_name = 'prior'
col_name = ['prior_name', 'density_id', 'lower', 'upper']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['zero', 0, 0.0, 0.0], ['one', 0, 1.0, 1.0],
['uniform_01', 0, 0.0, 1.0], ['gaussian_01', 1, None, None],
['log_gaussian', 4, None, None]]
assert row_list == check_list
#
# smooth_table
tbl_name = 'smooth'
col_name = [
'smooth_name', 'n_age', 'n_time', 'mulstd_value_prior_id',
'mulstd_dage_prior_id', 'mulstd_dtime_prior_id'
]
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['uniform_01_constant', 1, 1, None, None, None],
['gaussian_01_constant', 1, 1, None, None, None]]
assert row_list == check_list
#
# smooth_grid_table
tbl_name = 'smooth_grid'
col_name = [
'smooth_id', 'age_id', 'time_id', 'value_prior_id', 'dage_prior_id',
'dtime_prior_id'
]
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [[0, 0, 0, 2, None, None], [1, 0, 0, 3, None, None]]
assert row_list == check_list
#
# rate_table
tbl_name = 'rate'
col_name = ['rate_name', 'parent_smooth_id', 'child_smooth_id']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['pini', 0, 1], ['iota', 0, 1], ['rho', 0, 1], ['chi', 0, 1],
['omega', 0, 1]]
assert row_list == check_list
#
# mulcov_table
tbl_name = 'mulcov'
col_name = [
'mulcov_type', 'rate_id', 'integrand_id', 'covariate_id', 'group_id',
'subgroup_smooth_id', 'group_smooth_id'
]
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['rate_value', 4, None, 0, 0, None, 0]]
assert row_list == check_list
#
# option_table
tbl_name = 'option'
col_name = ['option_name', 'option_value']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [['parent_node_name', 'world'], ['ode_step_size', '10.0'],
['random_seed', '0'], ['rate_case', 'iota_pos_rho_pos'],
['tolerance', '1e-8'], ['max_num_iter', '100'],
['print_level', '0'], ['derivative_test', 'second-order']]
assert row_list == check_list
#
# avgint_table
tbl_name = 'avgint'
col_name = ['integrand_id', 'node_id', 'age_lower', 'age_upper']
row_list = dismod_at.get_row_list(connection, tbl_name, col_name)
check_list = [[0, 1, 0.0, 100.0], [0, 2, 0.0, 100.0], [0, 3, 0.0, 100.0]]
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('create_database: OK')
def plot_data_fit(
# BEGIN syntax
# n_fit_dict = plot_data_fit(
database=None,
pdf_file=None,
plot_title=None,
max_plot=None,
integrand_list=None,
# )
# END syntax
):
assert not database is None
assert not pdf_file is None
#
# tables
new = False
connection = dismod_at.create_connection(database, new)
tables = dict()
for name in [
'data',
'data_subset',
'integrand',
'fit_data_subset',
]:
tables[name] = dismod_at.get_table_dict(connection, name)
connection.close()
#
# pdf
pdf = matplotlib.backends.backend_pdf.PdfPages(pdf_file)
#
# integrand_list
if integrand_list is None:
integrand_list = list()
for row in tables['integrand']:
integrand_name = row['integrand_name']
if not integrand_name.startswith('mulcov_'):
integrand_list.append(integrand_name)
#
n_fit_dict = dict()
for integrand_name in integrand_list:
#
# integrand_id
integrand_id = None
for (row_id, row) in enumerate(tables['integrand']):
if row['integrand_name'] == integrand_name:
integrand_id = row_id
if integrand_id is None:
msg = f'Cannot find {integrand_name} in integrand table for '
msg += database
assert False, msg
#
# info_list
info_list = list()
#
# subset_id, subset_row
for (subset_id, subset_row) in enumerate(tables['data_subset']):
#
# data_row
data_id = subset_row['data_id']
data_row = tables['data'][data_id]
#
if data_row['integrand_id'] == integrand_id:
#
# meas_value
meas_value = data_row['meas_value']
#
# hold_out
hold_out = data_row['hold_out']
#
# age
age = (data_row['age_lower'] + data_row['age_upper']) / 2.0
#
# time
time = (data_row['time_lower'] + data_row['time_upper']) / 2.0
#
# avg_integreand, weighted_residual
row = tables['fit_data_subset'][subset_id]
avg_integrand = row['avg_integrand']
weighted_residual = row['weighted_residual']
#
# info_list
info = {
'meas_value': meas_value,
'model': avg_integrand,
'residual': weighted_residual,
'hold_out': hold_out,
'index': len(info_list),
'age': age,
'time': time,
}
info_list.append(info)
#
# n_point
n_point = len(info_list)
#
if n_point == 0:
#
# n_hold_out
n_hold_out = 0
else:
# numpy_info
keys = info_list[0].keys()
numpy_info = dict()
for key in keys:
vector = numpy.zeros(n_point, dtype=float)
for i in range(n_point):
vector[i] = info_list[i][key]
numpy_info[key] = vector
#
# hold_out, not_hold_out, n_hold_out
hold_out = (numpy_info['hold_out'] == 1)
not_hold_out = numpy.logical_not(hold_out)
n_hold_out = sum(hold_out)
#
# n_fit_dict
n_fit_dict[integrand_name] = n_point - n_hold_out
#
if n_point - n_hold_out > 1:
#
#
# y_min, y_max
d_fit = numpy_info['meas_value'][not_hold_out]
d_median = numpy.median(d_fit)
d_max = d_median * 1e+3
d_min = d_median * 1e-3
assert d_min >= 0.0
#
# r_min, r_max
r_fit = numpy_info['residual'][not_hold_out]
r_norm = numpy.linalg.norm(r_fit)
r_avg_sq = r_norm * r_norm / (n_point - n_hold_out)
r_max = 4.0 * numpy.sqrt(r_avg_sq)
r_min = -r_max
#
# subplot_list
subplot_list = ['meas_value', 'model', 'residual']
#
# numpy_info
for name in ['meas_value', 'model']:
numpy_info[name] = numpy.maximum(numpy_info[name], d_min)
numpy_info[name] = numpy.minimum(numpy_info[name], d_max)
for name in ['residual']:
numpy_info[name] = numpy.maximum(numpy_info[name], r_min)
numpy_info[name] = numpy.minimum(numpy_info[name], r_max)
#
if max_plot is None or n_point <= max_plot:
#
# n_plot
n_plot = n_point
else:
#
# n_plot
n_plot = max_plot
#
# subsample
subsample = random.sample(range(n_point), max_plot)
subsample = sorted(subsample)
#
# numpy_info
for key in numpy_info:
numpy_info[key] = numpy_info[key][subsample]
#
# hold_out, not_hold_out
hold_out = (numpy_info['hold_out'] == 1)
not_hold_out = numpy.logical_not(hold_out)
#
#
# point_size, marker_size
point_size = numpy.array(n_plot * [1])
marker_size = numpy.array(n_plot * [10])
#
for x_name in ['index', 'age', 'time']:
#
# subplot setup
fig, axes = pyplot.subplots(3, 1, sharex=True)
fig.subplots_adjust(hspace=0)
#
# x
x = numpy_info[x_name]
#
for subplot_index in range(3):
# sp
sp = pyplot.subplot(3, 1, subplot_index + 1)
#
# name, y
name = subplot_list[subplot_index]
y = numpy_info[name]
#
# ylabel
pyplot.ylabel(name)
#
# clip_list, limit_list
if name == 'residual':
clip_list = [r_min, r_max]
limit_list = [1.1 * r_min, 1.1 * r_max]
else:
pyplot.yscale('log')
clip_list = [d_min, d_max]
limit_list = [0.9 * d_min, 1.1 * d_max]
#
# ylim
pyplot.ylim(limit_list[0], limit_list[1])
#
# clipped, not_clipped
clipped = (y == clip_list[0])
clipped = numpy.logical_or(clipped, (y == clip_list[1]))
not_clipped = numpy.logical_not(clipped)
#
green_point = numpy.logical_and(hold_out, not_clipped)
green_marker = numpy.logical_and(hold_out, clipped)
black_point = numpy.logical_and(not_hold_out, not_clipped)
red_marker = numpy.logical_and(not_hold_out, clipped)
#
# plot green points
size = point_size[green_point]
pyplot.scatter(x[green_point],
y[green_point],
marker='.',
color='green',
s=size)
#
# plot green marker
size = marker_size[green_marker]
pyplot.scatter(x[green_marker],
y[green_marker],
marker='+',
color='green',
s=size)
#
# plot black points
size = point_size[black_point]
pyplot.scatter(x[black_point],
y[black_point],
marker='.',
color='black',
s=size)
#
# plot red marker
size = marker_size[red_marker]
pyplot.scatter(x[red_marker],
y[red_marker],
marker='+',
color='red',
s=size)
if name == 'residual':
y = 0.0
pyplot.axhline(y,
linestyle='solid',
color='black',
alpha=0.3)
if subplot_index == 0:
if plot_title is None:
pyplot.title(integrand_name)
else:
pyplot.title(plot_title + ': ' + integrand_name)
# x-axis label
pyplot.xlabel(x_name)
#
# save plot
pdf.savefig(fig)
pyplot.close(fig)
# end of pages in pdf file
pdf.close()
return n_fit_dict
def get_limit_var_table(database):
#
# table
new = False
connection = dismod_at.create_connection(database, new)
table = dict()
for tbl_name in [
'node',
'option',
'prior',
'smooth',
'smooth_grid',
'var',
]:
table[tbl_name] = dismod_at.get_table_dict(connection, tbl_name)
connection.close()
#
# parent_node_id
parent_node_id = None
parent_node_name = None
for row in table['option']:
if row['option_name'] == 'parent_node_id':
parent_node_id = int(row['option_value'])
if row['option_name'] == 'parent_node_name':
parent_node_name = row['option_value']
assert not (parent_node_id is None and parent_node_name is None)
if parent_node_id is None:
for (node_id, row) in enumerate(table['node']):
if row['node_name'] == parent_node_name:
parent_node_id = node_id
if parent_node_id is None:
msg = f'parent_node_name = {parent_node_name} is not in node table'
assert False, msg
#
# random_seed
random_seed = 0
for row in table['option']:
if row['option_name'] == 'random_seed':
random_seed = int(row['option_value'])
if random_seed != 0:
random.seed(random_seed)
#
# result
limit_var_table = list()
#
# var_id, var_row
for (var_id, var_row) in enumerate(table['var']):
#
# var_type, node_id, subgroup_id, smooth_id
var_type = var_row['var_type']
node_id = var_row['node_id']
subgoup_id = var_row['subgroup_id']
group_id = var_row['group_id']
subgroup_id = var_row['subgroup_id']
smooth_id = var_row['smooth_id']
assert (subgroup_id is None) or (group_id is None)
#
# smooth_row
smooth_row = table['smooth'][smooth_id]
#
# value_prior_id, const_value
value_prior_id = None
const_value = None
if var_type.startswith('mulstd_'):
value_prior_id = smooth_row['mulstd_value_prior_id']
else:
# age_id, time_id
age_id = var_row['age_id']
time_id = var_row['time_id']
#
# grid_row
for grid_row in table['smooth_grid']:
#
# match
# does grid_row match this variable
match = True
match = match and grid_row['smooth_id'] == smooth_id
match = match and grid_row['age_id'] == age_id
match = match and grid_row['time_id'] == time_id
if match:
#
# value_prior_id, const_value
value_prior_id = grid_row['value_prior_id']
const_value = grid_row['const_value']
#
# lower, upper, mean
if value_prior_id is None:
assert const_value is not None
lower = const_value
upper = const_value
mean = const_value
else:
assert const_value is None
prior_row = table['prior'][value_prior_id]
lower = prior_row['lower']
upper = prior_row['upper']
mean = prior_row['mean']
#
row = {'lower': lower, 'mean': mean, 'upper': upper}
limit_var_table.append(row)
#
return limit_var_table
def smooth_grid_table():
import dismod_at
import copy
import collections
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
cursor = connection.cursor()
#
# create smooth table
ptype = 'integer primary key'
col_name2type = collections.OrderedDict([
('smooth_name', 'text'), ('n_age', 'integer'), ('n_time', 'integer'),
('mulstd_value_prior_id', 'integer'),
('mulstd_dage_prior_id', 'integer'),
('mulstd_dtime_prior_id', 'integer')
])
col_name = list(col_name2type.keys())
col_type = list(col_name2type.values())
row_list = [['constant', 1, 1, 1, 1, 1], ['age_only', 3, 1, 1, 1, 1],
['time_only', 1, 2, 1, 1, 1], ['bilinear', 3, 2, 1, 1, 1]]
tbl_name = 'smooth'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
# smooth_grid table column names
col_name2type = collections.OrderedDict([('smooth_id', 'integer'),
('age_id', 'integer'),
('time_id', 'integer'),
('value_prior_id', 'integer'),
('dage_prior_id', 'integer'),
('dtime_prior_id', 'integer'),
('const_value', 'real')])
col_name = list(col_name2type.keys())
col_type = list(col_name2type.values())
#
# smooth_grid table values
row_list = list()
default = [
3, # smooth_id (smooth_id == 3 is bilinear)
None, # age_id (age_id index is 1 in default)
None, # time_id (time_id index is 2 in default)
1, # value_prior_id
2, # dage_prior_id
3, # dtime_prior_id
None # const_value
]
age_time_list = list()
for age_id in [0, 1, 2]: # n_age is 3
for time_id in [0, 1]: # n_time is 2
default[1] = age_id
default[2] = time_id
row = copy.copy(default)
if age_id == 2:
row[4] = None # dage_prior_id null for this case
if time_id == 1:
row[5] = None # dtime_prior_id null for this case
row_list.append(row)
age_time_list.append((age_id, time_id))
#
# write the table
tbl_name = 'smooth_grid'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
#
# check values in the table
columns = ','.join(col_name)
columns = 'smooth_grid_id,' + columns
cmd = 'SELECT ' + columns + ' FROM smooth_grid'
cmd += ' JOIN smooth USING(smooth_id) '
cmd += ' WHERE smooth_name = "bilinear"'
count = 0
cursor = connection.cursor()
for row in cursor.execute(cmd):
assert len(row) == 8
assert row[0] == count
assert row[1] == 3
assert row[2] == age_time_list[count][0]
assert row[3] == age_time_list[count][1]
assert row[4] == 1
if row[2] == 2:
assert row[5] == None
else:
assert row[5] == 2
if row[3] == 1:
assert row[6] == None
else:
assert row[6] == 3
assert row[7] == None
count += 1
assert count == len(row_list)
#
connection.close()
print('smooth_grid_table: OK')
usage = 'python3 ' + test_program + '\n'
usage += 'where python3 is the python 3 program on your system\n'
usage += 'and working directory is the dismod_at distribution directory\n'
sys.exit(usage)
#
# import dismod_at
local_dir = os.getcwd() + '/python'
if (os.path.isdir(local_dir + '/dismod_at')):
sys.path.insert(0, local_dir)
import dismod_at
#
# change into the build/example/user directory
if not os.path.exists('build/example/user'):
os.makedirs('build/example/user')
os.chdir('build/example/user')
#
# connection
new = True
database = 'example.db'
connection = dismod_at.create_connection(database, new)
#
# file_name
file_name = dismod_at.connection_file(connection)
#
# check
assert os.path.samefile(file_name, database)
#
print('connection_file.py: OK')
sys.exit(0)
# END PYTHON
prior_table, smooth_table, nslist_table,
rate_table, mulcov_table, option_table)
# ===========================================================================
file_name = 'example.db'
example_db(file_name)
#
program = '../../devel/dismod_at'
dismod_at.system_command_prc([program, file_name, 'init'])
dismod_at.system_command_prc([program, file_name, 'fit', 'both'])
dismod_at.system_command_prc([program, file_name, 'predict', 'fit_var'])
# -----------------------------------------------------------------------
# connect to database
new = False
connection = dismod_at.create_connection(file_name, new)
predict_table = dismod_at.get_table_dict(connection, 'predict')
avgint_table = dismod_at.get_table_dict(connection, 'avgint')
node_table = dismod_at.get_table_dict(connection, 'node')
#
# check that all the avgint_table values were predicted (no subsetting)
assert len(predict_table) == 3
#
# S(a) = exp( - iota * a )
iota_canada = math.exp(canada_effect) * iota_north_america
iota_united_states = math.exp(united_states_effect) * iota_north_america
S_north_america = math.exp(-iota_north_america * 50.0)
S_canada = math.exp(-iota_canada * 50.0)
S_united_states = math.exp(-iota_united_states * 50.0)
truth = {
'north_america': S_north_america,
def data_table():
import dismod_at
import copy
import collections
#
file_name = 'example.db'
new = True
connection = dismod_at.create_connection(file_name, new)
#
col_name2type = collections.OrderedDict([
# required columns
('data_name', 'text'),
('integrand_id', 'integer'),
('density_id', 'integer'),
('node_id', 'integer'),
('weight_id', 'integer'),
('hold_out', 'integer'),
('meas_value', 'real'),
('meas_std', 'real'),
('age_lower', 'real'),
('age_upper', 'real'),
('time_lower', 'real'),
('time_upper', 'real'),
# covariates
('x_sex', 'real'),
('x_income', 'real'),
# comments
('c_data_source', 'text')
])
col_name = list(col_name2type.keys())
col_type = list(col_name2type.values())
row_list = [[
'one', # data_name
1, # integrand_id
0, # density_id
3, # node_id
4, # weight_id
0, # hold_out
1e-4, # meas_value
1e-5, # meas_std
10.0, # age_lower
90.0, # age_upper
2000., # time_lower
2005., # time_upper
0.5, # x_sex
1000., # x_income
'www.healthdata.org' # c_data_source
]]
# create the data table
tbl_name = 'data'
dismod_at.create_table(connection, tbl_name, col_name, col_type, row_list)
# ----------------------------------------------------------------------
# include primary key in test
check_name = [tbl_name + '_id'] + col_name
check_list = list()
for i in range(len(row_list)):
check_list.append([i] + row_list[i])
#
row_list = dismod_at.get_row_list(connection, tbl_name, check_name)
assert row_list == check_list
# ----------------------------------------------------------------------
connection.close()
print('data_table: OK')