def main():
args = main_parser()
design_location: str = args.design
scales_location: str = args.scales
disease_location: str = args.disease
# Grab data from disk
try:
with open(design_location) as design_file, \
open(scales_location) as scales_file:
design_data: List[List[str]] = list(csv.reader(design_file))
scales_data: List[List[str]] = list(csv.reader(scales_file))
design_names: List[str] = design_data[0]
design_data.pop(0)
scales_names: List[str] = scales_data[0]
scales_data.pop(0)
except IOError as error:
print("File system error: " + str(error.msg) + " when operating on " + str(error.filename))
sys.exit(1)
# Check that the designs are the right size to scale and all scale vars have been defined
n_design_columns = len(design_names)
n_scale_columns = len(scales_names)
n_design_points = len(design_data)
# We need at least 2 columns for a valid design
if n_design_columns < 2:
print("Design has no input parameters!")
sys.exit(1)
# Check the size of the inputs (+/- 1 accounts for repeat column)
if n_scale_columns < (n_design_columns - 1):
print("Variable limits file does not have enough entries")
sys.exit(1)
if n_design_columns < (n_scale_columns + 1):
print("Design potentially incomplete")
sys.exit(1)
# Check that all the scales are defined
if not all([x in scales_names for x in design_names[:-1]]):
print("scale limits file does not define the design completely")
sys.exit(1)
# Just in case the columns are in a different order...
col_indices = [next(index for index, item in enumerate(scales_names) if item == x) for x in design_names[:-1]]
minimums = [float(x) for x in scales_data[0]]
maximums = [float(x) for x in scales_data[1]]
# Additional keys for the database components
key_names: List[str] = \
[
".design_index"
]
# Epidemiological parameteres
epidemiology_headers: List[str] = key_names + design_names
# Disease parameters
user_var_names: List[str] = [f".{x}" for x in design_names[:-1]]
disease_headers: List[str] = key_names + user_var_names + \
[
"beta[2]",
"beta[3]",
"progress[1]",
"progress[2]",
"progress[3]",
"repeats"
]
n_epidemiology_columns = len(epidemiology_headers)
n_disease_columns = len(disease_headers)
epidemiology_table = [[''] * n_epidemiology_columns] * n_design_points
disease_table = [[''] * n_disease_columns] * n_design_points
# Iterate down the design table and rescale as needed
for i, row in enumerate(design_data):
hypercube = [float(x) for x in row[:-1]]
hyper_01 = [(x / 2.0) + 0.5 for x in hypercube]
hyper_scale = [(x * (maximums[col_indices[i]] - minimums[col_indices[i]])) + minimums[col_indices[i]]
for i, x in enumerate(hyper_01)]
metawards_params = list(transform_epidemiological_to_disease(hyper_scale[col_indices[0]],
hyper_scale[col_indices[1]],
hyper_scale[col_indices[2]]))
disease_row = [''] * n_disease_columns
epidemiology_row = [''] * n_epidemiology_columns
# Design keys
epidemiology_row[0:len(key_names)] = [str(i)]
disease_row[0:len(key_names)] = [str(i)]
# Hypercubes
epidemiology_row[len(key_names):len(key_names) + len(hyper_scale)] = [str(x) for x in hyper_scale]
disease_row[len(key_names):len(key_names) + len(hypercube)] = [str(x) for x in hypercube]
# Metawards parameters
epidemiology_row[len(key_names) + len(hyper_scale):] = str(row[-1])
disease_row[len(key_names) + len(hyper_scale):] = [str(x) for x in metawards_params] + [str(row[-1])]
epidemiology_table[i] = epidemiology_row
disease_table[i] = disease_row
epidemiology_table.insert(0, epidemiology_headers)
disease_table.insert(0, disease_headers)
# Write output
str_mode = 'x'
if args.force:
str_mode = "w"
try:
with open(disease_location, str_mode, newline='') as disease_file:
writer = csv.writer(disease_file)
writer.writerows(disease_table)
except FileExistsError:
print("Disease table already exists, use -f to force overwriting")
sys.exit(1)
except IOError as error:
print("File system error: " + str(error.msg) + " when operating on " + str(error.filename))
sys.exit(1)
if args.epidemiology:
in_name, in_ext = os.path.splitext(disease_location)
e_name = in_name + "_epidemiology.csv"
try:
with open(e_name, str_mode, newline='') as e_file:
writer = csv.writer(e_file)
writer.writerows(epidemiology_table)
except FileExistsError:
print("Epidemiology table already exists, use -f to force overwriting")
sys.exit(1)
except IOError as error:
print("File system error: " + str(error.msg) + " when operating on " + str(error.filename))
sys.exit(1)
print("Done! See output in " + str(disease_location))
sys.exit(0)
def main():
argv = main_parser()
# Query the parser - is getattr() safer?
in_location = argv.input
out_location = argv.output
mode_str = "x"
if argv.force:
mode_str = "w"
print(
"force option passed, disease matrix will be over-written if it exists"
)
# Touch the file system
try:
with open(in_location) as epidemiology_file, \
open(out_location, mode_str) as disease_file:
epidemiology_file, epidemiology_header_names = load_csv(
epidemiology_file)
# Only pass the first three fields to the disease builder
# TODO: Do this by name rather than assuming order?
disease_matrix = np.zeros((epidemiology_file.shape[0], 8))
for i, _ in enumerate(disease_matrix):
row = epidemiology_file[i]
new_row = np.zeros(8)
new_row[0:5] = np.asarray(
transform_epidemiological_to_disease(
row[0], row[1], row[2]))
# Pass scale rates and repeats through from epidemiology file
new_row[5] = row[3]
new_row[6] = row[4]
new_row[7] = row[5]
disease_matrix[i] = new_row
head_names: List[str] = [
"beta[2]", "beta[3]", "progress[1]", "progress[2]",
"progress[3],.scale_rate[1],.scale_rate[2],repeats"
]
header = ','.join(head_names)
np.savetxt(fname=disease_file,
X=disease_matrix,
fmt="%f",
delimiter=",",
header=header,
comments='')
# TODO: Extra design variables need to go somewhere
except FileExistsError:
print("Output already exists, use -f to force overwriting")
sys.exit(1)
except FileNotFoundError as error:
print(str(error.filename) + " not found.")
sys.exit(1)
except IOError as error:
print("File system error: " + str(error.msg) + " when operating on " +
str(error.filename))
sys.exit(1)
print("Done! See output at " + str(out_location))
sys.exit(0)
def main():
argv = main_parser()
# Query the parser - is getattr() safer?
in_location: str = argv.input
out_location: str = argv.output
mode_str: str = "x"
if argv.force:
mode_str = "w"
print("force option passed, disease matrix will be over-written if it exists")
# Touch the file system
try:
with open(in_location) as epidemiology_file_handle, \
open(out_location, mode_str, newline='') as disease_file:
epidemiology_file: List[List[str]] = list(csv.reader(epidemiology_file_handle))
epidemiology_header_names: List[str] = epidemiology_file[0]
epidemiology_file.pop(0)
# Turn the original design variables into custom variables for metawards
# Ignore the repeats column
e_names: List[str] = [f".{x}" for x in epidemiology_header_names[0:-1]]
# Additional keys
key_names: List[str] = \
[
".design_index"
]
# Disease parameters
head_names: List[str] = key_names + e_names + \
[
"beta[2]",
"beta[3]",
"progress[1]",
"progress[2]",
"progress[3]",
"repeats"
]
# Only pass the first three fields to the disease builder
# TODO: Do this by name rather than assuming order?
disease_matrix: List[List[str]] = [[''] * len(head_names)] * len(epidemiology_file)
for i, _ in enumerate(disease_matrix):
row: List[Any] = epidemiology_file[i]
# Data-type conversion
row[:-1] = [float(x) for x in row[:-1]]
row[-1] = int(float(row[-1]))
new_row: List[Any] = [0] * len(head_names)
# Create a unique design key
write_pos: int = len(key_names)
new_row[0:write_pos] = [i]
# Pass scale rates and repeats through from epidemiology file
new_row[write_pos:write_pos + len(e_names)] = row[0:-1]
new_row[-1] = row[-1]
write_pos += len(e_names)
# Transform the other parameters to disease parameters
new_row[write_pos:len(head_names) - 1] = \
list(transform_epidemiological_to_disease(row[0], row[1], row[2]))
# Stringify it for csv writing
disease_matrix[i] = [str(x) for x in new_row]
disease_matrix.insert(0, head_names)
writer = csv.writer(disease_file)
writer.writerows(disease_matrix)
except FileExistsError:
print("Output already exists, use -f to force overwriting")
sys.exit(1)
except FileNotFoundError as error:
print(str(error.filename) + " not found.")
sys.exit(1)
except IOError as error:
print("File system error: " + str(error.msg) + " when operating on " + str(error.filename))
sys.exit(1)
print("Done! See output at " + str(out_location))
sys.exit(0)
def main():
args = main_parser()
design_location: str = args.design
scales_location: str = args.scales
disease_location: str = args.disease
# Grab data from disk
try:
with open(design_location) as design_file, \
open(scales_location) as scales_file:
design_data: List[List[str]] = list(csv.reader(design_file))
scales_data: List[List[str]] = list(csv.reader(scales_file))
design_names: List[str] = design_data[0]
design_data.pop(0)
scales_names: List[str] = scales_data[0]
scales_data.pop(0)
except IOError as error:
print("File system error: " + str(error.msg) + " when operating on " +
str(error.filename))
sys.exit(1)
# Check that the designs are the right size to scale and all scale vars have been defined
n_design_columns = len(design_names)
n_scale_columns = len(scales_names)
n_design_points = len(design_data)
# We need at least 2 columns for a valid design
if n_design_columns < 2:
print("Design has no input parameters!")
sys.exit(1)
# Check the size of the inputs (+/- 1 accounts for repeat column)
if n_scale_columns < (n_design_columns - 1):
print("Variable limits file does not have enough entries")
sys.exit(1)
if n_design_columns < (n_scale_columns + 1):
print("Design potentially incomplete")
sys.exit(1)
# Check that all the scales are defined
if not all([x in scales_names for x in design_names[:-1]]):
print("scale limits file does not define the design completely")
sys.exit(1)
# Just in case the columns are in a different order...
col_indices = [
next(index for index, item in enumerate(scales_names) if item == x)
for x in design_names[:-1]
]
minimums = [float(x) for x in scales_data[0]]
maximums = [float(x) for x in scales_data[1]]
# Additional keys for the database components
key_names: List[str] = \
[
".design_index"
]
# Epidemiological parameteres
epidemiology_headers: List[str] = key_names + design_names
# Disease parameters
user_var_names: List[str] = [f".{x}" for x in design_names[:-1]]
disease_headers: List[str] = key_names + user_var_names + \
[
"beta[2]",
"beta[3]",
"progress[1]",
"progress[2]",
"progress[3]",
"repeats"
]
n_epidemiology_columns = len(epidemiology_headers)
n_disease_columns = len(disease_headers)
epidemiology_table = [[''] * n_epidemiology_columns] * n_design_points
disease_table = [[''] * n_disease_columns] * n_design_points
# Iterate down the design table and rescale as needed
for i, row in enumerate(design_data):
hypercube = [float(x) for x in row[:-1]]
hyper_01 = [(x / 2.0) + 0.5 for x in hypercube]
hyper_scale = [
(x * (maximums[col_indices[i]] - minimums[col_indices[i]])) +
minimums[col_indices[i]] for i, x in enumerate(hyper_01)
]
metawards_params = list(
transform_epidemiological_to_disease(hyper_scale[col_indices[0]],
hyper_scale[col_indices[1]],
hyper_scale[col_indices[2]]))
disease_row = [''] * n_disease_columns
epidemiology_row = [''] * n_epidemiology_columns
# Design keys
epidemiology_row[0:len(key_names)] = [str(i)]
disease_row[0:len(key_names)] = [str(i)]
# Hypercubes
epidemiology_row[len(key_names):len(key_names) +
len(hyper_scale)] = [str(x) for x in hyper_scale]
disease_row[len(key_names):len(key_names) +
len(hypercube)] = [str(x) for x in hypercube]
# Metawards parameters
epidemiology_row[len(key_names) + len(hyper_scale):] = str(row[-1])
disease_row[len(key_names) +
len(hyper_scale):] = [str(x) for x in metawards_params
] + [str(row[-1])]
epidemiology_table[i] = epidemiology_row
disease_table[i] = disease_row
epidemiology_table.insert(0, epidemiology_headers)
disease_table.insert(0, disease_headers)
# Write output disease table
str_mode = 'x'
if args.force:
str_mode = "w"
try:
with open(disease_location, str_mode, newline='') as disease_file:
writer = csv.writer(disease_file)
writer.writerows(disease_table)
except FileExistsError:
print("Disease table already exists, use -f to force overwriting")
sys.exit(1)
except IOError as error:
print("File system error: " + str(error.msg) + " when operating on " +
str(error.filename))
sys.exit(1)
# Write epidemiology table
if args.epidemiology:
in_name, in_ext = os.path.splitext(disease_location)
e_name = in_name + "_epidemiology.csv"
try:
with open(e_name, str_mode, newline='') as e_file:
writer = csv.writer(e_file)
writer.writerows(epidemiology_table)
except FileExistsError:
print(
"Epidemiology table already exists, use -f to force overwriting"
)
sys.exit(1)
except IOError as error:
print("File system error: " + str(error.msg) +
" when operating on " + str(error.filename))
sys.exit(1)
# Write database
if args.remote:
p = ConfigParser()
p.read(args.remote)
params: dict = {}
if p.has_section("postgresql"):
items = p.items("postgresql")
params = {v[0]: v[1] for v in items}
# Add a timestamp
from datetime import datetime
extra: str = datetime.now().strftime('%Y%m%d%H%M%S')
params["database"] = params["ident"] + f"_{extra}"
# Connect to the "master" database, then create one for this run
# NOTE: No finally block -> leave the connection alone
connection_params: dict = select_dictionary_keys(
params, ["host", "user", "password"])
connection_params["database"] = "postgres"
try:
maintenance_connection = psg.connect(**connection_params)
# A strange quirk of psycopg2 is that 'create database' will fail every time unless autocommit is on
maintenance_connection.autocommit = True
cur = maintenance_connection.cursor()
cur.execute(f"CREATE DATABASE {params['database']}")
maintenance_connection.autocommit = False
except psg.Error as err:
if maintenance_connection is not None:
maintenance_connection.close()
print(
"Problem with creating the remote database, consider a local solution instead?"
)
sys.exit(1)
# Connect to the new database and attempt to create the design table
# Any exceptions before the commit() will cause a rollback
# NOTE: Don't get lazy and use "with" / context managers here, we NEED to see the right exceptions
connection_params: dict = select_dictionary_keys(
params, ["host", "user", "password", "database"])
new_database = None
try:
# Get a connection
new_database = psg.connect(**connection_params)
cur = new_database.cursor()
# Make design table
qstring = make_design_table_schema("design_index",
design_names[:-1], "design")
cur.execute(qstring)
# Send design data
for i, row in enumerate(design_data):
data = [float(x) for x in row[:-1]]
qstring = f"INSERT INTO design ({','.join(['design_index'] + design_names[:-1])}) " \
f"VALUES ({','.join(['%s']*(len(design_names[:-1])+1))})"
cur.execute(qstring, tuple([i] + data))
# Make index table of design index and repeat count
# Finally commit
new_database.commit()
except psg.Error as err:
# if we get here we need to drop the new database
if new_database is not None:
new_database.close()
# If this fails the there is nothing we can do
maintenance_connection.autocommit = True
cur = maintenance_connection.cursor()
cur.execute(
sql.SQL("DROP DATABASE IF EXISTS {};").format(
sql.Identifier(params["database"])))
maintenance_connection.autocommit = False
sys.exit(1)
finally:
if new_database is not None:
new_database.close()
if maintenance_connection is not None:
maintenance_connection.close()
# Make database
make_memory_database([key_names[0][1:]] + design_names, design_data,
None)
# Create a valid URI to use the SQLite access rights
data_base_file_name: str = os.path.join(
os.path.dirname(disease_location), "sql_wards.dat")
_sql_file_name = data_base_file_name
fixed_path = os.path.abspath(data_base_file_name).replace("\\", "/")
db_uri = f"file:{fixed_path}?mode=rw"
print("Done! See output in " + str(disease_location))
sys.exit(0)