import pickle
import shutil
import tempfile
import subprocess
from pkg_resources import resource_filename
import numpy as np
from pyabm.file_io import read_single_band_raster
from chitwanabm import rc_params
from chitwanabm.agents import World
logger = logging.getLogger(__name__)
rcParams = rc_params.get_params()
def main():
ch = logging.StreamHandler()
ch.setLevel(logging.INFO)
log_console_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
datefmt='%I:%M:%S%p')
ch.setFormatter(log_console_formatter)
logger.addHandler(ch)
world = generate_world()
if world == 1:
logger.critical("Problem generating world")
return 1
def read_CVFS_data(textfile, key_field):
def main():
parser = argparse.ArgumentParser(description='Run the chitwanabm agent-based model (ABM).')
parser.add_argument('--rc', dest="rc_file", metavar="RC_FILE", type=str, default=None,
help='Path to a rc file to initialize a model run with custom parameters')
parser.add_argument('--log', metavar="LEVEL", type=str, default="info",
help='The logging threshold for logging to the console')
parser.add_argument('--logf', metavar="LEVEL", type=str,
default="debug", help='The logging threshold for logging to the log file')
parser.add_argument('--tail', dest='tail', action='store_const',
const=True, default=False, help='Tail the logfile with the default tail application specified in the rc parameters')
parser.add_argument('--output-path', dest='output_path', default=None,
help='Path in which to store the model output (overrides any value set in the rc-file)')
parser.add_argument('--run-id', dest='run_ID_number', default=None,
help='Run ID number (automatically generated if it is not specified)')
args = parser.parse_args()
# Setup logging according to the desired levels
ch_level = getattr(logging, args.log.upper(), None)
if not isinstance(ch_level, int):
logger.critical('Invalid log level: %s' %args.log)
root_logger.handlers[1].setLevel(ch_level)
fh_level = getattr(logging, args.logf.upper(), None)
if not isinstance(fh_level, int):
logger.critical('Invalid log level: %s' %args.logf)
root_logger.handlers[0].setLevel(fh_level)
# Wait to load rcParams until here as logging statements are often
# triggered when the rcParams are loaded.
from chitwanabm import rc_params
# Make sure the rc_params are setup before loading any other chitwanabm
# modules, so that they will all take the default params including any that
# might be specified in user_rc_file
root_logger.handlers[0].setLevel(fh_level)
rc_params.load_default_params('chitwanabm')
if not args.rc_file==None and not os.path.exists(args.rc_file):
logger.critical('Custom rc file %s does not exist'%args.rc_file)
rc_params.initialize('chitwanabm', args.rc_file)
global rcParams
rcParams = rc_params.get_params()
from chitwanabm.initialize import generate_world
from chitwanabm.modelloop import main_loop
from pyabm.file_io import write_single_band_raster
from pyabm.utility import save_git_diff
from pyabm import __version__ as pyabm_version
from chitwanabm import __version__ as chitwanabm_version
if args.output_path != None:
scenario_path = os.path.join(args.output_path, rcParams['scenario.name'])
if not os.path.exists(args.output_path):
try:
os.mkdir(args.output_path)
except OSError:
logger.critical("Could not create output folder %s"%scenario_path)
return 1
else:
scenario_path = os.path.join(rcParams['model.resultspath'], rcParams['scenario.name'])
if not os.path.exists(scenario_path):
try:
os.mkdir(scenario_path)
except OSError:
logger.critical("Could not create scenario directory %s"%scenario_path)
return 1
if args.run_ID_number == None:
# Get machine hostname to print it in the results file and use in the
# run_ID_number.
hostname = socket.gethostname()
# The run_ID_number provides an ID number (built from the start time
# and machine name) to uniquely identify this model run.
run_ID_number = time.strftime("%Y%m%d-%H%M%S") + '_' + hostname
results_path = os.path.join(scenario_path, run_ID_number)
try:
os.mkdir(results_path)
except OSError:
logger.critical("Could not create results directory %s"%results_path)
return 1
else:
run_ID_number = args.run_ID_number
results_path = os.path.join(scenario_path, run_ID_number)
try:
os.mkdir(results_path)
except OSError:
logger.critical("Could not create results directory %s"%results_path)
return 1
# Setup a special logger to log demographic events while the model is
# running (births, migrations, deaths, marriages, etc.)
person_event_log_file_path = os.path.join(results_path, "person_events.log")
person_event_log_file = open(person_event_log_file_path, mode='w')
person_event_log_header = ",".join(["time", "event",
"pid", "hid", "nid", "rid", "gender", "age",
"ethnicity", "mother_id", "father_id",
"spouseid", "marrtime", "schooling",
"num_children", "alive", "is_away",
"is_initial_agent", "is_in_migrant",
"mother_num_children",
"mother_years_schooling",
"mother_work",
"father_years_schooling","father_work",
"parents_contracep"])
person_event_log_file.write(person_event_log_header + '\n')
person_event_log_file.close()
person_event_fh = logging.FileHandler(os.path.join(results_path, "person_events.log"), mode='a')
person_event_fh.setLevel(logging.INFO)
person_event_fh.setFormatter(logging.Formatter('%(modeltime)s,%(message)s,%(personinfo)s'))
# Setup a filter so the agent event log will contain only agent events.
class PassEventFilter(logging.Filter):
def filter(self, record):
logger_name = getattr(record, 'name', None)
return 'person_events' in logger_name
person_event_fh.addFilter(PassEventFilter())
person_event_logger = logging.getLogger('person_events')
person_event_logger.addHandler(person_event_fh)
# Now that we know the rcParams and log file path, write the temp_log
# stream to the log file in the proper output directory, and direct all
# further logging to append to that file.
log_file_path = os.path.join(results_path, "chitwanabm.log")
shutil.copyfile(temp_log_file.name, log_file_path)
temp_log_file.close()
root_logger.handlers.remove(temp_log)
temp_log.close()
os.unlink(temp_log_file.name)
new_fh = logging.FileHandler(log_file_path, mode='a')
new_fh.setLevel(fh_level)
new_fh.setFormatter(log_file_formatter)
root_logger.addHandler(new_fh)
# Dont' log agent event records to the main or console loggers
class DontPassEventFilter(logging.Filter):
def filter(self, record):
logger_name = getattr(record, 'name', None)
return 'person_events' not in logger_name
for handler in root_logger.handlers:
handler.addFilter(DontPassEventFilter())
if args.tail:
try:
subprocess.Popen([rcParams['path.tail_binary'], log_file_path], cwd=results_path)
except:
logger.warning('Error tailing model log file: %s'%sys.exc_info()[1])
# TODO: Fix the loading of a pickled world. For now, force regenerating the
# world each model run.
# if rcParams['model.reinitialize']:
# # Generate a new world (with new resampling, etc.)
# world = generate_world()
# if world == 1:
# logger.critical('Error initializing model world')
# return 1
# else:
# # Load a pickled World for use in the model.
# input_data_file = rcParams['path.input_data_file']
# file = open(input_data_file, "r")
# try:
# world = pickle.load(file)
# except IOError:
# logger.critical('Error loading world data from %s'%input_data_file)
# return 1
# Generate a new world (with new resampling, etc.)
world = generate_world()
if world == 1:
logger.critical('Error initializing model world')
return 1
# Run the model loop
start_time = time.localtime()
logger.info('Beginning model run %s'%run_ID_number)
run_results, time_strings = main_loop(world, results_path) # This line actually runs the model.
end_time = time.localtime()
logger.info('Finished model run number %s'%run_ID_number)
# Save the results to a pickled file
if rcParams['save_pickled_end_results']:
logger.info("Saving results")
pop_data_file = os.path.join(results_path, "run_results.P")
output = open(pop_data_file, 'w')
pickle.dump(run_results, output)
output.close()
# Save the results to a CSV
run_results = reformat_run_results(run_results)
run_results_csv_file = os.path.join(results_path, "run_results.csv")
write_results_csv(run_results, run_results_csv_file, "neighid")
# Write neighborhood LULC, pop, x, y coordinates, etc. for the last
# timestep.
world.write_NBHs_to_csv("END", results_path)
# Write out the world file and mask used to run the model. Update the
# rcparams to point to these files so they will be reused if this run is
# rerun.
DEM_data_file = os.path.join(results_path, "chitwanabm_DEM.tif")
array, gt, prj = world.get_DEM_data()
write_single_band_raster(array, gt, prj, DEM_data_file)
world_mask_data_file = os.path.join(results_path, "chitwanabm_world_mask.tif")
array, gt, prj = world.get_world_mask_data()
write_single_band_raster(array, gt, prj, world_mask_data_file)
# Save the SHA-1 of the commit used to run the model, along with any diffs
# from the commit (the output of the git diff command). sys.path[0] gives
# the path of the currently running chitwanabm code.
git_diff_file = os.path.join(results_path, "git_diff.patch")
commit_hash = save_git_diff(sys.path[0], git_diff_file)
time_csv_file = os.path.join(results_path, "time.csv")
write_time_csv(time_strings, time_csv_file)
if rcParams['model.make_plots']:
logger.info("Plotting population results")
Rscript_binary = rcParams['path.Rscript_binary']
plot_pop_script = resource_filename(__name__, 'R/plot_pop.R')
try:
output = subprocess.check_output([Rscript_binary, plot_pop_script,
results_path], cwd=sys.path[0], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError, e:
logger.exception("Problem running plot_pop.R. R output: %s"%e.output)
if rcParams['save_NBH_data']:
logger.info("Plotting LULC results")
plot_LULC_script = resource_filename(__name__, 'R/plot_LULC.R')
try:
output = subprocess.check_output([Rscript_binary, plot_LULC_script,
results_path], cwd=sys.path[0], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError, e:
logger.exception("Problem running plot_LULC.R. R output: %s"%e.output)
def main(argv=None):
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
log_console_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
datefmt='%I:%M:%S%p')
ch.setFormatter(log_console_formatter)
root_logger.addHandler(ch)
# Save args to pass on to runmodel
if argv is None:
argv = sys.argv
if len(argv) > 1:
# Enquote the parameters as the original quotes were stripped by the #
# parser, so this could lead to problems with things like spaces in
# path names. But: only enquote the parameters if there is more than 1
# entry in sys.argv since the # first item in sys.argv is the script
# name.
argv = map(lambda x: '"' + x + '"', argv)
process_args = " ".join(argv[1:])
parser = argparse.ArgumentParser(description='Run the chitwanabm agent-based model (ABM).')
parser.add_argument('--rc', dest="rc_file", metavar="RC_FILE", type=str, default=None,
help='Path to a rc file to initialize a model run with custom parameters')
args = parser.parse_args()
from chitwanabm import rc_params
from pyabm.utility import email_logfile
rc_params.load_default_params(__name__)
if not args.rc_file==None and not os.path.exists(args.rc_file):
logger.critical('Custom rc file %s does not exist'%args.rc_file)
sys.exit()
rc_params.initialize('chitwanabm', args.rc_file)
global rcParams
rcParams = rc_params.get_params()
scenario_path = os.path.join(str(rcParams['model.resultspath']), rcParams['scenario.name'])
if not os.path.exists(scenario_path):
try:
os.mkdir(scenario_path)
except OSError:
logger.critical("Could not create scenario directory %s"%scenario_path)
return 1
batchrun_name = "Batch_" + socket.gethostname() + time.strftime("_%Y%m%d-%H%M%S")
logfile = os.path.join(scenario_path, 'chitwanabm_' + batchrun_name + '.log')
logger.info("Logging to %s"%logfile)
fh = logging.FileHandler(logfile)
log_file_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
datefmt='%Y/%m/%d %H:%M:%S')
fh.setFormatter(log_file_formatter)
root_logger.addHandler(fh)
global pool_sema
pool_sema = threading.BoundedSemaphore(value=(rcParams['batchrun.num_cores'] + 1))
logger.info("Starting batch run %s, running '%s' scenario"%(batchrun_name, rcParams['scenario.name']))
run_count = 1
while run_count <= rcParams['batchrun.num_runs']:
with pool_sema:
script_path = resource_filename(__name__, 'runmodel.py')
new_thread = ProcessThread(run_count, script_path, process_args)
logger.info("Starting run %s"%new_thread.name)
new_thread.start()
run_count += 1
# Wait until all active threads have finished before emailing the log.
for thread in list(active_threads):
thread.join()
if rcParams['email_log']:
logger.info("Emailing log to %s"%rcParams['email_log.to'])
subject = 'chitwanabm Log - %s - %s'%(rcParams['scenario.name'],
batchrun_name)
email_logfile(logfile, subject)
logger.info("Finished batch run %s"%batchrun_name)
