import pickle import shutil import tempfile import subprocess from pkg_resources import resource_filename import numpy as np from pyabm.file_io import read_single_band_raster from chitwanabm import rc_params from chitwanabm.agents import World logger = logging.getLogger(__name__) rcParams = rc_params.get_params() def main(): ch = logging.StreamHandler() ch.setLevel(logging.INFO) log_console_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', datefmt='%I:%M:%S%p') ch.setFormatter(log_console_formatter) logger.addHandler(ch) world = generate_world() if world == 1: logger.critical("Problem generating world") return 1 def read_CVFS_data(textfile, key_field):
def main(): parser = argparse.ArgumentParser(description='Run the chitwanabm agent-based model (ABM).') parser.add_argument('--rc', dest="rc_file", metavar="RC_FILE", type=str, default=None, help='Path to a rc file to initialize a model run with custom parameters') parser.add_argument('--log', metavar="LEVEL", type=str, default="info", help='The logging threshold for logging to the console') parser.add_argument('--logf', metavar="LEVEL", type=str, default="debug", help='The logging threshold for logging to the log file') parser.add_argument('--tail', dest='tail', action='store_const', const=True, default=False, help='Tail the logfile with the default tail application specified in the rc parameters') parser.add_argument('--output-path', dest='output_path', default=None, help='Path in which to store the model output (overrides any value set in the rc-file)') parser.add_argument('--run-id', dest='run_ID_number', default=None, help='Run ID number (automatically generated if it is not specified)') args = parser.parse_args() # Setup logging according to the desired levels ch_level = getattr(logging, args.log.upper(), None) if not isinstance(ch_level, int): logger.critical('Invalid log level: %s' %args.log) root_logger.handlers[1].setLevel(ch_level) fh_level = getattr(logging, args.logf.upper(), None) if not isinstance(fh_level, int): logger.critical('Invalid log level: %s' %args.logf) root_logger.handlers[0].setLevel(fh_level) # Wait to load rcParams until here as logging statements are often # triggered when the rcParams are loaded. from chitwanabm import rc_params # Make sure the rc_params are setup before loading any other chitwanabm # modules, so that they will all take the default params including any that # might be specified in user_rc_file root_logger.handlers[0].setLevel(fh_level) rc_params.load_default_params('chitwanabm') if not args.rc_file==None and not os.path.exists(args.rc_file): logger.critical('Custom rc file %s does not exist'%args.rc_file) rc_params.initialize('chitwanabm', args.rc_file) global rcParams rcParams = rc_params.get_params() from chitwanabm.initialize import generate_world from chitwanabm.modelloop import main_loop from pyabm.file_io import write_single_band_raster from pyabm.utility import save_git_diff from pyabm import __version__ as pyabm_version from chitwanabm import __version__ as chitwanabm_version if args.output_path != None: scenario_path = os.path.join(args.output_path, rcParams['scenario.name']) if not os.path.exists(args.output_path): try: os.mkdir(args.output_path) except OSError: logger.critical("Could not create output folder %s"%scenario_path) return 1 else: scenario_path = os.path.join(rcParams['model.resultspath'], rcParams['scenario.name']) if not os.path.exists(scenario_path): try: os.mkdir(scenario_path) except OSError: logger.critical("Could not create scenario directory %s"%scenario_path) return 1 if args.run_ID_number == None: # Get machine hostname to print it in the results file and use in the # run_ID_number. hostname = socket.gethostname() # The run_ID_number provides an ID number (built from the start time # and machine name) to uniquely identify this model run. run_ID_number = time.strftime("%Y%m%d-%H%M%S") + '_' + hostname results_path = os.path.join(scenario_path, run_ID_number) try: os.mkdir(results_path) except OSError: logger.critical("Could not create results directory %s"%results_path) return 1 else: run_ID_number = args.run_ID_number results_path = os.path.join(scenario_path, run_ID_number) try: os.mkdir(results_path) except OSError: logger.critical("Could not create results directory %s"%results_path) return 1 # Setup a special logger to log demographic events while the model is # running (births, migrations, deaths, marriages, etc.) person_event_log_file_path = os.path.join(results_path, "person_events.log") person_event_log_file = open(person_event_log_file_path, mode='w') person_event_log_header = ",".join(["time", "event", "pid", "hid", "nid", "rid", "gender", "age", "ethnicity", "mother_id", "father_id", "spouseid", "marrtime", "schooling", "num_children", "alive", "is_away", "is_initial_agent", "is_in_migrant", "mother_num_children", "mother_years_schooling", "mother_work", "father_years_schooling","father_work", "parents_contracep"]) person_event_log_file.write(person_event_log_header + '\n') person_event_log_file.close() person_event_fh = logging.FileHandler(os.path.join(results_path, "person_events.log"), mode='a') person_event_fh.setLevel(logging.INFO) person_event_fh.setFormatter(logging.Formatter('%(modeltime)s,%(message)s,%(personinfo)s')) # Setup a filter so the agent event log will contain only agent events. class PassEventFilter(logging.Filter): def filter(self, record): logger_name = getattr(record, 'name', None) return 'person_events' in logger_name person_event_fh.addFilter(PassEventFilter()) person_event_logger = logging.getLogger('person_events') person_event_logger.addHandler(person_event_fh) # Now that we know the rcParams and log file path, write the temp_log # stream to the log file in the proper output directory, and direct all # further logging to append to that file. log_file_path = os.path.join(results_path, "chitwanabm.log") shutil.copyfile(temp_log_file.name, log_file_path) temp_log_file.close() root_logger.handlers.remove(temp_log) temp_log.close() os.unlink(temp_log_file.name) new_fh = logging.FileHandler(log_file_path, mode='a') new_fh.setLevel(fh_level) new_fh.setFormatter(log_file_formatter) root_logger.addHandler(new_fh) # Dont' log agent event records to the main or console loggers class DontPassEventFilter(logging.Filter): def filter(self, record): logger_name = getattr(record, 'name', None) return 'person_events' not in logger_name for handler in root_logger.handlers: handler.addFilter(DontPassEventFilter()) if args.tail: try: subprocess.Popen([rcParams['path.tail_binary'], log_file_path], cwd=results_path) except: logger.warning('Error tailing model log file: %s'%sys.exc_info()[1]) # TODO: Fix the loading of a pickled world. For now, force regenerating the # world each model run. # if rcParams['model.reinitialize']: # # Generate a new world (with new resampling, etc.) # world = generate_world() # if world == 1: # logger.critical('Error initializing model world') # return 1 # else: # # Load a pickled World for use in the model. # input_data_file = rcParams['path.input_data_file'] # file = open(input_data_file, "r") # try: # world = pickle.load(file) # except IOError: # logger.critical('Error loading world data from %s'%input_data_file) # return 1 # Generate a new world (with new resampling, etc.) world = generate_world() if world == 1: logger.critical('Error initializing model world') return 1 # Run the model loop start_time = time.localtime() logger.info('Beginning model run %s'%run_ID_number) run_results, time_strings = main_loop(world, results_path) # This line actually runs the model. end_time = time.localtime() logger.info('Finished model run number %s'%run_ID_number) # Save the results to a pickled file if rcParams['save_pickled_end_results']: logger.info("Saving results") pop_data_file = os.path.join(results_path, "run_results.P") output = open(pop_data_file, 'w') pickle.dump(run_results, output) output.close() # Save the results to a CSV run_results = reformat_run_results(run_results) run_results_csv_file = os.path.join(results_path, "run_results.csv") write_results_csv(run_results, run_results_csv_file, "neighid") # Write neighborhood LULC, pop, x, y coordinates, etc. for the last # timestep. world.write_NBHs_to_csv("END", results_path) # Write out the world file and mask used to run the model. Update the # rcparams to point to these files so they will be reused if this run is # rerun. DEM_data_file = os.path.join(results_path, "chitwanabm_DEM.tif") array, gt, prj = world.get_DEM_data() write_single_band_raster(array, gt, prj, DEM_data_file) world_mask_data_file = os.path.join(results_path, "chitwanabm_world_mask.tif") array, gt, prj = world.get_world_mask_data() write_single_band_raster(array, gt, prj, world_mask_data_file) # Save the SHA-1 of the commit used to run the model, along with any diffs # from the commit (the output of the git diff command). sys.path[0] gives # the path of the currently running chitwanabm code. git_diff_file = os.path.join(results_path, "git_diff.patch") commit_hash = save_git_diff(sys.path[0], git_diff_file) time_csv_file = os.path.join(results_path, "time.csv") write_time_csv(time_strings, time_csv_file) if rcParams['model.make_plots']: logger.info("Plotting population results") Rscript_binary = rcParams['path.Rscript_binary'] plot_pop_script = resource_filename(__name__, 'R/plot_pop.R') try: output = subprocess.check_output([Rscript_binary, plot_pop_script, results_path], cwd=sys.path[0], stderr=subprocess.STDOUT) except subprocess.CalledProcessError, e: logger.exception("Problem running plot_pop.R. R output: %s"%e.output) if rcParams['save_NBH_data']: logger.info("Plotting LULC results") plot_LULC_script = resource_filename(__name__, 'R/plot_LULC.R') try: output = subprocess.check_output([Rscript_binary, plot_LULC_script, results_path], cwd=sys.path[0], stderr=subprocess.STDOUT) except subprocess.CalledProcessError, e: logger.exception("Problem running plot_LULC.R. R output: %s"%e.output)
def main(argv=None): root_logger = logging.getLogger() root_logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() log_console_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', datefmt='%I:%M:%S%p') ch.setFormatter(log_console_formatter) root_logger.addHandler(ch) # Save args to pass on to runmodel if argv is None: argv = sys.argv if len(argv) > 1: # Enquote the parameters as the original quotes were stripped by the # # parser, so this could lead to problems with things like spaces in # path names. But: only enquote the parameters if there is more than 1 # entry in sys.argv since the # first item in sys.argv is the script # name. argv = map(lambda x: '"' + x + '"', argv) process_args = " ".join(argv[1:]) parser = argparse.ArgumentParser(description='Run the chitwanabm agent-based model (ABM).') parser.add_argument('--rc', dest="rc_file", metavar="RC_FILE", type=str, default=None, help='Path to a rc file to initialize a model run with custom parameters') args = parser.parse_args() from chitwanabm import rc_params from pyabm.utility import email_logfile rc_params.load_default_params(__name__) if not args.rc_file==None and not os.path.exists(args.rc_file): logger.critical('Custom rc file %s does not exist'%args.rc_file) sys.exit() rc_params.initialize('chitwanabm', args.rc_file) global rcParams rcParams = rc_params.get_params() scenario_path = os.path.join(str(rcParams['model.resultspath']), rcParams['scenario.name']) if not os.path.exists(scenario_path): try: os.mkdir(scenario_path) except OSError: logger.critical("Could not create scenario directory %s"%scenario_path) return 1 batchrun_name = "Batch_" + socket.gethostname() + time.strftime("_%Y%m%d-%H%M%S") logfile = os.path.join(scenario_path, 'chitwanabm_' + batchrun_name + '.log') logger.info("Logging to %s"%logfile) fh = logging.FileHandler(logfile) log_file_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s', datefmt='%Y/%m/%d %H:%M:%S') fh.setFormatter(log_file_formatter) root_logger.addHandler(fh) global pool_sema pool_sema = threading.BoundedSemaphore(value=(rcParams['batchrun.num_cores'] + 1)) logger.info("Starting batch run %s, running '%s' scenario"%(batchrun_name, rcParams['scenario.name'])) run_count = 1 while run_count <= rcParams['batchrun.num_runs']: with pool_sema: script_path = resource_filename(__name__, 'runmodel.py') new_thread = ProcessThread(run_count, script_path, process_args) logger.info("Starting run %s"%new_thread.name) new_thread.start() run_count += 1 # Wait until all active threads have finished before emailing the log. for thread in list(active_threads): thread.join() if rcParams['email_log']: logger.info("Emailing log to %s"%rcParams['email_log.to']) subject = 'chitwanabm Log - %s - %s'%(rcParams['scenario.name'], batchrun_name) email_logfile(logfile, subject) logger.info("Finished batch run %s"%batchrun_name)