Beispiel #1
0
from __future__ import division

import os

import logging

logger = logging.getLogger(__name__)

from chitwanabm import rc_params, np

if not rc_params.is_initialized():
    # Load the rc parameters if this module was imported directly (needed for 
    # Sphinx autodoc).
    rc_params.load_default_params('chitwanabm')
    rc_params.initialize('chitwanabm')
rcParams = rc_params.get_params()

from pyabm import boolean_choice
from pyabm.statistics import convert_probability_units, get_probability_index, \
        draw_from_prob_dist, calc_prob_from_prob_dist, UnitsError, \
        StatisticsError

prob_time_units = rcParams['probability.time_units']

#TODO: these probabilities should be derived from the region, not directly from rcParams
death_probabilities_male = convert_probability_units(rcParams['probability.death.male'], prob_time_units)
death_probabilities_female = convert_probability_units(rcParams['probability.death.female'], prob_time_units)
marriage_probabilities_male = convert_probability_units(rcParams['probability.marriage.male'], prob_time_units)
marriage_probabilities_female = convert_probability_units(rcParams['probability.marriage.female'], prob_time_units)
migration_probabilities_male = convert_probability_units(rcParams['probability.migration.male'], prob_time_units)
Beispiel #2
0
def main():
    parser = argparse.ArgumentParser(description='Run the chitwanabm agent-based model (ABM).')
    parser.add_argument('--rc', dest="rc_file", metavar="RC_FILE", type=str, default=None,
            help='Path to a rc file to initialize a model run with custom parameters')
    parser.add_argument('--log', metavar="LEVEL", type=str, default="info", 
            help='The logging threshold for logging to the console')
    parser.add_argument('--logf', metavar="LEVEL", type=str, 
            default="debug", help='The logging threshold for logging to the log file')
    parser.add_argument('--tail', dest='tail', action='store_const', 
            const=True, default=False, help='Tail the logfile with the default tail application specified in the rc parameters')
    parser.add_argument('--output-path', dest='output_path', default=None,
            help='Path in which to store the model output (overrides any value set in the rc-file)')
    parser.add_argument('--run-id', dest='run_ID_number', default=None,
            help='Run ID number (automatically generated if it is not specified)')
    args = parser.parse_args()

    # Setup logging according to the desired levels
    ch_level = getattr(logging, args.log.upper(), None)
    if not isinstance(ch_level, int):
        logger.critical('Invalid log level: %s' %args.log)
    root_logger.handlers[1].setLevel(ch_level)
    fh_level = getattr(logging, args.logf.upper(), None)
    if not isinstance(fh_level, int):
        logger.critical('Invalid log level: %s' %args.logf)
    root_logger.handlers[0].setLevel(fh_level)

    # Wait to load rcParams until here as logging statements are often 
    # triggered when the rcParams are loaded.
    from chitwanabm import rc_params
    # Make sure the rc_params are setup before loading any other chitwanabm 
    # modules, so that they will all take the default params including any that 
    # might be specified in user_rc_file
    root_logger.handlers[0].setLevel(fh_level)
    rc_params.load_default_params('chitwanabm')
    if not args.rc_file==None and not os.path.exists(args.rc_file):
        logger.critical('Custom rc file %s does not exist'%args.rc_file)
    rc_params.initialize('chitwanabm', args.rc_file)
    global rcParams
    rcParams = rc_params.get_params()

    from chitwanabm.initialize import generate_world
    from chitwanabm.modelloop import main_loop

    from pyabm.file_io import write_single_band_raster
    from pyabm.utility import save_git_diff
    from pyabm import __version__ as pyabm_version
    from chitwanabm import __version__ as chitwanabm_version

    if args.output_path != None:
        scenario_path = os.path.join(args.output_path, rcParams['scenario.name'])
        if not os.path.exists(args.output_path):
            try:
                os.mkdir(args.output_path)
            except OSError:
                logger.critical("Could not create output folder %s"%scenario_path)
                return 1
    else:
        scenario_path = os.path.join(rcParams['model.resultspath'], rcParams['scenario.name'])
    if not os.path.exists(scenario_path):
        try:
            os.mkdir(scenario_path)
        except OSError:
            logger.critical("Could not create scenario directory %s"%scenario_path)
            return 1

    if args.run_ID_number == None:
        # Get machine hostname to print it in the results file and use in the 
        # run_ID_number.
        hostname = socket.gethostname()
        # The run_ID_number provides an ID number (built from the start time 
        # and machine name) to uniquely identify this model run.
        run_ID_number = time.strftime("%Y%m%d-%H%M%S") + '_' + hostname
        results_path = os.path.join(scenario_path, run_ID_number)
        try:
            os.mkdir(results_path)
        except OSError:
            logger.critical("Could not create results directory %s"%results_path)
            return 1
    else:
        run_ID_number = args.run_ID_number
        results_path = os.path.join(scenario_path, run_ID_number)
        try:
            os.mkdir(results_path)
        except OSError:
            logger.critical("Could not create results directory %s"%results_path)
            return 1
        
    # Setup a special logger to log demographic events while the model is 
    # running (births, migrations, deaths, marriages, etc.)
    person_event_log_file_path = os.path.join(results_path, "person_events.log")
    person_event_log_file = open(person_event_log_file_path, mode='w')
    person_event_log_header = ",".join(["time", "event",
                                        "pid", "hid", "nid", "rid", "gender", "age", 
                                        "ethnicity", "mother_id", "father_id", 
                                        "spouseid", "marrtime", "schooling", 
                                        "num_children", "alive", "is_away", 
                                        "is_initial_agent", "is_in_migrant",
                                        "mother_num_children",
                                        "mother_years_schooling", 
                                        "mother_work", 
                                        "father_years_schooling","father_work", 
                                        "parents_contracep"])
    person_event_log_file.write(person_event_log_header + '\n')
    person_event_log_file.close()
    person_event_fh = logging.FileHandler(os.path.join(results_path, "person_events.log"), mode='a')
    person_event_fh.setLevel(logging.INFO)
    person_event_fh.setFormatter(logging.Formatter('%(modeltime)s,%(message)s,%(personinfo)s'))

    # Setup a filter so the agent event log will contain only agent events.
    class PassEventFilter(logging.Filter):
        def filter(self, record):
            logger_name = getattr(record, 'name', None)
            return 'person_events' in logger_name
    person_event_fh.addFilter(PassEventFilter())

    person_event_logger = logging.getLogger('person_events')
    person_event_logger.addHandler(person_event_fh)

    # Now that we know the rcParams and log file path, write the temp_log 
    # stream to the log file in the proper output directory, and direct all 
    # further logging to append to that file.
    log_file_path = os.path.join(results_path, "chitwanabm.log")
    shutil.copyfile(temp_log_file.name, log_file_path)
    temp_log_file.close()
    root_logger.handlers.remove(temp_log)
    temp_log.close()
    os.unlink(temp_log_file.name)
    new_fh = logging.FileHandler(log_file_path, mode='a')
    new_fh.setLevel(fh_level)
    new_fh.setFormatter(log_file_formatter)
    root_logger.addHandler(new_fh)
    # Dont' log agent event records to the main or console loggers
    class DontPassEventFilter(logging.Filter):
        def filter(self, record):
            logger_name = getattr(record, 'name', None)
            return 'person_events' not in logger_name
    for handler in root_logger.handlers:
        handler.addFilter(DontPassEventFilter())

    if args.tail:
        try:
            subprocess.Popen([rcParams['path.tail_binary'], log_file_path], cwd=results_path)
        except:
            logger.warning('Error tailing model log file: %s'%sys.exc_info()[1])

    # TODO: Fix the loading of a pickled world. For now, force regenerating the 
    # world each model run.
    # if rcParams['model.reinitialize']:
    #     # Generate a new world (with new resampling, etc.)
    #     world = generate_world()
    #     if world == 1:
    #         logger.critical('Error initializing model world')
    #         return 1
    # else:
    #     # Load a pickled World for use in the model.
    #     input_data_file = rcParams['path.input_data_file']
    #     file = open(input_data_file, "r")
    #     try:
    #         world = pickle.load(file)
    #     except IOError:
    #         logger.critical('Error loading world data from  %s'%input_data_file)
    #         return 1
    
    # Generate a new world (with new resampling, etc.)
    world = generate_world()
    if world == 1:
        logger.critical('Error initializing model world')
        return 1

    # Run the model loop
    start_time = time.localtime()
    logger.info('Beginning model run %s'%run_ID_number)
    run_results, time_strings = main_loop(world, results_path) # This line actually runs the model.
    end_time = time.localtime()
    logger.info('Finished model run number %s'%run_ID_number)
    
    # Save the results to a pickled file
    if rcParams['save_pickled_end_results']:
        logger.info("Saving results")
        pop_data_file = os.path.join(results_path, "run_results.P")
        output = open(pop_data_file, 'w')
        pickle.dump(run_results, output)
        output.close()
    # Save the results to a CSV
    run_results = reformat_run_results(run_results)
    run_results_csv_file = os.path.join(results_path, "run_results.csv")
    write_results_csv(run_results, run_results_csv_file, "neighid")

    # Write neighborhood LULC, pop, x, y coordinates, etc. for the last 
    # timestep.
    world.write_NBHs_to_csv("END", results_path)

    # Write out the world file and mask used to run the model. Update the 
    # rcparams to point to these files so they will be reused if this run is 
    # rerun.
    DEM_data_file = os.path.join(results_path, "chitwanabm_DEM.tif")
    array, gt, prj = world.get_DEM_data()
    write_single_band_raster(array, gt, prj, DEM_data_file)
    world_mask_data_file = os.path.join(results_path, "chitwanabm_world_mask.tif")
    array, gt, prj = world.get_world_mask_data()
    write_single_band_raster(array, gt, prj, world_mask_data_file)

    # Save the SHA-1 of the commit used to run the model, along with any diffs 
    # from the commit (the output of the git diff command). sys.path[0] gives 
    # the path of the currently running chitwanabm code.
    git_diff_file = os.path.join(results_path, "git_diff.patch")
    commit_hash = save_git_diff(sys.path[0], git_diff_file)

    time_csv_file = os.path.join(results_path, "time.csv")
    write_time_csv(time_strings, time_csv_file)
    
    if rcParams['model.make_plots']:
        logger.info("Plotting population results")
        Rscript_binary = rcParams['path.Rscript_binary']
        plot_pop_script = resource_filename(__name__, 'R/plot_pop.R')
        try:
            output = subprocess.check_output([Rscript_binary, plot_pop_script, 
                results_path], cwd=sys.path[0], stderr=subprocess.STDOUT)
        except subprocess.CalledProcessError, e:
            logger.exception("Problem running plot_pop.R. R output: %s"%e.output)

        if rcParams['save_NBH_data']:
            logger.info("Plotting LULC results")
            plot_LULC_script = resource_filename(__name__, 'R/plot_LULC.R')
            try:
                output = subprocess.check_output([Rscript_binary, plot_LULC_script, 
                    results_path], cwd=sys.path[0], stderr=subprocess.STDOUT)
            except subprocess.CalledProcessError, e:
                logger.exception("Problem running plot_LULC.R. R output: %s"%e.output)
def main(argv=None):
    root_logger = logging.getLogger()
    root_logger.setLevel(logging.DEBUG)
    ch = logging.StreamHandler()
    log_console_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
            datefmt='%I:%M:%S%p')
    ch.setFormatter(log_console_formatter)
    root_logger.addHandler(ch)

    # Save args to pass on to runmodel
    if argv is None:
        argv = sys.argv
    if len(argv) > 1:
        # Enquote the parameters as the original quotes were stripped by the # 
        # parser, so this could lead to problems with things like spaces in 
        # path names. But: only enquote the parameters if there is more than 1 
        # entry in sys.argv since the # first item in sys.argv is the script 
        # name.
        argv = map(lambda x: '"' + x + '"', argv)
    process_args = " ".join(argv[1:])

    parser = argparse.ArgumentParser(description='Run the chitwanabm agent-based model (ABM).')
    parser.add_argument('--rc', dest="rc_file", metavar="RC_FILE", type=str, default=None,
            help='Path to a rc file to initialize a model run with custom parameters')
    args = parser.parse_args()

    from chitwanabm import rc_params
    from pyabm.utility import email_logfile

    rc_params.load_default_params(__name__)
    if not args.rc_file==None and not os.path.exists(args.rc_file):
        logger.critical('Custom rc file %s does not exist'%args.rc_file)
        sys.exit()
    rc_params.initialize('chitwanabm', args.rc_file)
    global rcParams
    rcParams = rc_params.get_params()

    scenario_path = os.path.join(str(rcParams['model.resultspath']), rcParams['scenario.name'])
    if not os.path.exists(scenario_path):
        try:
            os.mkdir(scenario_path)
        except OSError:
            logger.critical("Could not create scenario directory %s"%scenario_path)
            return 1

    batchrun_name = "Batch_" + socket.gethostname() + time.strftime("_%Y%m%d-%H%M%S")
    logfile = os.path.join(scenario_path, 'chitwanabm_' + batchrun_name + '.log')
    logger.info("Logging to %s"%logfile)
    fh = logging.FileHandler(logfile)
    log_file_formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s',
            datefmt='%Y/%m/%d %H:%M:%S')
    fh.setFormatter(log_file_formatter)
    root_logger.addHandler(fh)

    global pool_sema
    pool_sema = threading.BoundedSemaphore(value=(rcParams['batchrun.num_cores'] + 1))

    logger.info("Starting batch run %s, running '%s' scenario"%(batchrun_name, rcParams['scenario.name']))
    run_count = 1
    while run_count <= rcParams['batchrun.num_runs']:
        with pool_sema:
            script_path = resource_filename(__name__, 'runmodel.py')
            new_thread = ProcessThread(run_count, script_path, process_args)
            logger.info("Starting run %s"%new_thread.name)
            new_thread.start()
            run_count += 1

    # Wait until all active threads have finished before emailing the log.
    for thread in list(active_threads):
        thread.join()

    if rcParams['email_log']:
        logger.info("Emailing log to %s"%rcParams['email_log.to'])
        subject = 'chitwanabm Log - %s - %s'%(rcParams['scenario.name'], 
                batchrun_name)
        email_logfile(logfile, subject)
    logger.info("Finished batch run %s"%batchrun_name)