def main(log_level='INFO') -> int:
""" Data directories are copied to S3 """
logger = root_logger_setup(consolehandlerlevel=log_level, filehandlerlevel='DEBUG')
logger.debug(locals())
version = get_bayota_version()
logger.info('v----------------------------------------------v')
logger.info(' ******* %s *******' % ('BayOTA version ' + version).center(30, ' '))
logger.info(' ********** Moving source data to s3 **********')
logger.info('^----------------------------------------------^')
# Connection with S3 is established.
s3ops = establish_s3_connection(log_level, logger)
# Source CSV files are copied.
local_path = get_source_csvs_dir(s3=False)
s3_path = get_source_csvs_dir(s3=True)
logger.info(f"copying source csv files from {local_path} to the s3 location: {s3_path}")
s3ops.move_to_s3(local_path=local_path, destination_path=s3_path, move_directory=True)
# Metadata CSV files are copied.
local_path = get_metadata_csvs_dir(s3=False)
s3_path = get_metadata_csvs_dir(s3=True)
logger.info(f"copying metadata csv files from {local_path} to the s3 location: {s3_path}")
s3ops.move_to_s3(local_path=local_path, destination_path=s3_path, move_directory=True)
# Raw Data files are copied.
local_path = get_raw_data_dir(s3=False)
s3_path = get_raw_data_dir(s3=True)
logger.info(f"copying other raw data files from {local_path} to the s3 location: {s3_path}")
s3ops.move_to_s3(local_path=local_path, destination_path=s3_path, move_directory=True)
return 0 # a clean, no-issue, exit
def read_study_control_file(control_file_name):
""" The control file is read. """
control_dict = read_control(control_file_name)
studydict = control_dict['study']
studyshortname = studydict['studyshortname']
studyid = studydict['id']
# Geography
geodict = control_dict['geography']
geo_entity_name = geodict['entity']
compact_geo_entity_str = compact_capitalized_geography_string(
geo_entity_name)
geodict['shortname'] = compact_geo_entity_str
control_dict['geography'] = geodict
# Model Specification
model_spec_name = studydict['model_spec']
model_dict = read_spec(
spec_file_name=model_spec_name, spectype='model'
) # Model generation details are saved to control file.
saved_model_name_for_this_study = 'mdlspec--' + model_spec_name + '--_geo--' + compact_geo_entity_str + '--.pickle'
control_dict['model'] = {
'spec_name': model_spec_name,
'objectiveshortname': model_dict['objectiveshortname'],
'constraintshortname': model_dict['constraintshortname'],
'saved_name_for_this_study': saved_model_name_for_this_study
}
# Study Unique ID
control_dict['study']['uuid'] = 's' + studyid + '-' + str(uuid.uuid4())
# Experiments
experiments = studydict['experiments']
control_dict['experiments'] = experiments.copy()
# Base Loading Condition
baseloadingfilename = studydict['base_loading_file_name']
control_dict['base_loading_file_name'] = baseloadingfilename
# Run log
control_dict['code_version']: get_bayota_version()
control_dict['submission_timestamps'] = {
'step1_study': datetime.datetime.today().strftime('%Y-%m-%d-%H:%M:%S')
}
# Write (or replace existing) study control file with updated dictionary entries
overwrite_control(control_file_name=control_file_name,
control_dict=control_dict)
return control_dict, experiments, baseloadingfilename, \
geo_entity_name, compact_geo_entity_str, model_spec_name, studyshortname, studyid
def main(control_file,
dryrun=False,
no_slurm=False,
save_to_s3=False,
log_level='INFO') -> int:
version = get_bayota_version()
# Control file is read.
control_dict, \
actionlist, \
compact_geo_entity_str, \
expid, \
expname, \
list_of_trialdicts, \
saved_model_file, \
studyid = read_expcon_file(control_file)
# Logging formats are set up.
logger = set_up_detailedfilelogger(
loggername=expname, # same name as module, so logger is shared
filename=f"step3_s{studyid}_e{expid}_{compact_geo_entity_str}.log",
level=log_level,
also_logtoconsole=True,
add_filehandler_if_already_exists=True,
add_consolehandler_if_already_exists=False)
logger.info('----------------------------------------------')
logger.info('************* Model Modification *************')
logger.info('----------------------------------------------')
# Job command is built and submitted.
CMD = f"{modify_model_script} {control_file} --log_level={log_level}"
if save_to_s3:
CMD = CMD + ' --save_to_s3'
if not no_slurm:
slurm_options = f"--nodes={1} " \
f"--ntasks={1} " \
f"--cpus-per-task={1} " \
f"--exclusive "
CMD = 'srun ' + slurm_options + CMD
logger.info(f'Job command is: "{CMD}"')
if notdry(dryrun, logger, '--Dryrun-- Would submit command, then wait.'):
p = subprocess.Popen([CMD], shell=True)
p.wait()
if p.returncode != 0: # Return code from process is checked.
logger.error(
f"Model Modification finished with non-zero code <{p.returncode}>"
)
return 1
logger.info('----------------------------------------------')
logger.info('*************** Trials looping ***************')
logger.info('----------------------------------------------')
# List of trial sets to be conducted for this experiment are logged.
tempstr = 'set' if len(list_of_trialdicts) == 1 else 'sets'
logger.info(
f"{logprefix} {expname} - trial {tempstr} to be conducted: {list_of_trialdicts}"
)
# Loop through and start each trial
trialnum = 0
p_list = []
for i, dictwithtrials in enumerate(list_of_trialdicts):
logger.info(
f'{logprefix} {expname} - trial set #{i}: {dictwithtrials}')
modvar = dictwithtrials['variable']
logger.info(f'variable to modify: {modvar}')
varvalue = dictwithtrials['value']
logger.info('values: %s' % varvalue)
varindexer = None
try:
varindexer = dictwithtrials['indexer']
logger.info(f'indexed over: {varindexer}')
except KeyError:
pass
for vi in varvalue:
trialnum += 1
trialidstr = '{:04}'.format(trialnum)
logger.info(f'trial #{trialidstr}, setting <{modvar}> to <{vi}>')
modificationstr = f"\'{{\"variable\": \"{modvar}\", " \
f"\"value\": {vi}, " \
f"\"indexer\": \"{varindexer}\"}}\'"
# A trial control ("trialcon") file is generated by adding to the expcon file.
control_dict['trial'] = {
'id':
trialidstr,
'trial_name':
'exp--' + expname + '--_modvar--' + modvar + '--_trial' +
trialidstr,
'modification':
modificationstr,
'solutions_folder_name':
expname
}
control_dict['trial']['uuid'] = control_dict['experiment'][
'uuid'] + '_t' + trialidstr
control_dict['code_version']: version
control_dict['run_timestamps'][
'step4_trial'] = datetime.datetime.today().strftime(
'%Y-%m-%d-%H:%M:%S')
unique_control_name = write_control_with_uniqueid(
control_dict=control_dict,
name_prefix='step4_trialcon',
logger=logger)
# Job command is built and submitted.
CMD = f"{solve_trial_script} {unique_control_name} --log_level={log_level}"
if save_to_s3:
CMD = CMD + ' --save_to_s3'
if not no_slurm:
slurm_options = f"--nodes={1} " \
f"--ntasks={1} " \
f"--cpus-per-task={2} " \
f"--exclusive "
CMD = 'srun ' + slurm_options + CMD
logger.info(f'Job command is: "{CMD}"')
if notdry(dryrun, logger, '--Dryrun-- Would submit command'):
p_list.append(subprocess.Popen([CMD], shell=True))
[p.wait() for p in p_list]
return 0 # a clean, no-issue, exit
import math
import yaml
import numpy as np
import pandas as pd
import pyomo.environ as pyo
from bayota_settings.base import get_bayota_version
print("using bayota version '%s'" % get_bayota_version())
from bayota_settings.base import get_source_csvs_dir
print("getting source csvs from: '%s'" % get_source_csvs_dir())
from bayom_e.model_handling.interface import build_model
modelspec = '/Users/Danny/bayota_ws_0.1b1.dev2/specification_files/model_specs/costmin_total_Npercentreduction_updated.yaml'
geoscale = 'county'
geoentities = 'Adams, PA'
baseloadingfilename = '2010NoActionLoads_updated.csv'
with open(modelspec, 'r') as stream:
modelspec_dict = yaml.safe_load(stream)
modelspec_dict['variant'] = 'lp'
print(modelspec_dict)
my_model, dp = build_model(modelspec_dict,
geoscale,
geoentities,
baseloadingfilename,
def main(batch_spec_file, dryrun=False, no_s3=False, no_docker=False, log_level='INFO') -> int:
starttime = time.time()
# Logging formats are set up.
logger = root_logger_setup(consolehandlerlevel=log_level, filehandlerlevel='DEBUG')
logger.debug(locals())
version = get_bayota_version()
logger.info('v----------------------------------------------v')
logger.info(' ******* %s *******' % ('BayOTA version ' + version).center(30, ' '))
logger.info(' ************** Batch of studies **************')
logger.info(f" docker image: '{docker_image}' ")
logger.info('^----------------------------------------------^')
# Script locations in the docker image
if no_docker:
model_generator_script = '${BAYOTA_HOME}/bin/run_steps/step1_generatemodel.py'
modify_model_script = '${BAYOTA_HOME}/bin/run_steps/step2_modifymodel.py'
solve_trial_script = '${BAYOTA_HOME}/bin/run_steps/step3_solveonetrial.py'
else:
model_generator_script = '/root/bayota/bin/run_steps/step1_generatemodel.py'
modify_model_script = '/root/bayota/bin/run_steps/step2_modifymodel.py'
solve_trial_script = '/root/bayota/bin/run_steps/step3_solveonetrial.py'
""" Batch specification file is read. """
geo_scale, study_pairs, control_options = parse_batch_spec(batch_spec_file, logger=logger)
# From the batch spec, we retrieve:
# - list of study pairs, i.e. a list of tuples with (geo, model_form_dict)
# - control options (a dictionary)
""" Specification files are moved to s3. """
s3ops = None
local_specfiles_path = get_spec_files_dir(s3=False)
s3_specfiles_path = get_spec_files_dir(s3=True)
if not no_s3:
s3ops = establish_s3_connection(log_level, logger)
logger.info(f"copying specification files from {local_specfiles_path} to the s3 location: {s3_specfiles_path}")
s3ops.move_to_s3(local_path=local_specfiles_path,
destination_path=s3_specfiles_path,
move_directory=True)
else:
logger.info(f"would copy local specification files from {local_specfiles_path} to the s3 location: {s3_specfiles_path}")
""" Each study (geography, model form) is iterated over. """
jobids = dict()
jobids['study'] = dict()
for index, sp in enumerate(study_pairs):
# (ModelGeography, ModelForm+Experiments) are combined to form studies, which are submitted as jobs.
# Study information is pulled apart.
geoname = sp[0] # model geography (first part of the study_pair tuples)
studyspecdict = sp[1] # model form + experiment (second part of the study_pair tuples)
# A name and ID are formed for this study.
filesafegeostring = geoname.replace(' ', '').replace(',', '')
studyid = '{:04}'.format(index + 1)
spname = filesafegeostring + f"_s{studyid}"
studyspecdict['studyshortname'] = spname
studyspecdict['id'] = studyid
# "STUDYCON": A study control file w/unique identifier (uuid4) is created and written locally.
control_dict = {"geography": {'scale': geo_scale, 'entity': geoname},
"study": studyspecdict, "control_options": control_options,
"code_version": version}
studycon_name = write_control_with_uniqueid(control_dict=control_dict, name_prefix='step1_studycon',
logger=logger)
# The local study control file is read....
control_dict, \
experiments, \
baseloadingfilename, \
geography_name, \
compact_geo_entity_str, \
model_spec_name, \
studyshortname, \
studyid = read_study_control_file(studycon_name)
move_controlfile_to_s3(logger, s3ops, controlfile_name=studycon_name, no_s3=no_s3)
logger.info('v----------------------------------------------v')
logger.info(' *************** Single Study *****************')
logger.info(f" Geography = {geography_name}")
logger.info(f" Model specification name = {model_spec_name}")
logger.info(f" Experiments = {experiments}")
logger.info(f" Base_loading_file_name = {baseloadingfilename}")
logger.info('^----------------------------------------------^')
logger.info('')
logger.info('v--------------------------------------------v')
logger.info(' ************** Model Generation ************')
logger.info('^--------------------------------------------^')
""" GENERATE MODEL VIA DOCKER IMAGE """
# Job is submitted to AWS Batch.
CMD = ['python', model_generator_script, studycon_name,
'--use_s3_ws', '--save_to_s3', f"--log_level={log_level}"]
response = batch.submit_job(jobName=f"BAYOTA_modelgeneration_using_{studycon_name}",
jobQueue='Modeling',
jobDefinition='Modeling-Bayota:6',
containerOverrides={"command": CMD},
retryStrategy={'attempts': 2}
)
print(f"submitted command: {CMD}")
print("Job ID is {}.".format(response['jobId']))
jobids['study'][studyid] = dict()
jobids['study'][studyid]['self'] = response['jobId']
""" Each experiment is iterated over. """
# A job is submitted for each experiment in the list.
p_list = []
jobids['study'][studyid]['exp'] = dict()
for ii, exp_spec_name in enumerate(experiments):
expactiondict = read_spec(spec_file_name=exp_spec_name, spectype='experiment')
expid = '{:04}'.format(ii + 1)
logger.info(f"Exp. #{expid}: {exp_spec_name}")
# "EXPCON": An experiment control file w/unique identifier (uuid4) is written by adding to studycon file.
try:
del control_dict["experiments"]
except KeyError:
logger.info("Key 'experiments' not found")
expactiondict['id'] = expid
control_dict['experiment'] = expactiondict
control_dict['experiment_name'] = exp_spec_name
control_dict['experiment']['uuid'] = control_dict['study']['uuid'] + '_e' + expid
expcon_name = write_control_with_uniqueid(control_dict=control_dict, name_prefix='step3_expcon',
logger=logger)
# The local experiment control file is read....
control_dict, \
actionlist, \
compact_geo_entity_str, \
expid, \
expname, \
list_of_trialdicts, \
saved_model_file, \
studyid = read_expcon_file(expcon_name)
logger.info('v--------------------------------------------v')
logger.info(' ************* Model Modification ***********')
logger.info('^--------------------------------------------^')
move_controlfile_to_s3(logger, s3ops, controlfile_name=expcon_name, no_s3=no_s3)
""" MODIFY MODEL VIA DOCKER IMAGE """
# Job is submitted to AWS Batch.
CMD = ['python', modify_model_script, expcon_name,
'--use_s3_ws', '--save_to_s3', f"--log_level={log_level}"]
response = batch.submit_job(jobName=f"Bayota_modelmods_using_{expcon_name}",
jobQueue='Modeling',
jobDefinition='Modeling-Bayota:6',
dependsOn=[{'jobId': jobids['study'][studyid]['self']}],
containerOverrides={"command": CMD},
retryStrategy={'attempts': 2}
)
print(f"submitted command: {CMD}")
print("Job ID is {}.".format(response['jobId']))
jobids['study'][studyid]['exp'][expid] = dict()
jobids['study'][studyid]['exp'][expid]['self'] = response['jobId']
""" Each trial is iterated over. """
# List of trial sets to be conducted for this experiment are logged.
tempstr = 'set' if len(list_of_trialdicts) == 1 else 'sets'
logger.debug(f"** Single Experiment **: {expname} - trial {tempstr} to be conducted: {list_of_trialdicts}")
trialnum = 0
p_list = []
jobids['study'][studyid]['exp'][expid]['trial'] = dict()
for i, dictwithtrials in enumerate(list_of_trialdicts):
logger.info('v--------------------------------------------v')
logger.info(' **************** Trial Set *****************')
logger.info(f" Geography = {compact_geo_entity_str}")
logger.info(f" Experiment = {expname}")
logger.info(f" Set #{i}: = {dictwithtrials}")
logger.info('^--------------------------------------------^')
# Variable value(s) for this trial set are parsed.
modvar = dictwithtrials['variable']
varvalue = dictwithtrials['value']
logger.debug(f'variable to modify: {modvar}')
logger.debug('values: %s' % varvalue)
varindexer = None
try:
varindexer = dictwithtrials['indexer']
logger.debug(f'indexed over: {varindexer}')
except KeyError:
pass
# Loop through and start each trial
for vi in varvalue:
trialnum += 1
trialidstr = '{:04}'.format(trialnum)
modificationstr = f"\'{{\"variable\": \"{modvar}\", " \
f"\"value\": {vi}, " \
f"\"indexer\": \"{varindexer}\"}}\'"
logger.info(f'trial #{trialidstr}, setting <{modvar}> to <{vi}>')
# "TRIALCON": A trial control file w/unique identifier (uuid4) is written by adding to expcon file.
control_dict['trial'] = {'id': trialidstr,
'trial_name': 'exp--' + expname + '--_modvar--' + modvar + '--_trial' + trialidstr,
'modification': modificationstr,
'solutions_folder_name': expname}
control_dict['trial']['uuid'] = control_dict['experiment']['uuid'] + '_t' + trialidstr
control_dict['code_version']: version
control_dict['submission_timestamps']['step4_trial'] = datetime.datetime.today().strftime(
'%Y-%m-%d-%H:%M:%S')
trialcon_name = write_control_with_uniqueid(control_dict=control_dict, name_prefix='step4_trialcon',
logger=logger)
move_controlfile_to_s3(logger, s3ops, controlfile_name=trialcon_name, no_s3=no_s3)
""" SOLVE TRIAL VIA DOCKER IMAGE """
# Job is submitted to AWS Batch.
CMD = ['python', solve_trial_script, trialcon_name,
'--use_s3_ws', '--save_to_s3', f"--log_level={log_level}"]
response = batch.submit_job(jobName=f"Bayota_solvetrial_using_{trialcon_name}",
jobQueue='Modeling',
jobDefinition='Modeling-Bayota:6',
dependsOn=[{'jobId': jobids['study'][studyid]['exp'][expid]['self']}],
containerOverrides={"command": CMD},
retryStrategy={'attempts': 10},
timeout={'attemptDurationSeconds': 3600}
)
print(f"submitted command: {CMD}")
print("Job ID is {}.".format(response['jobId']))
jobids['study'][studyid]['exp'][expid]['trial'][trialidstr] = dict()
jobids['study'][studyid]['exp'][expid]['trial'][trialidstr]['self'] = response['jobId']
print(jobids)
def myprint(d):
for k, v in d.items():
if isinstance(v, dict):
myprint(v)
else:
print(v, end=' ')
print('jobids all together:')
print('------------------------------')
print('jobids=', end='') # Note: this "jobids=" string is used as a key when extracting jobids from a file.
myprint(jobids)
print('\n------------------------------')
endtime = time.time()
print(f"time elapsed (seconds) since starting the batch_job_runner: {endtime - starttime}")
return 0 # a clean, no-issue, exit
choices=['debug', 'info', 'warning', 'error', 'critical'],
help="change logging level to {debug, info, warning, error, critical}")
opts = parser.parse_args()
# Set up logging
level_config = {'debug': logging.DEBUG, 'info': logging.INFO,
'warning': logging.WARNING, 'error': logging.ERROR, 'critical': logging.CRITICAL}
log_level = level_config[opts.loglvl.lower()]
logging.basicConfig(level=log_level)
logger = logging.getLogger(__name__)
logging.debug('This will get logged')
import pyomo.environ as pyo
from bayota_settings.base import get_bayota_version
logging.info("using bayota version '%s'" % get_bayota_version())
from bayota_settings.base import get_source_csvs_dir
logging.info("getting source csvs from: '%s'" % get_source_csvs_dir())
from bayota_util.spec_and_control_handler import read_spec
from bayom_e.data_handling.data_interface import get_dataplate
from bayom_e.model_handling.interface import check_for_problems_in_data_before_model_construction
from bayom_e.model_handling.builders.nonlinear import NonlinearVariant
from bayom_e.model_handling.builders.linear import LinearVariant
from bayom_e.model_handling.builders.modelbuilder import ModelBuilder
from bayom_e.model_handling.interface import build_model