class Setup(object):
"""
setup the structure needed to run the model
class invariants:
self.model_root: is the model root path <string>
self.communities: list of communities to setup <string>
self.data_repo: path data repo <string>
self.tag: tag used a the directory to setup the model in
model_root <string>
"""
def __init__(self, model_root, data_dir, communities=None, tag=None):
"""
initilizer
inputs:
model_root: model root path <string>
communities: list of communities to setup <string>
data_repo: path to data repo <sting>
tag: (optional) tag to use as self.tag setup sub directory,
if not provided self.tag will be m<version>_d<version> <string>
postconditions:
see invatiants
"""
self.model_root = model_root
self.communities = communities
self.data_dir = data_dir
self.tag = tag
if tag is None:
self.tag = self.make_version_tag()
self.diagnostics = Diagnostics()
def make_version_tag(self):
"""
generate a version tag
precondition:
see invariants
'VERSION' file must exist in repo
outputs
returns tag
"""
data_version_file = os.path.join(self.data_dir, 'VERSION')
with open(data_version_file, 'r') as fd:
ver = fd.read().replace("\n", "")
ver = 'm' + __version__ + '_d' + ver
return ver
def setup_directories(self):
"""
setup the directories
preconditions:
see invariats
postconditions:
config and input_files are removed and then
config and input_files directories are created.
"""
setup_path = os.path.join(self.model_root, self.tag)
try:
shutil.rmtree(os.path.join(setup_path, "config"))
except OSError:
pass
os.makedirs(os.path.join(setup_path, "config"))
def setup_community_list(self):
"""
create the community list file from the repo
preconditions:
see invariants, community_list.csv sould exist in data repo
postcondition:
'__community_list.csv' saved in config directory
"""
config_path = os.path.join(self.model_root, self.tag, 'config',
'__community_list.csv')
src_path = os.path.join(self.data_dir, 'community_list.csv')
shutil.copy(src_path, config_path)
def write_preprocessor_metadata(self, save_path):
"""
write data metadata
inputs:
input_path: path to inputs directory <string>
outputs:
saves 'input_files_metadata.yaml' in "__metadata" subdirectory
"""
data_version_file = os.path.join(self.data_dir, 'VERSION')
with open(data_version_file, 'r') as fd:
ver = fd.read().replace("\n", "")
md_dir = os.path.join(save_path, "__metadata")
try:
os.makedirs(md_dir)
except OSError:
pass
#~ try:
#~ os.makedirs(os.path.join(md_dir,'diagnostic_files'))
#~ except OSError:
#~ pass
m = 'w'
with open(os.path.join(md_dir, 'preprocessor_metadata.yaml'),
m) as meta:
meta.write(
yaml.dump(
{
'upadted':
datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S"),
'data version':
ver
},
default_flow_style=False))
self.diagnostics.save_messages(os.path.join(md_dir, 'log.csv'))
data_version_file = os.path.join(self.data_dir, 'VERSION')
with open(data_version_file, 'r') as fd:
ver = fd.read().replace("\n", "")
z = zipfile.ZipFile(os.path.join(md_dir, "raw_data.zip"), "w")
for raw in [f for f in os.listdir(self.data_dir) if '.csv' in f]:
z.write(os.path.join(self.data_dir, raw), raw)
z.write(os.path.join(data_version_file), 'VERSION')
def load_communities(self):
""" Function doc """
data = read_csv(
os.path.join(self.model_root, self.tag, 'config',
'__community_list.csv'))
self.communities = [c for c in data['Community'].values]
def setup(self, force=False, ng_coms=[], make_globals=False):
"""
run the setup functionality
inputs:
force: (optional) overwrirte existing files <boolean>
outputs:
model structure is setup
"""
setup_path = os.path.join(self.model_root, self.tag)
if os.path.exists(setup_path) and force == False:
return False
self.setup_directories()
self.setup_community_list()
if self.communities is None:
self.load_communities()
f_path = os.path.join(self.model_root, self.tag, 'config')
for community in self.communities:
#~ print community
#~ f_path = os.path.join(self.model_root, self.tag, 'config')
preprocessor = Preprocessor(community,
self.data_dir,
diag=self.diagnostics,
process_intertie=False)
self.diagnostics.add_note('Preprocessing ' + community,
'---------')
if community in ng_coms:
preprocessor.run(show=True, ng_com=True)
else:
preprocessor.run(show=True)
if make_globals:
keys_to_split = KEYS_FOR_GLOBAL
preprocessor.save_config(f_path, keys_to_split)
f_name = os.path.join(f_path, '__global_config.yaml')
if not os.path.exists(f_name):
preprocessor.save_global_congfig(f_name, keys_to_split)
else:
preprocessor.save_config(f_path)
## the intertie, if it exists
try:
preprocessor = Preprocessor(community,
self.data_dir,
diag=self.diagnostics,
process_intertie=True)
self.diagnostics.add_note('Preprocessing ' + community,
'---------')
preprocessor.run(show=True)
if make_globals:
keys_to_split = KEYS_FOR_GLOBAL
preprocessor.save_config(f_path, keys_to_split)
else:
preprocessor.save_config(f_path)
except PreprocessorError:
pass
#~ self.setup_global_config()
#~ ids = self.setup_input_files()
#~ self.setup_community_configs()
#~ self.setup_construction_multipliers()
#~ self.setup_goals()
self.write_preprocessor_metadata(f_path)
return True
