class Plot(object):
'''
'''
def __init__(self, hdf5_file, nPlot=1, ntMax=0):
'''
Constructor
'''
self.diagnostics = Diagnostics(hdf5_file)
if ntMax > 0 and ntMax < self.diagnostics.nt:
self.nt = ntMax
else:
self.nt = self.diagnostics.nt
self.plot = PlotMHD2D(self.diagnostics,
args.hdf5_file.replace(".hdf5", ""), self.nt,
nPlot)
def update(self, itime):
self.diagnostics.read_from_hdf5(itime)
self.diagnostics.update_invariants(itime)
if itime > 0:
self.plot.add_timepoint()
self.plot.update()
def run(self):
for itime in range(1, self.nt + 1):
print("it = %4i" % (itime))
self.update(itime)
def main(**kwargs):
parser = argparse.ArgumentParser()
parser.add_argument("-rl", "--restore_last", help="restore last saved model", action="store_true")
parser.add_argument("-r", "--restore_path", help="path to model to be restored", type=str)
parser.add_argument("-opt", "--optimizer", default="entropy-sgd", help="Selected optimizer", type=str,
choices=['entropy-sgd', 'adam', 'momentum', 'sgd'])
parser.add_argument("-n", "--name", default="entropy-sgd", help="Checkpoint/Tensorboard label")
parser.add_argument("-d", "--dataset", default="cifar10", help="Dataset to train on (cifar10 || cifar100)",
type=str, choices=['cifar10', 'cifar100'])
parser.add_argument("-L", "--langevin_iterations", default=20, help="Number of Langevin iterations in inner loop.",
type=int)
args = parser.parse_args()
config = config_train
architecture = 'Layers: {} | Conv dropout: {} | Base LR: {} | SGLD Iterations {} | Epochs: {} | Optimizer: {}'.format(
config.n_layers,
config.conv_keep_prob,
config.learning_rate,
config.L,
config.num_epochs,
args.optimizer
)
Diagnostics.setup_dataset(args.dataset)
# Launch training
train(config_train, architecture, args)
class replay(object):
'''
'''
def __init__(self, hdf5_file, nPlot=1, write=False):
'''
Constructor
'''
self.diagnostics = Diagnostics(hdf5_file)
self.nPlot = nPlot
self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, nPlot,
write)
def init(self):
self.update(0)
def update(self, itime, final=False):
self.diagnostics.read_from_hdf5(itime)
self.diagnostics.update_invariants(itime)
if itime > 0:
self.plot.add_timepoint()
return self.plot.update(final=final)
def run(self):
for itime in range(1, self.diagnostics.nt + 1):
print("it = %4i" % (itime))
self.update(itime, final=(itime == self.diagnostics.nt))
class replay(object):
'''
'''
def __init__(self, hdf5_file, nPlot=1, output=0):
'''
Constructor
'''
self.diagnostics = Diagnostics(hdf5_file)
self.nPlot = nPlot
self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, nPlot,
output)
def init(self):
self.update(0)
def update(self, itime):
print(itime)
self.diagnostics.read_from_hdf5(itime)
self.diagnostics.update_invariants(itime)
if itime > 0:
self.plot.add_timepoint()
self.plot.update()
def run(self):
for itime in range(1, self.diagnostics.nt + 1):
self.update(itime)
def __init__(self, hdf5_file, nPlot=1):
'''
Constructor
'''
self.diagnostics = Diagnostics(hdf5_file)
self.nPlot = nPlot
self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, nPlot)
def diagnostics(self):
"""
>>> Product('DGKFL06JDHJP').diagnostics()
"""
if hasattr(self, "alternateDeviceId"):
diags = Diagnostics(alternateDeviceId=self.alternateDeviceId)
else:
diags = Diagnostics(serialNumber=self.serialNumber)
return diags.fetch()
def diagnostics(self):
"""
>>> Product('DGKFL06JDHJP').diagnostics()
"""
if hasattr(self, "alternateDeviceId"):
diags = Diagnostics(alternateDeviceId=self.alternateDeviceId)
else:
diags = Diagnostics(serialNumber=self.serialNumber)
return diags.fetch()
def __init__(self, hdf5_file, nPlot=1, ntMax=0):
'''
Constructor
'''
self.diagnostics = Diagnostics(hdf5_file)
if ntMax > 0 and ntMax < self.diagnostics.nt:
self.nt = ntMax
else:
self.nt = self.diagnostics.nt
self.plot = PlotMHD2D(self.diagnostics,
args.hdf5_file.replace(".hdf5", ""), self.nt,
nPlot)
def __init__(self, hdf5_file, nPlot=1, ntMax=0, write=False):
'''
Constructor
'''
self.diagnostics = Diagnostics(hdf5_file)
if ntMax > 0 and ntMax < self.diagnostics.nt:
self.nt = ntMax
else:
self.nt = self.diagnostics.nt
self.nPlot = nPlot
self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, self.nt,
nPlot, write)
def train(config, args):
start_time = time.time()
global_step, n_checkpoints, v_s1_best = 0, 0, 0.
ckpt = tf.train.get_checkpoint_state(directories.checkpoints)
# Build graph
cnn = Model(config, directories, args=args)
saver = tf.train.Saver()
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
train_handle = sess.run(cnn.train_iterator.string_handle())
test_handle = sess.run(cnn.test_iterator.string_handle())
if args.restore_last and ckpt.model_checkpoint_path:
# Continue training saved model
saver.restore(sess, ckpt.model_checkpoint_path)
print('{} restored.'.format(ckpt.model_checkpoint_path))
else:
if args.restore_path:
new_saver = tf.train.import_meta_graph('{}.meta'.format(args.restore_path))
new_saver.restore(sess, args.restore_path)
print('{} restored.'.format(args.restore_path))
sess.run(cnn.test_iterator.initializer)
for epoch in range(config.num_epochs):
sess.run(cnn.train_iterator.initializer)
# Run diagnostics
v_s1_best = Diagnostics.run_diagnostics(cnn, config_train, directories, sess, saver, train_handle,
test_handle, start_time, v_s1_best, epoch, args.name)
while True:
try:
# Update weights
sess.run([cnn.train_op, cnn.update_accuracy], feed_dict={cnn.training_phase: True,
cnn.handle: train_handle})
except tf.errors.OutOfRangeError:
print('End of epoch!')
break
except KeyboardInterrupt:
save_path = saver.save(sess, os.path.join(directories.checkpoints,
'{0}/bcmp_{0}_last.ckpt'.format(args.name)), global_step=epoch)
print('Interrupted, model saved to: ', save_path)
sys.exit()
save_path = saver.save(sess, os.path.join(directories.checkpoints,
'{0}/bcmp_{0}_end.ckpt'.format(args.name)),
global_step=epoch)
print("Training Complete. Model saved to file: {} Time elapsed: {:.3f} s".format(save_path, time.time()-start_time))
class replay(object):
'''
'''
def __init__(self, hdf5_file, nPlot=1):
'''
Constructor
'''
self.diagnostics = Diagnostics(hdf5_file)
self.nPlot = nPlot
self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, nPlot)
def init(self):
self.update(0)
def update(self, itime, final=False):
self.diagnostics.read_from_hdf5(itime)
self.diagnostics.update_invariants(itime)
if itime > 0:
self.plot.add_timepoint()
return self.plot.update(final=final)
def run(self):
for itime in range(1, self.diagnostics.nt + 1):
self.update(itime, final=(itime == self.diagnostics.nt))
def movie(self, outfile, fps=1):
self.plot.nPlot = 1
ani = animation.FuncAnimation(self.plot.figure,
self.update,
np.arange(1, self.diagnostics.nt + 1),
init_func=self.init,
repeat=False,
blit=True)
ani.save(outfile, fps=fps)
def __init__(self, model_root, data_dir, communities=None, tag=None):
"""
initilizer
inputs:
model_root: model root path <string>
communities: list of communities to setup <string>
data_repo: path to data repo <sting>
tag: (optional) tag to use as self.tag setup sub directory,
if not provided self.tag will be m<version>_d<version> <string>
postconditions:
see invatiants
"""
self.model_root = model_root
self.communities = communities
self.data_dir = data_dir
self.tag = tag
if tag is None:
self.tag = self.make_version_tag()
self.diagnostics = Diagnostics()
def requires(self):
# Public sources
yield PostsToDb()
# Internal sources
yield GomusToDb()
# Analysis tasks
yield AspectBasedSentimentAnalysis()
yield TopicModeling()
yield PredictionsToDb()
# Diagnostics
yield Diagnostics()
# Extended Tweet Gathering
yield TwitterExtendedDatasetToDB()
def __init__ (self, community_config, global_config = None, diag = None,
scalers = {'diesel price':1.0, 'diesel price adder':0},
intertie_config = None):
"""This class manages the input data for the an AAEM model instance
Parameters
----------
community_config: path to yaml file or dict
the dictionary or dictionary that would be loaded from the yaml file
should match the format to be validated for the model to run, unless
global_config is also provided, in which case all config values not
provided here have values in global_config
global_config: path to yaml file or dict, optional
Optional second config file of values that can be applied to many
model instances
diag: Diagnostics, optional
AAEM Diagnostics object for tracking messages from model
scalers: Dict, optional
Scalers to change model behaviour. The keys 'diesel price' and
'diesel price adder' are used in CommunityData, and will be multiplied
or added to the diesel prices respectively. This will carry over into
the electirc non-fuel prices.
Attributes
----------
data: Dict
Configuration data for model instance
diagnostics:Diagnostics
AAEM Diagnostics object for tracking messages from model
intertie: str, or None
status to track intertie placment, 'parent', 'child', or None
intertie_data: CommunityData
config data for the entire intertie if community is a
child community
"""
self.diagnostics = diag
if diag == None:
self.diagnostics = Diagnostics()
if type(community_config) is dict and type(global_config) is dict:
self.data = merge_configs(self.load_structure(), global_config)
self.data = merge_configs(self.data, community_config)
else:
self.data = self.load_config(community_config, global_config)
valid, reason = self.validate_config()
if not valid:
raise StandardError, 'INVALID CONFIG FILE: ' + reason
self.intertie = None
intertie = self.get_item('community', 'intertie')
if type (intertie) is list:
if self.get_item('community', 'model as intertie') :
self.intertie = 'parent'
else:
self.intertie = 'child'
self.intertie_data = None
## load if community is part of an intertie but not the intertie
## it's self. I.E. load info for Bethel or Oscarville,
## but not Bethel_intertie
if not self.intertie is None and \
not self.get_item('community', 'model as intertie'):
self.diagnostics.add_note(
'Community Data',
'Attempting to find intertie data'
)
## is the intertie_config a dict of file, also use same globals
if type(intertie_config) is dict \
or (type(intertie_config) is str \
and os.path.isfile(intertie_config)):
self.diagnostics.add_note(
'Community Data',
'using provided intertie_config argument'
)
self.intertie_data = CommunityData(
intertie_config,
global_config,
self.diagnostics,
scalers
)
## try to find the file
elif os.path.isfile(community_config):
rt_path = os.path.split(community_config)
it_file = \
self.get_item('community','intertie')[0].\
replace(' ','_').replace("'",'')\
+ '_intertie.yaml'
it_file = os.path.join(rt_path[0], it_file)
if os.path.isfile(it_file):
self.diagnostics.add_note(
'Community Data',
'Found interte data at ' + it_file
)
self.intertie_data = CommunityData(
it_file,
global_config,
self.diagnostics,
scalers
)
else:
self.diagnostics.add_note(
'Community Data',
'Could not find intertie_data Leaving it as None'
)
## self.intertie_data = None is initlized
## component spesifc plugins
for comp in comp_order:
config = import_module("aaem.components." + comp_lib[comp]).config
try:
plugins = config.plugins
except AttributeError:
continue
for plugin in plugins:
plugin(self, community_config, global_config, scalers)
convert = self.data['community']['diesel prices']
convert.index = [int(y) for y in convert.index]
convert.index.name = 'year'
convert = self.data['community']['electric prices']
convert.index = [int(y) for y in convert.index]
convert.index.name = 'year'
# modify diesel prices and electric prices
self.apply_scalers(scalers)
self.check_auto_disable_conditions ()
class CommunityData (object):
"""This class manages the input data for the an AAEM model instance
"""
def __init__ (self, community_config, global_config = None, diag = None,
scalers = {'diesel price':1.0, 'diesel price adder':0},
intertie_config = None):
"""This class manages the input data for the an AAEM model instance
Parameters
----------
community_config: path to yaml file or dict
the dictionary or dictionary that would be loaded from the yaml file
should match the format to be validated for the model to run, unless
global_config is also provided, in which case all config values not
provided here have values in global_config
global_config: path to yaml file or dict, optional
Optional second config file of values that can be applied to many
model instances
diag: Diagnostics, optional
AAEM Diagnostics object for tracking messages from model
scalers: Dict, optional
Scalers to change model behaviour. The keys 'diesel price' and
'diesel price adder' are used in CommunityData, and will be multiplied
or added to the diesel prices respectively. This will carry over into
the electirc non-fuel prices.
Attributes
----------
data: Dict
Configuration data for model instance
diagnostics:Diagnostics
AAEM Diagnostics object for tracking messages from model
intertie: str, or None
status to track intertie placment, 'parent', 'child', or None
intertie_data: CommunityData
config data for the entire intertie if community is a
child community
"""
self.diagnostics = diag
if diag == None:
self.diagnostics = Diagnostics()
if type(community_config) is dict and type(global_config) is dict:
self.data = merge_configs(self.load_structure(), global_config)
self.data = merge_configs(self.data, community_config)
else:
self.data = self.load_config(community_config, global_config)
valid, reason = self.validate_config()
if not valid:
raise StandardError, 'INVALID CONFIG FILE: ' + reason
self.intertie = None
intertie = self.get_item('community', 'intertie')
if type (intertie) is list:
if self.get_item('community', 'model as intertie') :
self.intertie = 'parent'
else:
self.intertie = 'child'
self.intertie_data = None
## load if community is part of an intertie but not the intertie
## it's self. I.E. load info for Bethel or Oscarville,
## but not Bethel_intertie
if not self.intertie is None and \
not self.get_item('community', 'model as intertie'):
self.diagnostics.add_note(
'Community Data',
'Attempting to find intertie data'
)
## is the intertie_config a dict of file, also use same globals
if type(intertie_config) is dict \
or (type(intertie_config) is str \
and os.path.isfile(intertie_config)):
self.diagnostics.add_note(
'Community Data',
'using provided intertie_config argument'
)
self.intertie_data = CommunityData(
intertie_config,
global_config,
self.diagnostics,
scalers
)
## try to find the file
elif os.path.isfile(community_config):
rt_path = os.path.split(community_config)
it_file = \
self.get_item('community','intertie')[0].\
replace(' ','_').replace("'",'')\
+ '_intertie.yaml'
it_file = os.path.join(rt_path[0], it_file)
if os.path.isfile(it_file):
self.diagnostics.add_note(
'Community Data',
'Found interte data at ' + it_file
)
self.intertie_data = CommunityData(
it_file,
global_config,
self.diagnostics,
scalers
)
else:
self.diagnostics.add_note(
'Community Data',
'Could not find intertie_data Leaving it as None'
)
## self.intertie_data = None is initlized
## component spesifc plugins
for comp in comp_order:
config = import_module("aaem.components." + comp_lib[comp]).config
try:
plugins = config.plugins
except AttributeError:
continue
for plugin in plugins:
plugin(self, community_config, global_config, scalers)
convert = self.data['community']['diesel prices']
convert.index = [int(y) for y in convert.index]
convert.index.name = 'year'
convert = self.data['community']['electric prices']
convert.index = [int(y) for y in convert.index]
convert.index.name = 'year'
# modify diesel prices and electric prices
self.apply_scalers(scalers)
self.check_auto_disable_conditions ()
#~ self.load_construction_multipliers(construction_multipliers)
def apply_scalers(self, scalers):
"""apply scalers to inupt variables
Parameters
----------
scalers: Dict
Scalers to change model behaviour. The keys 'diesel price' and
'diesel price adder' are used in CommunityData, and will be multiplied
or added to the diesel prices respectively. This will carry over into
the electirc non-fuel prices.
"""
## diesel
if scalers['diesel price'] != 1 or scalers['diesel price adder'] != 1:
self.diagnostics.add_note(
'Community Data',
'Adjusting disel and electric prices'
)
else:
return
self.data['community']['diesel prices'] = \
self.data['community']['diesel prices'] * scalers['diesel price'] +\
scalers['diesel price adder']
## electric
percent_diesel = \
float(self.data['community']['percent diesel generation']) / 100.0
efficiency = \
float(self.data['community']['diesel generation efficiency'])
adder = percent_diesel * \
self.data['community']['diesel prices'] / efficiency
self.data['community']['electric prices'] = \
float(self.data['community']['electric non-fuel price']) + adder
def check_auto_disable_conditions (self):
"""
check for any auto disable conditions and disable those components
"""
# no conditions at this time
pass
#~ st = self.get_item('Water & Wastewater Efficiency',"data").ix["assumption type used"]
#~ if st.values[0] == "UNKNOWN":
#~ self.set_item('Water & Wastewater Efficiency',"enabled", False)
#~ self.diagnostics.add_error("Community Data",
#~ ("(Checking Inputs) Water Wastewater system type unknown."
#~ " Fixing by disabling Wastewater component at runtime"))
def modify_diesel_prices(self,
scalers = {'diesel price':1.0, 'diesel price adder':0}):
pass
def modify_non_fuel_electricity_prices (self, N_slope_price = .15):
"""
calculate the electricity price
pre:
community: diesel generation efficiency should be a numeric type
> 0
community: elec non-fuel cost: is a floating point dollar value
community: diesel prices: is a diesel projections object.
post:
community: electric prices: is a data frame of dollar
values indexed by year
"""
# TODO: 1 is 100% need to change to a calculation
pass
def validate_config(self):
"""
validate a config library
pre:
lib should be a dictionary object
post:
returns true if lib is a valid config object; otherwise false
"""
return validate_dict(self.data, self.load_structure())
def load_structure (self):
"""
"""
try:
return self.structure
except AttributeError:
pass
structure = base_structure
for comp in comp_lib:
module = \
import_module('aaem.components.' + comp_lib[comp] + '.config')
structure = merge_configs(structure, module.structure)
self.structure = structure
return structure
def load_config (self, community_file, global_file = None):
"""
loads the input files and creates the model input object
pre:
community_file, and defaults should be .yaml files
post:
self.data is is usable
"""
community = read_config(community_file)
global_ = {}
if not global_file is None:
global_ = read_config(global_file)
return merge_configs(
merge_configs(
self.load_structure(),
global_
),
community
)
def get_item (self, section, key):
"""
get an item
pre:
self.data exists
section is a config section, and key is a key in said section
post:
returns an item
"""
return self.data[section][key]
def get_section (self, section):
"""
gets a sction
pre:
self.data exists
section is a config section
post:
returns a section library
"""
return self.data[section]
def set_item (self, section, key, data):
"""
set an item
pre:
self.data exists
section is a config section, and key is a key in said section, and
data is the type that make sense there.
post:
self.data[section][key] is data
"""
self.data[section][key] = data
def save (self, fname):
"""
save the inputs used
pre:
self.model_inputs exists, fname is the path to a file
pre:
a valid .yaml config file is created
"""
## save work around
import copy
copy = copy.deepcopy(self.data)
comment = "config used"
#~ conf, orders, comments, defaults = \
#~ config.get_config(config.non_component_config_sections)
#~ for comp in comp_lib:
#~ cfg = import_module("aaem.components." + comp_lib[comp]+ '.config')
#~ order = list(cfg.yaml_order) + \
#~ list(set(cfg.yaml_order) ^ set(cfg.yaml.keys()))
#~ orders[comp] = order
#~ comments[comp] = cfg.yaml_comments
#~ section_order = config.non_component_config_sections + comp_order
s_order = ['community'] + comp_order
i_order = {'community': base_order}
comments = base_comments
for comp in comp_lib:
module = import_module('aaem.components.' + comp_lib[comp])
i_order[comp] = module.config.order
comments[comp] = module.config.comments
save_config(fname,copy,
comments = comments,
s_order = s_order,
i_orders = i_order,
header = 'confiuration used to generate these results'
)
del copy
from flask import Flask
from flask import request
from google.auth.transport.urllib3 import AuthorizedHttp
from google.oauth2 import service_account
from diagnostics import Diagnostics
import settings
app = Flask(__name__)
credentials = service_account.Credentials.from_service_account_info(
json.loads(settings.GOOGLE_SERVICE_ACCOUNT_CREDENTIALS_JSON),
scopes=['https://www.googleapis.com/auth/bigquery.insertdata'])
diagnostics = Diagnostics(launched_at=datetime.datetime.utcnow())
@app.route('/log', methods=['POST'])
def log():
diagnostics.request_count += 1
auth = request.authorization
if not auth or auth.username != settings.LOG_DRAIN_USERNAME or auth.password != settings.LOG_DRAIN_PASSWORD:
diagnostics.unauthorized_count += 1
return '', 403
diagnostics.authorized_count += 1
log_records = []
for log_line in _parse_log_lines(request.data):
if log_line.startswith(settings.LOG_RECORD_PREFIX):
json_string = log_line.replace(settings.LOG_RECORD_PREFIX, '',
class Setup(object):
"""
setup the structure needed to run the model
class invariants:
self.model_root: is the model root path <string>
self.communities: list of communities to setup <string>
self.data_repo: path data repo <string>
self.tag: tag used a the directory to setup the model in
model_root <string>
"""
def __init__(self, model_root, data_dir, communities=None, tag=None):
"""
initilizer
inputs:
model_root: model root path <string>
communities: list of communities to setup <string>
data_repo: path to data repo <sting>
tag: (optional) tag to use as self.tag setup sub directory,
if not provided self.tag will be m<version>_d<version> <string>
postconditions:
see invatiants
"""
self.model_root = model_root
self.communities = communities
self.data_dir = data_dir
self.tag = tag
if tag is None:
self.tag = self.make_version_tag()
self.diagnostics = Diagnostics()
def make_version_tag(self):
"""
generate a version tag
precondition:
see invariants
'VERSION' file must exist in repo
outputs
returns tag
"""
data_version_file = os.path.join(self.data_dir, 'VERSION')
with open(data_version_file, 'r') as fd:
ver = fd.read().replace("\n", "")
ver = 'm' + __version__ + '_d' + ver
return ver
def setup_directories(self):
"""
setup the directories
preconditions:
see invariats
postconditions:
config and input_files are removed and then
config and input_files directories are created.
"""
setup_path = os.path.join(self.model_root, self.tag)
try:
shutil.rmtree(os.path.join(setup_path, "config"))
except OSError:
pass
os.makedirs(os.path.join(setup_path, "config"))
def setup_community_list(self):
"""
create the community list file from the repo
preconditions:
see invariants, community_list.csv sould exist in data repo
postcondition:
'__community_list.csv' saved in config directory
"""
config_path = os.path.join(self.model_root, self.tag, 'config',
'__community_list.csv')
src_path = os.path.join(self.data_dir, 'community_list.csv')
shutil.copy(src_path, config_path)
def write_preprocessor_metadata(self, save_path):
"""
write data metadata
inputs:
input_path: path to inputs directory <string>
outputs:
saves 'input_files_metadata.yaml' in "__metadata" subdirectory
"""
data_version_file = os.path.join(self.data_dir, 'VERSION')
with open(data_version_file, 'r') as fd:
ver = fd.read().replace("\n", "")
md_dir = os.path.join(save_path, "__metadata")
try:
os.makedirs(md_dir)
except OSError:
pass
#~ try:
#~ os.makedirs(os.path.join(md_dir,'diagnostic_files'))
#~ except OSError:
#~ pass
m = 'w'
with open(os.path.join(md_dir, 'preprocessor_metadata.yaml'),
m) as meta:
meta.write(
yaml.dump(
{
'upadted':
datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S"),
'data version':
ver
},
default_flow_style=False))
self.diagnostics.save_messages(os.path.join(md_dir, 'log.csv'))
data_version_file = os.path.join(self.data_dir, 'VERSION')
with open(data_version_file, 'r') as fd:
ver = fd.read().replace("\n", "")
z = zipfile.ZipFile(os.path.join(md_dir, "raw_data.zip"), "w")
for raw in [f for f in os.listdir(self.data_dir) if '.csv' in f]:
z.write(os.path.join(self.data_dir, raw), raw)
z.write(os.path.join(data_version_file), 'VERSION')
def load_communities(self):
""" Function doc """
data = read_csv(
os.path.join(self.model_root, self.tag, 'config',
'__community_list.csv'))
self.communities = [c for c in data['Community'].values]
def setup(self, force=False, ng_coms=[], make_globals=False):
"""
run the setup functionality
inputs:
force: (optional) overwrirte existing files <boolean>
outputs:
model structure is setup
"""
setup_path = os.path.join(self.model_root, self.tag)
if os.path.exists(setup_path) and force == False:
return False
self.setup_directories()
self.setup_community_list()
if self.communities is None:
self.load_communities()
f_path = os.path.join(self.model_root, self.tag, 'config')
for community in self.communities:
#~ print community
#~ f_path = os.path.join(self.model_root, self.tag, 'config')
preprocessor = Preprocessor(community,
self.data_dir,
diag=self.diagnostics,
process_intertie=False)
self.diagnostics.add_note('Preprocessing ' + community,
'---------')
if community in ng_coms:
preprocessor.run(show=True, ng_com=True)
else:
preprocessor.run(show=True)
if make_globals:
keys_to_split = KEYS_FOR_GLOBAL
preprocessor.save_config(f_path, keys_to_split)
f_name = os.path.join(f_path, '__global_config.yaml')
if not os.path.exists(f_name):
preprocessor.save_global_congfig(f_name, keys_to_split)
else:
preprocessor.save_config(f_path)
## the intertie, if it exists
try:
preprocessor = Preprocessor(community,
self.data_dir,
diag=self.diagnostics,
process_intertie=True)
self.diagnostics.add_note('Preprocessing ' + community,
'---------')
preprocessor.run(show=True)
if make_globals:
keys_to_split = KEYS_FOR_GLOBAL
preprocessor.save_config(f_path, keys_to_split)
else:
preprocessor.save_config(f_path)
except PreprocessorError:
pass
#~ self.setup_global_config()
#~ ids = self.setup_input_files()
#~ self.setup_community_configs()
#~ self.setup_construction_multipliers()
#~ self.setup_goals()
self.write_preprocessor_metadata(f_path)
return True
def run(self,
community_config,
global_config=None,
tag='',
scalers=None,
alt_save_name=None):
"""
run the model for a community
inputs:
community: the community <string>
c_config: (optional, default: none) alternate community config
file <string>
g_config: (optional, default: none) alternat global confing
file <string>
tag: (optional) tag for results dir <string>
outputs:
the model is run for a community/project/assigned 'name'
preconditions:
see invariants
postconditions:
None
"""
#~ print community_config
if scalers is None:
scalers = default_scalers
#~ if name is None:
#~ name = community
temp = tag
#~ if img_dir is None:
#~ if temp != '':
#~ temp = '_' + tag
#~ img_dir = os.path.join(self.model_root, 'results' + temp, 'plots')
#~ cd, fc, diag = self.setup_community(community, i_dir, c_config,
#~ g_config, c_mult, scalers)
diagnostics = Diagnostics()
community_data = CommunityData(community_config, global_config,
diagnostics, scalers)
name = community_data.get_item('community', 'file id')
#~ print name
forecast = Forecast(community_data, diagnostics, scalers)
comps_used = self.run_components(community_data, forecast, diagnostics,
scalers)
#~ name = community_data.get_item('community', 'file id')
self.save_components_output(comps_used,
name,
forecast,
tag,
alt_name=alt_save_name)
#~ self.save_forecast_output(forecast, name, img_dir, plot, tag)
self.save_input_files(community_data,
name,
tag,
alt_name=alt_save_name)
self.save_diagnostics(diagnostics, name, tag, alt_name=alt_save_name)
comps_used['community data'] = community_data
comps_used['forecast'] = forecast
#~ print name
#~ print 'rb', alt_save_name
self.store_results(comps_used, tag, name=alt_save_name)
def __init__(self,
community_data,
diag=None,
scalers={'kWh consumption': 1.0}):
"""
pre:
self.cd is a community_data instance.
post:
self.start_year < self.end_year are years(ints)
self.cd is a community_data instance.
"""
self.diagnostics = diag
if self.diagnostics == None:
self.diagnostics = Diagnostics()
self.cd = community_data
self.forecast_population()
#~ ## test block
#~ self.forecast_consumption(scalers['kWh consumption'])
#~ self.forecast_generation()
#~ return
if self.cd.get_item("community", "model electricity") is False:
pass
else:
try:
self.forecast_consumption(scalers['kWh consumption'])
self.forecast_generation()
self.forecast_average_kW()
except RuntimeError:
self.cd.set_item("community", "model financial", False)
self.cd.set_item("community", "model electricity", False)
self.diagnostics.add_warning(
'Forecast',
('Electricity data was insufficient to model'
' electricity or financial values. In community data'
' model financial, and model electricity have been'
' set to false.'))
self.calc_average_diesel_load()
self.cpi = np.array(self.cd.get_item('community', 'cpi multipliers'))
#~ DataFrame([
#~ 1, 0.9765625, 0.953674316, 0.931322575, 0.909494702, 0.88817842,
#~ 0.867361738, 0.847032947, 0.827180613, 0.807793567, 0.788860905,
#~ 0.770371978, 0.752316385, 0.734683969, 0.717464814, 0.700649232,
#~ 0.684227766, 0.668191178, 0.652530447, 0.637236765, 0.622301528,
#~ 0.607716336, 0.593472984, 0.579563461, 0.565979942, 0.552714788,
#~ 0.539760535, 0.527109897, 0.514755759, 0.502691171, 0.490909347,
#~ 0.479403659, 0.468167636, 0.457194957, 0.44647945, 0.436015088,
#~ 0.425795984, 0.415816391, 0.406070694, 0.396553412, 0.387259192,
#~ 0.378182804, 0.369319145, 0.360663227, 0.352210183, 0.343955257,
#~ 0.335893805, 0.328021294, 0.320333295, 0.312825484, 0.305493636,
#~ 0.298333629, 0.291341435, 0.28451312, 0.277844844, 0.271332855,
#~ 0.264973491, 0.258763175, 0.252698413, 0.246775794, 0.240991987],
#~ )
#~ last_cpi_year = self.cpi.index.tolist()[-1]
#~ if self.end_year < last_cpi_year:
#~ self.cpi = self.cpi[:self.end_year]
#~ elif self.end_year > last_cpi_year:
#~ self.diagnostics.add_note("Forecast",
#~ "extending cpi past avaiable data")
#~ last_cpi = float(self.cpi.ix[last_cpi_year])
#~ for i in range(last_cpi_year,self.end_year+1):
#~ self.cpi.ix[i] = last_cpi
#~ self.forecast_households()
self.heat_demand_cols = []
self.heating_fuel_cols = []
self.electric_columns = []
class Forecast(object):
""" Class doc """
def __init__(self,
community_data,
diag=None,
scalers={'kWh consumption': 1.0}):
"""
pre:
self.cd is a community_data instance.
post:
self.start_year < self.end_year are years(ints)
self.cd is a community_data instance.
"""
self.diagnostics = diag
if self.diagnostics == None:
self.diagnostics = Diagnostics()
self.cd = community_data
self.forecast_population()
#~ ## test block
#~ self.forecast_consumption(scalers['kWh consumption'])
#~ self.forecast_generation()
#~ return
if self.cd.get_item("community", "model electricity") is False:
pass
else:
try:
self.forecast_consumption(scalers['kWh consumption'])
self.forecast_generation()
self.forecast_average_kW()
except RuntimeError:
self.cd.set_item("community", "model financial", False)
self.cd.set_item("community", "model electricity", False)
self.diagnostics.add_warning(
'Forecast',
('Electricity data was insufficient to model'
' electricity or financial values. In community data'
' model financial, and model electricity have been'
' set to false.'))
self.calc_average_diesel_load()
self.cpi = np.array(self.cd.get_item('community', 'cpi multipliers'))
#~ DataFrame([
#~ 1, 0.9765625, 0.953674316, 0.931322575, 0.909494702, 0.88817842,
#~ 0.867361738, 0.847032947, 0.827180613, 0.807793567, 0.788860905,
#~ 0.770371978, 0.752316385, 0.734683969, 0.717464814, 0.700649232,
#~ 0.684227766, 0.668191178, 0.652530447, 0.637236765, 0.622301528,
#~ 0.607716336, 0.593472984, 0.579563461, 0.565979942, 0.552714788,
#~ 0.539760535, 0.527109897, 0.514755759, 0.502691171, 0.490909347,
#~ 0.479403659, 0.468167636, 0.457194957, 0.44647945, 0.436015088,
#~ 0.425795984, 0.415816391, 0.406070694, 0.396553412, 0.387259192,
#~ 0.378182804, 0.369319145, 0.360663227, 0.352210183, 0.343955257,
#~ 0.335893805, 0.328021294, 0.320333295, 0.312825484, 0.305493636,
#~ 0.298333629, 0.291341435, 0.28451312, 0.277844844, 0.271332855,
#~ 0.264973491, 0.258763175, 0.252698413, 0.246775794, 0.240991987],
#~ )
#~ last_cpi_year = self.cpi.index.tolist()[-1]
#~ if self.end_year < last_cpi_year:
#~ self.cpi = self.cpi[:self.end_year]
#~ elif self.end_year > last_cpi_year:
#~ self.diagnostics.add_note("Forecast",
#~ "extending cpi past avaiable data")
#~ last_cpi = float(self.cpi.ix[last_cpi_year])
#~ for i in range(last_cpi_year,self.end_year+1):
#~ self.cpi.ix[i] = last_cpi
#~ self.forecast_households()
self.heat_demand_cols = []
self.heating_fuel_cols = []
self.electric_columns = []
#~ def calc_electricity_values (self):
#~ """
#~ pre:
#~ 'fc_electricity_used' should contain the kWh used for each key type
#~ post:
#~ self.electricity_totals is a array of yearly values of total kWh used
#~ """
#~ kWh = self.fc_specs["electricity"]
#~ print kWh
#~ years = kWh.T.keys().values
#~ self.yearly_res_kWh = DataFrame({"year":years,
#~ "total":kWh['consumption residential'].values}).set_index("year")
#~ self.yearly_total_kWh = DataFrame({"year":years,
#~ "total":kWh['consumption'].values}).set_index("year")
#~ self.average_nr_kWh = kWh['consumption non-residential'].values[-3:].mean()
#~ if np.isnan(self.average_nr_kWh):
#~ temp = kWh['consumption non-residential']
#~ self.average_nr_kWh = temp[ np.logical_not(np.isnan(temp))].mean()
#~ self.yearly_nr_kWh = DataFrame({"year":years,
#~ "total":kWh['consumption non-residential'].values}).set_index("year")
#~ print self.average_nr_kWh
def forecast_population(self):
"""
pre:
tbd.
post:
self.population is a array of estimated populations for each
year between start and end
"""
# pop forecast is preprocessed now
population = self.cd.get_item('community', "population")
self.p_map = DataFrame(population["population_qualifier"])
self.population = DataFrame(population["population"])
#~ last_pop_year = self.population.index.tolist()[-1]
#~ if self.end_year < last_pop_year:
#~ self.population = self.population[:self.end_year]
#~ elif self.end_year > last_pop_year:
#~ self.diagnostics.add_note("Forecast",
#~ "extending popultation past avaiable data")
#~ last_pop = int(round(self.population.ix[last_pop_year]))
#~ for i in range(last_pop_year,self.end_year+1):
#~ self.population.ix[i] = last_pop
if len(self.p_map[self.p_map == "M"]) < 10:
msg = "the data range is < 10 for input population "\
"check population.csv in the models data directory"
self.diagnostics.add_warning("forecast", msg)
def forecast_consumption(self, consumption_scaler=1.0):
"""
pre:
tbd.
post:
self.consumption is a array of estimated kWh consumption for each
year between start and end
"""
total_consumption = \
self.cd.get_item('community','utility info')['consumption'] \
* consumption_scaler
residential_consumption = \
self.cd.get_item('community','utility info')\
['consumption residential'] * consumption_scaler
non_residential_consumption = \
self.cd.get_item('community','utility info')\
['consumption non-residential'] * consumption_scaler
#~ print self.cd.get_item('community','utility info')
index = list(residential_consumption.index)
if len(residential_consumption) < 10:
msg = "the data range is < 10 for input consumption "\
"check electricity.csv in the models data directory"
self.diagnostics.add_warning("Forecast: consumption", msg)
population = self.population.ix[index]
if any(population.isnull()):
v = population.isnull().values.T.tolist()[0]
for year in population[v].index.values.tolist():
index.remove(year)
if any(residential_consumption.isnull()):
to_remove = \
residential_consumption[residential_consumption.isnull()].index
for year in to_remove:
#~ print year
index.remove(year)
self.diagnostics.add_note("forecast",
"years with measured consumption and population " + str(index) +\
". List used to generate fit function")
population = self.population.ix[index]['population'].values
#~ self.measured_consumption = self.yearly_total_kWh.ix[idx]
consumption = residential_consumption.ix[index].values
if len(population) < 10:
self.diagnostics.add_warning("forecast",
"the data range is < 10 matching years for "\
"population and consumption "\
"check population.csv and electricity.csv "\
"in the models data directory")
try:
#~ print population,consumption
m, b = np.polyfit(population, consumption, 1)
except TypeError:
raise RuntimeError, "Known population & consumption do not overlap"
forcasted_consumption = (m * self.population + b
) #+ self.average_nr_kWh
forcasted_consumption.columns = ['consumption residential']
forcasted_consumption['consumption_qualifier'] = "P"
forcasted_consumption['consumption residential']\
[residential_consumption.ix[index].index] = \
residential_consumption.ix[index]
if consumption_scaler == 1.0:
## if the scaler is not 1 then none of the values are really
## measured
forcasted_consumption['consumption_qualifier'].ix[index] = \
(residential_consumption * np.nan).fillna("M")
mean_non_res_con = non_residential_consumption.values[-3:].mean()
if np.isnan(mean_non_res_con):
self.diagnostics.add_note(
'Forecast', ('Average of last 3 known years was null,'
' averaging all non null years'))
mean_non_res_con = \
non_residential_consumption\
[ ~non_residential_consumption.isnull()].mean()
#~ print non_residential_consumption.values
#~ print mean_non_res_con
forcasted_consumption['consumption non-residential'] = mean_non_res_con
#~ print forcasted_consumption
forcasted_consumption['consumption non-residential']\
[non_residential_consumption.ix[index].index] = \
non_residential_consumption.ix[index]
forcasted_consumption['consumption'] = \
forcasted_consumption['consumption non-residential'] + \
forcasted_consumption['consumption residential']
#~ forcasted_consumption['consumption non-residential']\
#~ [total_consumption.ix[index].index] = total_consumption.ix[index]
## don't forecast backwards
forcasted_consumption = \
forcasted_consumption.ix[residential_consumption.index[0]:]
self.consumption = forcasted_consumption
#~ print self.consumption
def forecast_generation(self):
"""
forecast generation
pre:
tbd.
self.consumption should be a float array of kWh/yr values
post:
self.generation is a array of estimated kWh generation for each
year between start and end
"""
measured_generation = \
self.cd.get_item('community','utility info')['net generation']
generation = self.consumption['consumption']/\
(1.0-self.cd.get_item('community','line losses')/100.0)
generation = DataFrame(generation)
generation.columns = ['generation']
generation = generation.round(-3)
index = set(measured_generation.index) & set(generation.index)
generation['generation'][index] = measured_generation.ix[index]
types = [
u'generation diesel', u'generation hydro',
u'generation natural gas', u'generation wind', u'generation solar',
u'generation biomass'
]
generation_types = self.cd.get_item('community', 'utility info')[types]
backup_type = 'generation diesel'
if (generation_types[['generation natural gas']].fillna(0)\
!= 0).any().bool():
backup_type = 'generation natural gas'
## forecast each type
for fuel in types:
current_fuel = generation_types[fuel]
#~ print current_rfuel
name = fuel.replace('generation ', '')
## get the hypothetical limit
try:
current_generation = self.cd.get_item(
'community', name + ' generation limit')
except KeyError:
current_generation = 0
## is there a non-zero limit defined
if current_generation != 0:
if fuel == 'generation wind':
measured = \
current_fuel[current_fuel.notnull()].values[-3:].mean()
max_wind_percent = .2
if measured > (current_generation * max_wind_percent):
pass
else:
generation[
fuel] = current_generation * max_wind_percent
generation[fuel].ix[index] = current_fuel.ix[index]
continue
else:
generation[fuel] = current_generation
generation[fuel].ix[index] = current_fuel.ix[index]
continue
else:
current_generation = \
current_fuel[current_fuel.notnull()].values[-3:].mean()
## no generation found
if current_generation == 0:
generation[fuel] = current_generation
continue
if fuel in [u'generation wind', u'generation hydro']:
generation[fuel] = current_generation
generation[fuel].ix[index] = current_fuel.ix[index]
for i in generation[fuel].ix[current_fuel.index[-1] +
1:].index:
generation[fuel].ix[i] = \
generation[fuel].ix[i-3:i-1].mean()
else:
generation[fuel] = current_generation
generation[fuel].ix[index] = current_fuel.ix[index]
#~ print fuel, current_generation
#~ print generation['generation hydro']
## scale back any excess
for fuel in [
'generation wind', 'generation hydro', 'generation biomass'
]:
current_fuel = generation_types[fuel]
fuel_sums = generation[types].sum(1)
#~ print fuel_sums
if any(generation['generation'] < fuel_sums):
#~ print 'a'
if any(generation['generation'] < generation[fuel]):
#~ print 'b'
msg = "scaling generation(" + fuel + ") where geneation(" + \
fuel + ") > total generation"
self.diagnostics.add_note('forecast', msg)
generation[fuel] = generation[fuel] -\
(fuel_sums - generation['generation'])
generation[fuel][generation[fuel] < 0] = 0
generation[fuel].ix[index] = current_fuel.ix[index]
corrected_backup_type = generation['generation'] - \
(generation[types].sum(1) - generation[backup_type])
#~ print corrected_backup_type
corrected_backup_type[corrected_backup_type < 0] = 0
generation[backup_type] = corrected_backup_type
generation[backup_type].ix[index] = generation_types[backup_type].ix[
index]
self.generation = generation
def forecast_average_kW(self):
"""
forecast the average kW used per community per year
pre:
post:
"""
self.average_kW = (self.consumption['consumption']/ 8760.0)\
/(1-self.cd.get_item('community','line losses'))
self.average_kW.columns = ["average load"]
def forecast_households(self):
"""
forcast # of houselholds
pre:
post:
"""
peeps_per_house = float(self.base_pop) / \
self.cd.get_item('Residential Buildings',
'data').ix['total_occupied']
self.households = \
np.round(self.population / np.float64(peeps_per_house))
self.households.columns = ["households"]
def get_population(self, start, end=None):
"""
get population values from the population forecast.
pre:
start is a year where start >= self.start_year
end(if provided) is a year where start <= end < self.end_year
post:
returns a float or list of floats
"""
if end is None:
return self.population.ix[start].T.values[0]
## dynamic extension
existing_len = len(self.population.ix[start:end])
extend_by = (end + 1) - start - existing_len
if extend_by > 0:
extend = DataFrame(index=range(
self.population.index[-1] + 1,
self.population.index[-1] + 1 + extend_by),
columns=['population'])
extend['population'] = self.population.iloc[-1]['population']
population = self.population.ix[start:end].append(extend)
else:
# -1 to ensure same behavour
population = self.population.ix[start:end]
return population['population'].values
def get_consumption(self, start, end=None):
"""
get consumption values from the consumption forecast.
pre:
start is a year where start >= self.start_year
end(if provided) is a year where start <= end < self.end_year
post:
returns a float or list of floats
"""
if end is None:
return self.consumption.ix[start]['consumption']
## dynamic extension
existing_len = len(self.consumption.ix[start:end])
extend_by = (end + 1) - start - existing_len
if extend_by > 0:
extend = DataFrame(index=range(
self.consumption.index[-1] + 1,
self.consumption.index[-1] + extend_by + 1),
columns=['consumption'])
extend['consumption'] = self.consumption.iloc[-1]['consumption']
consumption = \
DataFrame(self.consumption.ix[start:end]['consumption']).\
append(extend)
else:
# -1 to ensure same behavour
consumption = \
DataFrame(self.consumption['consumption'].ix[start:end])
return consumption['consumption'].values
def get_generation(self, start, end=None):
"""
get population values from the population forecast.
pre:
start is a year where start >= self.start_year
end(if provided) is a year where start <= end < self.end_year
post:
returns a float or list of floats
"""
if end is None:
return self.generation.ix[start]['generation']
## dynamic extension
existing_len = len(self.generation.ix[start:end])
extend_by = (end + 1) - start - existing_len
if extend_by > 0:
extend = DataFrame(index=range(
self.generation.index[-1] + 1,
self.generation.index[-1] + extend_by + 1),
columns=['generation'])
extend['generation'] = self.generation.iloc[-1]['generation']
generation = \
DataFrame(self.generation.ix[start:end]['generation']).\
append(extend)
else:
generation = \
DataFrame(self.generation['generation'].ix[start:end])
return generation['generation'].values
def get_households(self, start, end=None):
"""
get households values from the households forecast.
pre:
start is a year where start >= self.start_year
end(if provided) is a year where start <= end < self.end_year
post:
returns a float or list of floats
"""
if end is None:
return self.households.ix[start].T.values[0]
## dynamic extension
existing_len = len(self.households.ix[start:end])
extend_by = (end + 1) - start - existing_len
if extend_by > 0:
extend = DataFrame(index=range(
self.households.index[-1] + 1,
self.households.index[-1] + extend_by + 1),
columns=['households'])
extend['households'] = self.households.iloc[-1]['households']
households = self.households.ix[start:end].append(extend)
else:
# -1 to ensure same behavour
households = self.households.ix[start:end]
return households['households'].values
#~ def save_forecast (self, path, png_path = None, do_plots = False):
#~ """
#~ save the forecast to a csv
#~ pre:
#~ everything needs to be foretasted
#~ path: path to each communities output dir
#~ png_path: altenat location for pngs
#~ post:
#~ saves 3 files
#~ """
#~ tag = self.cd.get_item("community", "name").replace(" ", "_") + "_"
#~ pathrt = os.path.join(path, tag)
#~ if png_path is None:
#~ if do_plots:
#~ os.makedirs(os.path.join(path,"images"))
#~ png_path = os.path.join(path, 'images',tag)
#~ epng_path = png_path
#~ hdpng_path = png_path
#~ hfpng_path = png_path;
#~ gpng_path = png_path
#~ else:
#~ epng_path = os.path.join(png_path,'electric_forecast',tag)
#~ try:
#~ if do_plots:
#~ os.makedirs(os.path.join(png_path,'electric_forecast'))
#~ except OSError:
#~ pass
#~ gpng_path = os.path.join(png_path,'generation_forecast',tag)
#~ try:
#~ if do_plots:
#~ os.makedirs(os.path.join(png_path,'generation_forecast'))
#~ except OSError:
#~ pass
#~ hdpng_path = os.path.join(png_path,'heat_demand_forecast',tag)
#~ try:
#~ if do_plots:
#~ os.makedirs(os.path.join(png_path,'heat_demand_forecast'))
#~ except OSError:
#~ pass
#~ hfpng_path = os.path.join(png_path,'heating_fuel_forecast',tag)
#~ try:
#~ if do_plots:
#~ os.makedirs(os.path.join(png_path,'heating_fuel_forecast'))
#~ except OSError:
#~ pass
#~ if self.cd.intertie != 'child' and \
#~ self.cd.get_item("community","model electricity"):
##start = datetime.now()
#~ self.save_electric(pathrt, epng_path, do_plots)
#~ self.save_generation_forecast(pathrt, gpng_path, do_plots)
##print "saving electric:" + str(datetime.now() - start)
#~ if self.cd.intertie != 'parent' and \
#~ self.cd.get_item("community","model heating fuel"):
##start = datetime.now()
#~ self.save_heat_demand(pathrt, hdpng_path, do_plots)
##print "saving heat demand:" + str(datetime.now() - start)
##start = datetime.now()
#~ self.save_heating_fuel(pathrt, hfpng_path, do_plots)
##print "saving heating fuel:" + str( datetime.now() - start)
#~ def add_heat_demand_column (self, key, year_col, data_col):
#~ """
#~ add a column to be saved with the heat demand forecast
#~ pre:
#~ Key is a string. year_col is a numpy array of values, date_col
#~ is a corresponding array of years
#~ post:
#~ a dataframe is added to self.heat_demand_cols
#~ """
#~ self.heat_demand_cols.append(DataFrame({"year":year_col,
#~ key:data_col}).set_index("year"))
#~ def add_heating_fuel_column (self, key, year_col, data_col):
#~ """
#~ add a column to be saved with the heating fuel forecast
#~ pre:
#~ Key is a string. year_col is a numpy array of values, date_col
#~ is a corresponding array of years
#~ post:
#~ a dataframe is added to self.heating_fuel_cols
#~ """
#~ self.heating_fuel_cols.append(DataFrame({"year":year_col,
#~ key:data_col}).set_index("year"))
#~ def generate_electric_output_dataframe (self):
#~ """
#~ """
#~ from copy import deepcopy
#~ kWh_con = deepcopy(self.consumption)
#~ kWh_con.columns = ["total_electricity_consumed [kWh/year]",
#~ 'residential_electricity_consumed [kWh/year]',
#~ 'non-residential_electricity_consumed [kWh/year]']
#~ c_map = copy.deepcopy(self.c_map)
#~ c_map.columns = ["total_electricity_consumed_qualifier"]
#~ kWh_gen = self.generation
#~ kWh_gen.columns = ["total_electricity_generation [kWh/year]"]
#~ g_map = copy.deepcopy(c_map)
#~ g_map.columns = ["total_electricity_generation_qualifier"]
#~ community = copy.deepcopy(self.c_map)
#~ community.columns = ["community"]
#~ community.ix[:] = self.cd.get_item("community","name")
#~ try:
#~ self.electric_dataframe = \
#~ concat([community,self.population.round().astype(int),
#~ self.p_map, kWh_con.round().astype(int), c_map,
#~ kWh_gen.round().astype(int), g_map] ,axis=1)
#~ except ValueError:
#~ self.electric_dataframe = None
#~ def save_electric (self, csv_path, png_path, do_plots):
#~ """
#~ save the electric forecast
#~ pre:
#~ self.population, consumption, and generation. should be dataframes
#~ as calculated in this class. path should be the path to the output
#~ directory.
#~ post:
#~ a file, electricity_forecast.csv, is save in the output directory.
#~ """
#~ self.generate_electric_output_dataframe()
#~ self.save_electric_csv(csv_path)
#~ if do_plots:
#~ self.save_electric_png(png_path)
#~ def save_electric_csv (self, path):
#~ """
#~ save the electric forecast
#~ pre:
#~ self.population, consumption, and generation. should be dataframes
#~ as calculated in this class. path should be the path to the output
#~ directory.
#~ post:
#~ a file, electricity_forecast.csv, is save in the output directory.
#~ """
#~ if self.electric_dataframe is None:
#~ self.diagnostics.add_warning("Forecast",
#~ ("when saving null values were found in "
#~ "generation/consumption data. Electricity "
#~ "forecast is suspect not saving summary(csv)"))
#~ return
#~ data = self.electric_dataframe
#~ #not joining path adding file name
#~ f_name = path + "electricity_forecast.csv"
#~ fd = open(f_name ,"w")
#~ fd.write("# Electricity Forecast for " + \
#~ self.cd.get_item("community","name") + "\n")
#~ fd.write("# Qualifier info: \n")
#~ fd.write("# M indicates a measured value \n")
#~ fd.write("# P indicates a projected value \n")
#~ fd.write("# I indicates a value carried over from the input data. May be projected or measured see input data metadata.\n")
#~ fd.close()
#~ data.index = data.index.values.astype(int)
#~ data.to_csv(f_name, index_label="year", mode = 'a')
#~ def save_electric_png (self, path):
#~ """ Function doc """
#~ if self.electric_dataframe is None:
#~ self.diagnostics.add_warning("Forecast",
#~ ("when saving null values were found in "
#~ "generation/consumption data. Electricity "
#~ "forecast is suspect not saving summary(png)"))
#~ return
#~ path = path+ "electricity_forecast.png"
#~ start = self.c_map[self.c_map['consumption_qualifier'] == 'P'].index[0]
#~ df2 = self.electric_dataframe[['population',
#~ 'total_electricity_consumed [kWh/year]',
#~ 'total_electricity_generation [kWh/year]',
#~ 'residential_electricity_consumed [kWh/year]',
#~ 'non-residential_electricity_consumed [kWh/year]']]
#~ name = self.cd.get_item("community","name") + ' Electricity Forecast'
#~ fig, ax = plot.setup_fig(name ,'years','population')
#~ ax1 = plot.add_yaxis(fig,'kWh')
#~ c_dict = {'population': [min(1,r*4) for r in colors.red],
#~ 'total_electricity_consumed [kWh/year]':colors.orange,
#~ 'total_electricity_generation [kWh/year]':colors.green,
#~ 'residential_electricity_consumed [kWh/year]':colors.yellow,
#~ 'non-residential_electricity_consumed [kWh/year]':colors.violet
#~ }
#~ plot.plot_dataframe(ax1,df2,ax,['population'],
#~ {'population':'population',
#~ 'total_electricity_consumed [kWh/year]':'consumption',
#~ 'total_electricity_generation [kWh/year]':'generation',
#~ 'residential_electricity_consumed [kWh/year]':'residential consumption',
#~ 'non-residential_electricity_consumed [kWh/year]':'non residential consumption'},
#~ c_dict)
#~ fig.subplots_adjust(right=.85)
#~ fig.subplots_adjust(left=.12)
#~ plot.add_vertical_line(ax,start, 'forecasting starts' )
#~ plot.add_horizontal_line(ax1,0)
#~ plot.create_legend(fig,.20)
#~ plot.save(fig,path)
#~ plot.clear(fig)
#~ def generate_generation_forecast_dataframe(self):
#~ """
#~ """
#~ data = copy.deepcopy(self.generation)
#~ g_map = copy.deepcopy(self.c_map)
#~ g_map.columns = ["generation_qualifier"]
#~ order = []
#~ try:
#~ for col in ['generation total', 'generation diesel',
#~ 'generation hydro',
#~ 'generation natural gas', 'generation wind',
#~ 'generation solar', 'generation biomass']:
#~ if col == 'generation total' and \
#~ any(np.isnan(data[col].ix[self.start_year:])):
#~ raise ValueError, "nans found"
#~ kWh = col.replace(" ", "_") + " [kWh/year]"
#~ mmbtu = col.replace(" ", "_") + " [mmbtu/year]"
#~ data[kWh] = data[col].fillna(0).round().astype(int)
#~ data[mmbtu] = (data[col] / constants.mmbtu_to_kWh)\
#~ .fillna(0).round().astype(int)
#~ del data[col]
#~ order += [kWh, mmbtu]
#~ data = concat([self.population.round().astype(int),
#~ self.p_map,g_map,data],axis=1)
#~ data["community"] = self.cd.get_item("community","name")
#~ self.generation_forecast_dataframe = \
#~ data[[data.columns[-1]] + data.columns[:-1].tolist()]
#~ del data
#~ except ValueError:
#~ self.generation_forecast_dataframe = None
#~ def save_generation_forecast (self, csv_path, png_path, do_plots):
#~ self.generate_generation_forecast_dataframe()
#~ if self.generation_forecast_dataframe is None:
#~ self.diagnostics.add_warning("Forecast",
#~ ("when saving null values were found in "
#~ "generation data. generation generation "
#~ "forecast is suspect not saving summary(csv and png)"))
#~ else:
#~ self.save_generation_forecast_csv(csv_path)
#~ if do_plots:
#~ self.save_generation_forecast_png(png_path)
#~ def save_generation_forecast_csv (self, path):
#~ """
#~ """
#~ out_file = path + "generation_forecast.csv"
#~ fd = open(out_file, 'w')
#~ fd.write("# Generation forecast\n")
#~ fd.write(("# The column 'generation_qualifier' applies to all "
#~ "of the following generation columns"))
#~ fd.write("# Qualifier info: \n")
#~ fd.write("# M indicates a measured value \n")
#~ fd.write("# P indicates a projected value \n")
#~ fd.write("# I indicates a value carried over from the input data. May be projected or measured see input data metadata.\n")
#~ fd.close()
#~ self.generation_forecast_dataframe.to_csv(out_file,
#~ index_label="year", mode = 'a')
#~ def save_generation_forecast_png (self, path):
#~ """ Function doc """
#~ path = path + "generation_forecast.png"
#~ png_list = ['population',
#~ 'generation_total [kWh/year]',
#~ 'generation_diesel [kWh/year]',
#~ 'generation_hydro [kWh/year]',
#~ 'generation_natural_gas [kWh/year]',
#~ 'generation_wind [kWh/year]',
#~ 'generation_solar [kWh/year]',
#~ 'generation_biomass [kWh/year]']
#~ png_dict = {'population':'population',
#~ 'generation_total [kWh/year]':'total',
#~ 'generation_diesel [kWh/year]':'diesel',
#~ 'generation_hydro [kWh/year]':'hydro',
#~ 'generation_natural_gas [kWh/year]':'natural gas',
#~ 'generation_wind [kWh/year]':'wind',
#~ 'generation_solar [kWh/year]':'solar',
#~ 'generation_biomass [kWh/year]':'biomass'}
#~ c_dict = {'population': [min(1,r*4) for r in colors.red],
#~ 'generation_total [kWh/year]':colors.jet,
#~ 'generation_diesel [kWh/year]':colors.red,
#~ 'generation_hydro [kWh/year]':colors.blue,
#~ 'generation_natural_gas [kWh/year]':colors.violet,
#~ 'generation_wind [kWh/year]':[ min(1,r*2) for r in colors.blue],
#~ 'generation_solar [kWh/year]':colors.orange,
#~ 'generation_biomass [kWh/year]':colors.avacado}
#~ temp = []
#~ for i in png_list:
#~ if all(self.generation_forecast_dataframe[i].fillna(0) == 0):
#~ continue
#~ temp.append(i)
#~ png_list = temp
#~ if len(png_list) == 3:
#~ png_list = list(set(png_list).\
#~ difference(['generation_total [kWh/year]']))
#~ df2 = self.generation_forecast_dataframe[png_list]
#~ plot_name = self.cd.get_item("community","name") +' Generation Forecast'
#~ fig, ax = plot.setup_fig(plot_name ,'years','population')
#~ ax1 = plot.add_yaxis(fig,'Generation kWh')
#~ plot.plot_dataframe(ax1,df2,ax,['population'],png_dict,c_dict)
#~ fig.subplots_adjust(right=.85)
#~ fig.subplots_adjust(left=.12)
#~ start = self.c_map[self.c_map['consumption_qualifier'] == 'P'].index[0]
#~ plot.add_vertical_line(ax,start, 'forecasting starts' )
#~ plot.add_horizontal_line(ax1,0)
#~ plot.create_legend(fig,.20)
#~ plot.save(fig,path)
#~ plot.clear(fig)
#~ def generate_heat_demand_dataframe (self):
#~ """
#~ """
#~ data = concat([self.population.round().astype(int), self.p_map] + \
#~ self.heat_demand_cols,axis=1)
#~ for key in data.keys():
#~ try:
#~ col = data[key]
#~ idx = np.logical_not(col.apply(np.isnan))
#~ lv = col[idx].tail(1).values[0]
#~ yr = col[idx].tail(1).index[0]
#~ col[col.index > yr] = lv
#~ data[key] = col.round()
#~ except (TypeError, IndexError, AttributeError):
#~ pass
#~ idx = data.keys()[['mmbtu' in s for s in data.keys()]]
#~ data["heat_energy_demand_total [mmbtu/year]"] = data[idx].sum(1).round()
#~ data["community"] = self.cd.get_item("community","name")
#~ self.heat_demand_dataframe = data[[data.columns[-1]] + data.columns[:-1].tolist()]
#~ del data
#~ def save_heat_demand (self,csv_path, png_path, do_plots):
#~ """ Function doc """
#~ self.generate_heat_demand_dataframe()
#~ self.save_heat_demand_csv(csv_path)
#~ if do_plots:
#~ self.save_heat_demand_png(png_path)
#~ def save_heat_demand_csv (self, path):
#~ """
#~ save the heat demand forecast
#~ pre:
#~ self.population should be a dataframe. self.heat_demad_cols should
#~ be populated using add_heat_demad_column. path should be the path to the
#~ output directory.
#~ post:
#~ a file, heat_demand_forecast.csv, is save in the output directory.
#~ """
#~ f_name = path + "heat_demand_forecast.csv"
#~ data = self.heat_demand_dataframe
#~ fd = open(f_name ,"w")
#~ fd.write("# Heat Demand Forecast for " + \
#~ self.cd.get_item("community","name") + "\n")
#~ fd.write("# Qualifier info: \n")
#~ fd.write("# M indicates a measured value \n")
#~ fd.write("# P indicates a projected value \n")
#~ fd.write("# I indicates a value carried over from the input data. May be projected or measured see input data metadata.\n")
#~ fd.close()
#~ data.index = data.index.values.astype(int)
#~ data.to_csv(f_name, index_label="year", mode = 'a')
#~ def save_heat_demand_png(self,path):
#~ """
#~ """
#~ path = path+ "heat_demand_forecast.png"
##start = self.c_map[self.c_map['consumption_qualifier'] == 'P'].index[0]
#~ cols = ['population']
#~ for col in self.heat_demand_dataframe.columns:
#~ if "[mmbtu/year]" in col:
#~ cols += [col]
#~ df2 = self.heat_demand_dataframe[cols]
#~ name = self.cd.get_item("community","name") + ' Heat Demand Forecast'
#~ fig, ax = plot.setup_fig(name ,'years','population')
#~ ax1 = plot.add_yaxis(fig,'Heat Demand MMBtu')
#~ c_dict = {'population': [min(1,r*4) for r in colors.red],
#~ 'heat_energy_demand_residential [mmbtu/year]':colors.yellow,
#~ 'heat_energy_demand_water-wastewater [mmbtu/year]':colors.cobalt,
#~ 'heat_energy_demand_non-residential [mmbtu/year]':colors.green,
#~ 'heat_energy_demand_total [mmbtu/year]':colors.jet}
#~ plot.plot_dataframe(ax1,df2,ax,['population'],
#~ {'population':'population',
#~ 'heat_energy_demand_residential [mmbtu/year]':'residential',
#~ 'heat_energy_demand_water-wastewater [mmbtu/year]':'water & wastewater',
#~ 'heat_energy_demand_non-residential [mmbtu/year]':'non-residential',
#~ 'heat_energy_demand_total [mmbtu/year]':'total'},
#~ c_dict)
#~ fig.subplots_adjust(right=.85)
#~ fig.subplots_adjust(left=.12)
##plot.add_vertical_line(ax,start, 'forecasting starts' )
#~ plot.add_horizontal_line(ax1,0)
#~ plot.create_legend(fig,.2)
#~ plot.save(fig,path)
#~ plot.clear(fig)
#~ def generate_heating_fuel_dataframe(self):
#~ """ """
#~ data = concat([self.population.round().astype(int), self.p_map] + \
#~ self.heating_fuel_cols, axis=1)
#~ for key in data.keys():
#~ try:
#~ col = data[key]
#~ idx = np.logical_not(col.apply(np.isnan))
#~ lv = col[idx].tail(1).values[0]
#~ yr = col[idx].tail(1).index[0]
#~ col[col.index > yr] = lv
#~ data[key] = col.round()
#~ except (TypeError,IndexError, AttributeError):
#~ pass
#~ hf_gal_idx = data.keys()[['gallons' in s for s in data.keys()]]
#~ hf_gal_idx = hf_gal_idx[['heating_fuel' in s for s in hf_gal_idx]]
#~ hf_btu_idx = data.keys()[['mmbtu' in s for s in data.keys()]]
#~ hf_btu_idx = hf_btu_idx[['heating_fuel' in s for s in hf_btu_idx]]
#~ data["heating_fuel_total_consumed [gallons/year]"] = \
#~ data[hf_gal_idx].sum(1).round()
#~ data["heating_fuel_total_consumed [mmbtu/year]"]= \
#~ data[hf_btu_idx].sum(1).round()
#~ data["community"] = self.cd.get_item("community","name")
#~ self.heating_fuel_dataframe = \
#~ data[[data.columns[-1]] + data.columns[:-1].tolist()]
#~ del data
#~ def save_heating_fuel(self, csv_path, png_path, do_plots):
#~ """ """
##start = datetime.now()
#~ self.generate_heating_fuel_dataframe()
##print "saving heating fuel *1:" + str( datetime.now() - start)
##start = datetime.now()
#~ self.save_heating_fuel_csv(csv_path)
##print "saving heating fuel *2:" + str( datetime.now() - start)
##start = datetime.now()
#~ if do_plots:
#~ self.save_heating_fuel_png(png_path)
##print "saving heating fuel *3:" + str( datetime.now() - start)
#~ def save_heating_fuel_csv (self, path):
#~ """
#~ save the heating fuel
#~ pre:
#~ self.population should be a dataframe. self.heat_demad_cols should
#~ be populated using add_heating_fuel_column. path should be the path to
#~ the output directory.
#~ post:
#~ a file, heating_fuel_forecast.csv, is save in the output directory.
#~ """
#~ f_name = path + "heating_fuel_forecast.csv"
#~ data = self.heating_fuel_dataframe
#~ fd = open(f_name ,"w")
#~ fd.write("# Heating Fuel Forecast for " + \
#~ self.cd.get_item("community","name") + "\n")
#~ fd.write("# Qualifier info: \n")
#~ fd.write("# M indicates a measured value \n")
#~ fd.write("# P indicates a projected value \n")
#~ fd.write("# I indicates a value carried over from the input data. May be projected or measured see input data metadata.\n")
#~ fd.close()
#~ data.index = data.index.values.astype(int)
#~ data.to_csv(f_name, index_label="year", mode = 'a')
#~ def save_heating_fuel_png (self, path):
#~ """"""
#~ path = path+ "heating_fuel_forecast.png"
##start = self.c_map[self.c_map['consumption_qualifier'] == 'P'].index[0]
#~ cols = ['population']
#~ for col in self.heating_fuel_dataframe.columns:
#~ if "[mmbtu/year]" in col:
#~ cols += [col]
#~ df2 = self.heating_fuel_dataframe[cols]
#~ name = self.cd.get_item("community","name") + ' Heating fuel Forecast'
#~ fig, ax = plot.setup_fig(name ,'years','population')
#~ ax1 = plot.add_yaxis(fig,'Heating fuel MMBtu')
#~ c_dict = {'population': [min(1,r*4) for r in colors.red],
#~ "heating_fuel_residential_consumed [mmbtu/year]":
#~ colors.red,
#~ "cords_wood_residential_consumed [mmbtu/year]":
#~ colors.avacado,
#~ "gas_residential_consumed [mmbtu/year]":
#~ colors.violet,
#~ "electric_residential_consumed [mmbtu/year]":
#~ colors.goldenrod,
#~ "propane_residential_consumed [mmbtu/year]":
#~ colors.orange,
#~ "heating_fuel_water-wastewater_consumed [mmbtu/year]":
#~ colors.cobalt,
#~ "heating_fuel_non-residential_consumed [mmbtu/year]":
#~ colors.green,
#~ "heating_fuel_total_consumed [mmbtu/year]":colors.jet,}
#~ plot.plot_dataframe(ax1,df2,ax,['population'],
#~ {"population":'population',
#~ "heating_fuel_residential_consumed [mmbtu/year]":
#~ 'heating oil - residential',
#~ "cords_wood_residential_consumed [mmbtu/year]":
#~ 'wood - residential',
#~ "gas_residential_consumed [mmbtu/year]":
#~ 'natural gas - residential',
#~ "electric_residential_consumed [mmbtu/year]":
#~ 'electric - residential',
#~ "propane_residential_consumed [mmbtu/year]":
#~ 'propane - residential',
#~ "heating_fuel_water-wastewater_consumed [mmbtu/year]":
#~ 'heating oil - water wastewater',
#~ "heating_fuel_non-residential_consumed [mmbtu/year]":
#~ 'heating fuel(all) - non - residential',
#~ "heating_fuel_total_consumed [mmbtu/year]":'total'},
#~ c_dict)
#~ fig.subplots_adjust(right=.85)
#~ fig.subplots_adjust(left=.12)
##plot.add_vertical_line(ax,start, 'forecasting starts' )
#~ plot.add_horizontal_line(ax1,0)
#~ plot.create_legend(fig,.30)
#~ plot.save(fig,path)
#~ plot.clear(fig)
def calc_average_diesel_load(self):
"""
"""
#~ self.average_diesel_load = 0
try:
generation = self.generation['generation diesel']
except AttributeError:
self.diagnostics.add_error('Forecast',
('no generation diesel to caclulate'
' yearly average load. setting to 0'))
generation = 0.0
self.yearly_average_diesel_load = generation / constants.hours_per_year
from people import Customer
from logplot import LogPlot
from diagnostics import Diagnostics
C = Customer('Self', 1)
diagnostics = Diagnostics()
results = diagnostics.from_file(
'C:\\Users\\Imraan\\PycharmProjects\\crossLang\\corellian_74G.log')
LogPlot().plot(results, './results')
A = Customer('Imraan', 30)
C.deposit(50)
def diagnostics(self):
"""
>>> Product('DGKFL06JDHJP').diagnostics()
"""
diags = Diagnostics(serialNumber=self.serialNumber)
return diags.fetch()
class MainScreen(App):
ser_bytes = NumericProperty()
ser = serial.Serial(baudrate=115200)
data = ListProperty([0,0,0,0,0,0])
info = ObjectProperty()
errors = []
sm = ScreenManager()
main = Main(name='main')
data_screen = Data_Screen(name='data')
diagnostics = Diagnostics(name='diagnostics')
sm.add_widget(main)
sm.add_widget(data_screen)
sm.add_widget(diagnostics)
def build(self):
Window.clearcolor = (0, 0, 0, 1)
Window.fullscreen = 'auto'
Window.bind(on_key_down=self.press)
#create data dictionary
self.info = extract('telemetry2.xlsx')
self.main.info = self.info
self.diagnostics.info = self.info
self.diagnostics.tabs.info = self.info
#Clock.schedule_interval(self.readserial, 0.05)
Clock.schedule_interval(self.noserial, 0.5)
return self.sm
#keyboard inputs
def press(self, keyboard, keycode, text, modifiers, type):
if keycode == 275: # ascii code for left
if self.sm.current == 'main':
self.sm.transition = SlideTransition(direction="left")
self.sm.current = 'data'
elif self.sm.current == 'diagnostics':
self.sm.transition = SlideTransition(direction="left")
self.sm.current = 'main'
elif keycode == 276: # ascii code for right
if self.sm.current == 'main':
self.sm.transition = SlideTransition(direction="right")
self.sm.current = 'diagnostics'
elif self.sm.current == 'data':
self.sm.transition = SlideTransition(direction="right")
self.sm.current = 'main'
elif keycode == 27: # ascii code for ESC
App.stop(self)
elif keycode == 273: #up
if self.sm.current == 'diagnostics':
if self.diagnostics.tabs.opacity == 0: # reset
self.diagnostics.rv.pos_hint = {"x": 0.02, "y": 0.02}
self.diagnostics.rv.size_hint = (0.7, 0.2)
self.diagnostics.tabs.pos_hint = {"x": 0.02, "y": 0.25}
self.diagnostics.tabs.size_hint = (0.7, 0.65)
self.diagnostics.tabs.opacity = 1
else:
self.diagnostics.rv.pos_hint= {"x": 0.02, "y": 0.02}
self.diagnostics.rv.size_hint = (0.7, 0.05)
self.diagnostics.tabs.pos_hint = {"x": 0.02, "y": 0.08}
self.diagnostics.tabs.size_hint = (0.7, 0.9)
elif keycode == 274: #down
if self.sm.current == 'diagnostics':
if self.diagnostics.tabs.size_hint[1] == 0.9: # reset
self.diagnostics.rv.pos_hint = {"x": 0.02, "y": 0.02}
self.diagnostics.rv.size_hint = (0.7, 0.2)
self.diagnostics.tabs.pos_hint = {"x": 0.02, "y": 0.25}
self.diagnostics.tabs.size_hint = (0.7, 0.65)
else:
self.diagnostics.rv.pos_hint = {"x": 0.02, "y": 0.02}
self.diagnostics.rv.size_hint = (0.7, 0.95)
self.diagnostics.tabs.pos_hint = {"x": 0.02, "y": 0.06}
self.diagnostics.tabs.size_hint = (0.7, 0)
self.diagnostics.tabs.opacity = 0
return True
def readserial(self, dt):
try:
global ser
if ser.in_waiting:
#print(ser.in_waiting)
#length = ser.in_waiting
#temp = ser.readline()
self.data = ser.readline().split()
ser.reset_input_buffer()
#print(len(self.data))
#print(self.data[23])
self.main.data = self.data
#print(len(self.main.data))
#print(str(int(self.data[28])))
#self.diagnostics.data = self.data
#self.data_screen.data = self.data
except Exception as e:
print(e)
self.main.pc_status.serial_status = False
try:
ser = serial.Serial(
baudrate='115200',
timeout=1,
port=str(serial.tools.list_ports.comports()[0]).split()[0]
#port="/dev/ttyUSB0"
)
self.main.pc_status.serial_status = True
except Exception:
pass
def noserial(self, dt):
self.data = [random.randint(0,255) for i in range(60)] #instead of readserial
#passes all the values
self.main.data = self.data
self.diagnostics.update(self.data)
#self.data_screen.data = self.data
def update_errors2(self):
self.errors = []
for i in range(len(self.data)):
self.errors.append(0)
if int(self.data[7]) > 80:
self.errors[7] = 2
def update_errors(self):
#stringA
if self.data[0] == 0:
for i in range(1,250):
if self.data[i] > 250 or self.data[i] < 5:
self.errors.append({'datapos':i, 'value': self.data[i], 'timestamp': datetime.now().strftime("%H:%M:%S")})
#stringB
elif self.data[0] == 1:
pass
#stringC
elif self.data[0] == 2:
pass
#log and remove the errors if they bunch up
if len(self.errors) > 20:
self.errors = []
type=int,
default=1,
help='plot every i\'th frame')
parser.add_argument('-ntmax',
metavar='i',
type=int,
default=0,
help='limit to i points in time')
args = parser.parse_args()
print
print("Replay run with " + args.hdf5_file)
print
diagnostics = Diagnostics(args.hdf5_file)
ntMax = args.ntmax
nPlot = args.np
if ntMax > 0 and ntMax < diagnostics.nt:
nt = ntMax
else:
nt = diagnostics.nt
plot = PlotEnergy(diagnostics, args.hdf5_file.replace(".hdf5", ""), nt,
nPlot)
print
print("Replay finished.")
print
def birnn_dynamic(x, config, training, attention=False):
# reshape outputs to [batch_size, max_time_steps, n_features]
#max_time = tf.shape(x)[1]
#rnn_inputs = tf.reshape(x, [-1, max_time, config.embedding_dim])
#rnn_inputs = x
rnn_inputs = tf.reshape(x, [-1, config.max_seq_len, config.n_features])
sequence_lengths = Diagnostics.length(rnn_inputs)
init = tf.contrib.layers.xavier_initializer()
# Choose rnn cell type
if config.rnn_cell == 'lstm':
args = {
'num_units': config.hidden_units,
'forget_bias': 1.0,
'state_is_tuple': True
}
base_cell = tf.nn.rnn_cell.LSTMCell
elif config.rnn_cell == 'gru':
args = {'num_units': config.hidden_units}
base_cell = tf.nn.rnn_cell.GRUCell
elif config.rnn_cell == 'layer_norm':
args = {
'num_units': config.hidden_units,
'forget_bias': 1.0,
'dropout_keep_prob': config.recurrent_keep_prob
}
base_cell = tf.contrib.rnn.LayerNormBasicLSTMCell
cell = base_cell
if config.output_keep_prob < 1:
# rnn_inputs = tf.nn.dropout(rnn_inputs, self.keep_prob)
fwd_cells = [
tf.nn.rnn_cell.DropoutWrapper(
cell(**args),
output_keep_prob=config.output_keep_prob,
variational_recurrent=True,
dtype=tf.float32) for _ in range(config.rnn_layers)
]
bwd_cells = [
tf.nn.rnn_cell.DropoutWrapper(
cell(**args),
output_keep_prob=config.output_keep_prob,
variational_recurrent=True,
dtype=tf.float32) for _ in range(config.rnn_layers)
]
else:
fwd_cells = [cell(**args) for _ in range(config.rnn_layers)]
bwd_cells = [cell(**args) for _ in range(config.rnn_layers)]
fwd_cells = tf.contrib.rnn.MultiRNNCell(fwd_cells)
bwd_cells = tf.contrib.rnn.MultiRNNCell(bwd_cells)
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=fwd_cells,
cell_bw=bwd_cells,
inputs=rnn_inputs,
sequence_length=sequence_lengths,
dtype=tf.float32,
parallel_iterations=128)
birnn_output = tf.concat(outputs, 2)
if attention: # invoke soft attention mechanism - attend to different particles
summary_vector = attention_A(birnn_output,
config.attention_dim,
my_method=False)
else: # Select last relevant output
summary_vector = Diagnostics.last_relevant(birnn_output,
sequence_lengths)
print(summary_vector.get_shape().as_list())
# Fully connected layer for classification
with tf.variable_scope("fc"):
logits_RNN = tf.layers.dense(summary_vector,
units=config.n_classes,
kernel_initializer=init)
return logits_RNN
