Exemplo n.º 1
0
class Plot(object):
    '''
    
    '''
    def __init__(self, hdf5_file, nPlot=1, ntMax=0):
        '''
        Constructor
        '''

        self.diagnostics = Diagnostics(hdf5_file)

        if ntMax > 0 and ntMax < self.diagnostics.nt:
            self.nt = ntMax
        else:
            self.nt = self.diagnostics.nt

        self.plot = PlotMHD2D(self.diagnostics,
                              args.hdf5_file.replace(".hdf5", ""), self.nt,
                              nPlot)

    def update(self, itime):
        self.diagnostics.read_from_hdf5(itime)
        self.diagnostics.update_invariants(itime)

        if itime > 0:
            self.plot.add_timepoint()

        self.plot.update()

    def run(self):
        for itime in range(1, self.nt + 1):
            print("it = %4i" % (itime))
            self.update(itime)
Exemplo n.º 2
0
def main(**kwargs):
    parser = argparse.ArgumentParser()
    parser.add_argument("-rl", "--restore_last", help="restore last saved model", action="store_true")
    parser.add_argument("-r", "--restore_path", help="path to model to be restored", type=str)
    parser.add_argument("-opt", "--optimizer", default="entropy-sgd", help="Selected optimizer", type=str,
        choices=['entropy-sgd', 'adam', 'momentum', 'sgd'])
    parser.add_argument("-n", "--name", default="entropy-sgd", help="Checkpoint/Tensorboard label")
    parser.add_argument("-d", "--dataset", default="cifar10", help="Dataset to train on (cifar10 || cifar100)",
            type=str, choices=['cifar10', 'cifar100'])
    parser.add_argument("-L", "--langevin_iterations", default=20, help="Number of Langevin iterations in inner loop.",
            type=int)
    args = parser.parse_args()
    config = config_train

    architecture = 'Layers: {} | Conv dropout: {} | Base LR: {} | SGLD Iterations {} | Epochs: {} | Optimizer: {}'.format(
                    config.n_layers,
                    config.conv_keep_prob,
                    config.learning_rate,
                    config.L,
                    config.num_epochs,
                    args.optimizer
    )

    Diagnostics.setup_dataset(args.dataset)

    # Launch training
    train(config_train, architecture, args)
Exemplo n.º 3
0
class replay(object):
    '''
    
    '''
    def __init__(self, hdf5_file, nPlot=1, write=False):
        '''
        Constructor
        '''

        self.diagnostics = Diagnostics(hdf5_file)

        self.nPlot = nPlot
        self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, nPlot,
                              write)

    def init(self):
        self.update(0)

    def update(self, itime, final=False):
        self.diagnostics.read_from_hdf5(itime)
        self.diagnostics.update_invariants(itime)

        if itime > 0:
            self.plot.add_timepoint()

        return self.plot.update(final=final)

    def run(self):
        for itime in range(1, self.diagnostics.nt + 1):
            print("it = %4i" % (itime))
            self.update(itime, final=(itime == self.diagnostics.nt))
Exemplo n.º 4
0
class replay(object):
    '''
    
    '''
    def __init__(self, hdf5_file, nPlot=1, output=0):
        '''
        Constructor
        '''

        self.diagnostics = Diagnostics(hdf5_file)

        self.nPlot = nPlot
        self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, nPlot,
                              output)

    def init(self):
        self.update(0)

    def update(self, itime):
        print(itime)
        self.diagnostics.read_from_hdf5(itime)
        self.diagnostics.update_invariants(itime)

        if itime > 0:
            self.plot.add_timepoint()

        self.plot.update()

    def run(self):
        for itime in range(1, self.diagnostics.nt + 1):
            self.update(itime)
Exemplo n.º 5
0
    def __init__(self, hdf5_file, nPlot=1):
        '''
        Constructor
        '''

        self.diagnostics = Diagnostics(hdf5_file)

        self.nPlot = nPlot
        self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, nPlot)
Exemplo n.º 6
0
    def diagnostics(self):
        """
        >>> Product('DGKFL06JDHJP').diagnostics()
        """
        if hasattr(self, "alternateDeviceId"):
            diags = Diagnostics(alternateDeviceId=self.alternateDeviceId)
        else:
            diags = Diagnostics(serialNumber=self.serialNumber)

        return diags.fetch()
Exemplo n.º 7
0
    def diagnostics(self):
        """
        >>> Product('DGKFL06JDHJP').diagnostics()
        """
        if hasattr(self, "alternateDeviceId"):
            diags = Diagnostics(alternateDeviceId=self.alternateDeviceId)
        else:
            diags = Diagnostics(serialNumber=self.serialNumber)

        return diags.fetch()
Exemplo n.º 8
0
    def __init__(self, hdf5_file, nPlot=1, ntMax=0):
        '''
        Constructor
        '''

        self.diagnostics = Diagnostics(hdf5_file)

        if ntMax > 0 and ntMax < self.diagnostics.nt:
            self.nt = ntMax
        else:
            self.nt = self.diagnostics.nt

        self.plot = PlotMHD2D(self.diagnostics,
                              args.hdf5_file.replace(".hdf5", ""), self.nt,
                              nPlot)
Exemplo n.º 9
0
    def __init__(self, hdf5_file, nPlot=1, ntMax=0, write=False):
        '''
        Constructor
        '''

        self.diagnostics = Diagnostics(hdf5_file)

        if ntMax > 0 and ntMax < self.diagnostics.nt:
            self.nt = ntMax
        else:
            self.nt = self.diagnostics.nt

        self.nPlot = nPlot
        self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, self.nt,
                              nPlot, write)
Exemplo n.º 10
0
def train(config, args):

    start_time = time.time()
    global_step, n_checkpoints, v_s1_best = 0, 0, 0.
    ckpt = tf.train.get_checkpoint_state(directories.checkpoints)
    
    # Build graph
    cnn = Model(config, directories, args=args)
    saver = tf.train.Saver()

    with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess:
        sess.run(tf.global_variables_initializer())
        sess.run(tf.local_variables_initializer())
        train_handle = sess.run(cnn.train_iterator.string_handle())
        test_handle = sess.run(cnn.test_iterator.string_handle())

        if args.restore_last and ckpt.model_checkpoint_path:
            # Continue training saved model
            saver.restore(sess, ckpt.model_checkpoint_path)
            print('{} restored.'.format(ckpt.model_checkpoint_path))
        else:
            if args.restore_path:
                new_saver = tf.train.import_meta_graph('{}.meta'.format(args.restore_path))
                new_saver.restore(sess, args.restore_path)
                print('{} restored.'.format(args.restore_path))

        sess.run(cnn.test_iterator.initializer)

        for epoch in range(config.num_epochs):
            sess.run(cnn.train_iterator.initializer)

            # Run diagnostics
            v_s1_best = Diagnostics.run_diagnostics(cnn, config_train, directories, sess, saver, train_handle,
                test_handle, start_time, v_s1_best, epoch, args.name)
            while True:
                try:
                    # Update weights
                    sess.run([cnn.train_op, cnn.update_accuracy], feed_dict={cnn.training_phase: True,
                        cnn.handle: train_handle})

                except tf.errors.OutOfRangeError:
                    print('End of epoch!')
                    break

                except KeyboardInterrupt:
                    save_path = saver.save(sess, os.path.join(directories.checkpoints,
                        '{0}/bcmp_{0}_last.ckpt'.format(args.name)), global_step=epoch)
                    print('Interrupted, model saved to: ', save_path)
                    sys.exit()

        save_path = saver.save(sess, os.path.join(directories.checkpoints,
                               '{0}/bcmp_{0}_end.ckpt'.format(args.name)),
                               global_step=epoch)

    print("Training Complete. Model saved to file: {} Time elapsed: {:.3f} s".format(save_path, time.time()-start_time))
Exemplo n.º 11
0
class replay(object):
    '''
    
    '''
    def __init__(self, hdf5_file, nPlot=1):
        '''
        Constructor
        '''

        self.diagnostics = Diagnostics(hdf5_file)

        self.nPlot = nPlot
        self.plot = PlotMHD2D(self.diagnostics, self.diagnostics.nt, nPlot)

    def init(self):
        self.update(0)

    def update(self, itime, final=False):
        self.diagnostics.read_from_hdf5(itime)
        self.diagnostics.update_invariants(itime)

        if itime > 0:
            self.plot.add_timepoint()

        return self.plot.update(final=final)

    def run(self):
        for itime in range(1, self.diagnostics.nt + 1):
            self.update(itime, final=(itime == self.diagnostics.nt))

    def movie(self, outfile, fps=1):
        self.plot.nPlot = 1

        ani = animation.FuncAnimation(self.plot.figure,
                                      self.update,
                                      np.arange(1, self.diagnostics.nt + 1),
                                      init_func=self.init,
                                      repeat=False,
                                      blit=True)
        ani.save(outfile, fps=fps)
    def __init__(self, model_root, data_dir, communities=None, tag=None):
        """
        initilizer 
        
        inputs:
            model_root: model root path <string>
            communities: list of communities to setup <string>
            data_repo: path to data repo <sting>
            tag: (optional) tag to use as self.tag setup sub directory,
                if not provided self.tag will be m<version>_d<version> <string> 
            
        postconditions:
            see invatiants
        """
        self.model_root = model_root
        self.communities = communities
        self.data_dir = data_dir

        self.tag = tag
        if tag is None:
            self.tag = self.make_version_tag()
        self.diagnostics = Diagnostics()
    def requires(self):

        # Public sources
        yield PostsToDb()

        # Internal sources
        yield GomusToDb()

        # Analysis tasks
        yield AspectBasedSentimentAnalysis()
        yield TopicModeling()
        yield PredictionsToDb()

        # Diagnostics
        yield Diagnostics()

        # Extended Tweet Gathering
        yield TwitterExtendedDatasetToDB()
    def __init__ (self, community_config, global_config = None, diag = None, 
        scalers = {'diesel price':1.0, 'diesel price adder':0},
        intertie_config = None):
        """This class manages the input data for the an AAEM model instance
        
        Parameters
        ----------
        community_config: path to yaml file or dict
            the dictionary or dictionary that would be loaded from the yaml file 
        should match the format to be validated for the model to run, unless
        global_config is also provided, in which case all config values not 
        provided here have values in global_config
        global_config: path to yaml file or dict, optional
            Optional second config file of values that can be applied to many
        model instances
        diag: Diagnostics, optional
            AAEM Diagnostics object for tracking messages from model
        scalers: Dict, optional
            Scalers to change model behaviour. The keys 'diesel price' and
        'diesel price adder' are used in CommunityData, and will be multiplied 
        or added to the diesel prices respectively. This will carry over into
        the electirc non-fuel prices.
        
        Attributes
        ----------
        data: Dict
            Configuration data for model instance
        diagnostics:Diagnostics
            AAEM Diagnostics object for tracking messages from model
        intertie: str, or None
            status to track intertie placment, 'parent', 'child', or None
        intertie_data: CommunityData
            config data for the entire intertie if community is a 
        child community 
        
        
        """
        self.diagnostics = diag
        if diag == None:
            self.diagnostics = Diagnostics()
            
        if type(community_config) is dict and type(global_config) is dict:
            self.data = merge_configs(self.load_structure(), global_config)
            self.data = merge_configs(self.data, community_config)
        else: 
            self.data = self.load_config(community_config, global_config)
            
            
        valid, reason = self.validate_config()
        if not valid:
            raise StandardError, 'INVALID CONFIG FILE: ' + reason

        self.intertie = None
        
        intertie = self.get_item('community', 'intertie')
        if type (intertie) is list:
            if self.get_item('community', 'model as intertie') :
                self.intertie = 'parent'
            else:
                self.intertie = 'child'
         
        self.intertie_data = None
        
        ## load if community is part of an intertie but not the intertie 
        ## it's self. I.E. load info for Bethel or Oscarville,
        ## but not Bethel_intertie
        if not self.intertie is None and \
                not self.get_item('community', 'model as intertie'):
            self.diagnostics.add_note(
                'Community Data',
                'Attempting to find intertie data'
            )
            ## is the intertie_config a dict of file, also use same globals
            if type(intertie_config) is dict \
                    or (type(intertie_config) is str \
                    and os.path.isfile(intertie_config)):
                self.diagnostics.add_note(
                    'Community Data',
                    'using provided intertie_config argument'
                )
                self.intertie_data = CommunityData(
                    intertie_config, 
                    global_config,
                    self.diagnostics,
                    scalers
                )
            ## try to find the file
            elif os.path.isfile(community_config):
                rt_path = os.path.split(community_config)
                it_file = \
                    self.get_item('community','intertie')[0].\
                        replace(' ','_').replace("'",'')\
                     + '_intertie.yaml'
                it_file = os.path.join(rt_path[0], it_file)
                if os.path.isfile(it_file):
                    self.diagnostics.add_note(
                        'Community Data',
                        'Found interte data at ' + it_file
                    )
                    
                    self.intertie_data = CommunityData(
                        it_file,
                        global_config,
                        self.diagnostics,
                        scalers
                    )
            else:
                self.diagnostics.add_note(
                    'Community Data',
                    'Could not find intertie_data Leaving it as None'
                )
                ## self.intertie_data = None is initlized 
                
                        
        ## component spesifc plugins
        for comp in comp_order:
            config = import_module("aaem.components." + comp_lib[comp]).config
            try:
                plugins = config.plugins
            except AttributeError:
                continue
                
            for plugin in plugins:
                plugin(self, community_config, global_config, scalers)
         
        convert = self.data['community']['diesel prices']
        convert.index = [int(y) for y in convert.index]
        convert.index.name = 'year'
        convert = self.data['community']['electric prices']
        convert.index = [int(y) for y in convert.index]
        convert.index.name = 'year'
        # modify diesel prices and electric prices
        self.apply_scalers(scalers)

    
        self.check_auto_disable_conditions ()
class CommunityData (object):
    """This class manages the input data for the an AAEM model instance
    """
    
    def __init__ (self, community_config, global_config = None, diag = None, 
        scalers = {'diesel price':1.0, 'diesel price adder':0},
        intertie_config = None):
        """This class manages the input data for the an AAEM model instance
        
        Parameters
        ----------
        community_config: path to yaml file or dict
            the dictionary or dictionary that would be loaded from the yaml file 
        should match the format to be validated for the model to run, unless
        global_config is also provided, in which case all config values not 
        provided here have values in global_config
        global_config: path to yaml file or dict, optional
            Optional second config file of values that can be applied to many
        model instances
        diag: Diagnostics, optional
            AAEM Diagnostics object for tracking messages from model
        scalers: Dict, optional
            Scalers to change model behaviour. The keys 'diesel price' and
        'diesel price adder' are used in CommunityData, and will be multiplied 
        or added to the diesel prices respectively. This will carry over into
        the electirc non-fuel prices.
        
        Attributes
        ----------
        data: Dict
            Configuration data for model instance
        diagnostics:Diagnostics
            AAEM Diagnostics object for tracking messages from model
        intertie: str, or None
            status to track intertie placment, 'parent', 'child', or None
        intertie_data: CommunityData
            config data for the entire intertie if community is a 
        child community 
        
        
        """
        self.diagnostics = diag
        if diag == None:
            self.diagnostics = Diagnostics()
            
        if type(community_config) is dict and type(global_config) is dict:
            self.data = merge_configs(self.load_structure(), global_config)
            self.data = merge_configs(self.data, community_config)
        else: 
            self.data = self.load_config(community_config, global_config)
            
            
        valid, reason = self.validate_config()
        if not valid:
            raise StandardError, 'INVALID CONFIG FILE: ' + reason

        self.intertie = None
        
        intertie = self.get_item('community', 'intertie')
        if type (intertie) is list:
            if self.get_item('community', 'model as intertie') :
                self.intertie = 'parent'
            else:
                self.intertie = 'child'
         
        self.intertie_data = None
        
        ## load if community is part of an intertie but not the intertie 
        ## it's self. I.E. load info for Bethel or Oscarville,
        ## but not Bethel_intertie
        if not self.intertie is None and \
                not self.get_item('community', 'model as intertie'):
            self.diagnostics.add_note(
                'Community Data',
                'Attempting to find intertie data'
            )
            ## is the intertie_config a dict of file, also use same globals
            if type(intertie_config) is dict \
                    or (type(intertie_config) is str \
                    and os.path.isfile(intertie_config)):
                self.diagnostics.add_note(
                    'Community Data',
                    'using provided intertie_config argument'
                )
                self.intertie_data = CommunityData(
                    intertie_config, 
                    global_config,
                    self.diagnostics,
                    scalers
                )
            ## try to find the file
            elif os.path.isfile(community_config):
                rt_path = os.path.split(community_config)
                it_file = \
                    self.get_item('community','intertie')[0].\
                        replace(' ','_').replace("'",'')\
                     + '_intertie.yaml'
                it_file = os.path.join(rt_path[0], it_file)
                if os.path.isfile(it_file):
                    self.diagnostics.add_note(
                        'Community Data',
                        'Found interte data at ' + it_file
                    )
                    
                    self.intertie_data = CommunityData(
                        it_file,
                        global_config,
                        self.diagnostics,
                        scalers
                    )
            else:
                self.diagnostics.add_note(
                    'Community Data',
                    'Could not find intertie_data Leaving it as None'
                )
                ## self.intertie_data = None is initlized 
                
                        
        ## component spesifc plugins
        for comp in comp_order:
            config = import_module("aaem.components." + comp_lib[comp]).config
            try:
                plugins = config.plugins
            except AttributeError:
                continue
                
            for plugin in plugins:
                plugin(self, community_config, global_config, scalers)
         
        convert = self.data['community']['diesel prices']
        convert.index = [int(y) for y in convert.index]
        convert.index.name = 'year'
        convert = self.data['community']['electric prices']
        convert.index = [int(y) for y in convert.index]
        convert.index.name = 'year'
        # modify diesel prices and electric prices
        self.apply_scalers(scalers)

    
        self.check_auto_disable_conditions ()
        
        #~ self.load_construction_multipliers(construction_multipliers)
    def apply_scalers(self, scalers):
        """apply scalers to inupt variables
        
        Parameters
        ----------
        scalers: Dict
            Scalers to change model behaviour. The keys 'diesel price' and
        'diesel price adder' are used in CommunityData, and will be multiplied 
        or added to the diesel prices respectively. This will carry over into
        the electirc non-fuel prices. 
        """
        ## diesel 
        if scalers['diesel price'] != 1 or scalers['diesel price adder'] != 1:
            self.diagnostics.add_note(
                'Community Data', 
                'Adjusting disel and electric prices'
            )
        else:
            return
        
        self.data['community']['diesel prices'] = \
            self.data['community']['diesel prices'] * scalers['diesel price'] +\
            scalers['diesel price adder']
            
        
        ## electric
        percent_diesel = \
            float(self.data['community']['percent diesel generation']) / 100.0
        efficiency = \
            float(self.data['community']['diesel generation efficiency'])
        adder = percent_diesel * \
            self.data['community']['diesel prices'] / efficiency
        self.data['community']['electric prices'] = \
            float(self.data['community']['electric non-fuel price']) + adder
    
    def check_auto_disable_conditions  (self):
        """
        check for any auto disable conditions and disable those components
        """
        # no conditions at this time 
        pass
        
        #~ st = self.get_item('Water & Wastewater Efficiency',"data").ix["assumption type used"]
        #~ if st.values[0] == "UNKNOWN":
            #~ self.set_item('Water & Wastewater Efficiency',"enabled",  False)
            #~ self.diagnostics.add_error("Community Data", 
                    #~ ("(Checking Inputs) Water Wastewater system type unknown."
                     #~ " Fixing by disabling Wastewater component at runtime"))
        
    def modify_diesel_prices(self,
        scalers = {'diesel price':1.0, 'diesel price adder':0}):
            
        pass
    
    
    def modify_non_fuel_electricity_prices (self, N_slope_price = .15):
        """
        calculate the electricity price
        
        pre:
            community: diesel generation efficiency should be a numeric type
                       > 0
            community: elec non-fuel cost: is a floating point dollar value
            community: diesel prices: is a diesel projections object. 
        post:
            community: electric prices: is a data frame of dollar 
                       values indexed by year
        """
        # TODO: 1 is 100% need to change to a calculation
        pass
    
    
    def validate_config(self):
        """
        validate a config library
        
        pre:
            lib should be a dictionary object
        post:
            returns true if lib is a valid config object; otherwise false
        """
        return validate_dict(self.data, self.load_structure())
        
        
        
    def load_structure (self):
        """ 

        """
        try:
            return self.structure
        except AttributeError:
            pass
        
        structure = base_structure
        
        for comp in comp_lib:
            module = \
                import_module('aaem.components.' + comp_lib[comp] + '.config')
            structure = merge_configs(structure, module.structure)
            
        self.structure = structure
        return structure
        
    def load_config (self, community_file, global_file = None):
        """ 
        loads the input files and creates the model input object
        
        pre: 
            community_file, and defaults should be .yaml files
        post:
            self.data is is usable
        """
        community = read_config(community_file)
        
        global_ = {}
        if not global_file is None:
            global_ = read_config(global_file)
        
        return merge_configs(
            merge_configs(
                self.load_structure(),
                global_
            ),
            community
        )
        


    def get_item (self, section, key):
        """ 
        get an item 
        
        pre:
            self.data exists
            section is a config section, and key is a key in said section
        post:
            returns an item 
        """
        return self.data[section][key]
        
    def get_section (self, section):
        """
        gets a sction
        pre:
            self.data exists
            section is a config section
        post:
            returns a section library 
        """
        return self.data[section]
        
    def set_item (self, section, key, data):
        """
        set an item 
        
        pre:
            self.data exists
            section is a config section, and key is a key in said section, and 
        data is the type that make sense there.
        post:
            self.data[section][key] is data 
        """
        self.data[section][key] = data
        
    
 

        
    def save (self, fname):
        """ 
        save the inputs used
        
        pre:
            self.model_inputs exists, fname is the path to a file
        pre:
            a valid .yaml config file is created
        """
        ## save work around 
        import copy
        copy = copy.deepcopy(self.data)
        
        
        comment = "config used"

        #~ conf, orders, comments, defaults = \
            #~ config.get_config(config.non_component_config_sections)
        
        #~ for comp in comp_lib:

            #~ cfg = import_module("aaem.components." + comp_lib[comp]+ '.config')
            #~ order = list(cfg.yaml_order) + \
                    #~ list(set(cfg.yaml_order) ^ set(cfg.yaml.keys()))
            #~ orders[comp] = order
            #~ comments[comp] = cfg.yaml_comments
            
        #~ section_order = config.non_component_config_sections + comp_order

        
        s_order = ['community'] + comp_order
        i_order = {'community': base_order}
        comments = base_comments
        for comp in comp_lib:
            module = import_module('aaem.components.' + comp_lib[comp])
            i_order[comp] = module.config.order
            comments[comp] = module.config.comments
        
        save_config(fname,copy, 
            comments = comments,
            s_order = s_order,
            i_orders = i_order,
            header = 'confiuration used to generate these results'
        )
        del copy
Exemplo n.º 16
0
from flask import Flask
from flask import request
from google.auth.transport.urllib3 import AuthorizedHttp
from google.oauth2 import service_account

from diagnostics import Diagnostics
import settings

app = Flask(__name__)

credentials = service_account.Credentials.from_service_account_info(
    json.loads(settings.GOOGLE_SERVICE_ACCOUNT_CREDENTIALS_JSON),
    scopes=['https://www.googleapis.com/auth/bigquery.insertdata'])

diagnostics = Diagnostics(launched_at=datetime.datetime.utcnow())


@app.route('/log', methods=['POST'])
def log():
    diagnostics.request_count += 1
    auth = request.authorization
    if not auth or auth.username != settings.LOG_DRAIN_USERNAME or auth.password != settings.LOG_DRAIN_PASSWORD:
        diagnostics.unauthorized_count += 1
        return '', 403
    diagnostics.authorized_count += 1

    log_records = []
    for log_line in _parse_log_lines(request.data):
        if log_line.startswith(settings.LOG_RECORD_PREFIX):
            json_string = log_line.replace(settings.LOG_RECORD_PREFIX, '',
class Setup(object):
    """
    setup the structure needed to run the model
    
    class invariants:
        self.model_root: is the model root path <string>
        self.communities: list of communities to setup <string>
        self.data_repo: path data repo <string>
        self.tag: tag used a the directory to setup the model in 
            model_root <string>
    """
    def __init__(self, model_root, data_dir, communities=None, tag=None):
        """
        initilizer 
        
        inputs:
            model_root: model root path <string>
            communities: list of communities to setup <string>
            data_repo: path to data repo <sting>
            tag: (optional) tag to use as self.tag setup sub directory,
                if not provided self.tag will be m<version>_d<version> <string> 
            
        postconditions:
            see invatiants
        """
        self.model_root = model_root
        self.communities = communities
        self.data_dir = data_dir

        self.tag = tag
        if tag is None:
            self.tag = self.make_version_tag()
        self.diagnostics = Diagnostics()

    def make_version_tag(self):
        """
        generate a version tag
        
        precondition:
            see invariants
            'VERSION' file must exist in repo
            
        outputs
            returns tag
        """
        data_version_file = os.path.join(self.data_dir, 'VERSION')
        with open(data_version_file, 'r') as fd:
            ver = fd.read().replace("\n", "")
            ver = 'm' + __version__ + '_d' + ver
        return ver

    def setup_directories(self):
        """ 
        setup the directories
        
        preconditions:
            see invariats
            
        postconditions:
            config and input_files are removed and then
            config and input_files directories are created.
        """
        setup_path = os.path.join(self.model_root, self.tag)

        try:
            shutil.rmtree(os.path.join(setup_path, "config"))
        except OSError:
            pass

        os.makedirs(os.path.join(setup_path, "config"))

    def setup_community_list(self):
        """
        create the community list file from the repo
        
        preconditions:
            see invariants, community_list.csv sould exist in data repo
            
        postcondition:
            '__community_list.csv' saved in config directory
        """
        config_path = os.path.join(self.model_root, self.tag, 'config',
                                   '__community_list.csv')
        src_path = os.path.join(self.data_dir, 'community_list.csv')
        shutil.copy(src_path, config_path)

    def write_preprocessor_metadata(self, save_path):
        """ 
        write data metadata
        
        inputs:
            input_path: path to inputs directory <string>
            
        outputs:
            saves 'input_files_metadata.yaml' in "__metadata" subdirectory
        """
        data_version_file = os.path.join(self.data_dir, 'VERSION')
        with open(data_version_file, 'r') as fd:
            ver = fd.read().replace("\n", "")

        md_dir = os.path.join(save_path, "__metadata")
        try:
            os.makedirs(md_dir)
        except OSError:
            pass
        #~ try:
        #~ os.makedirs(os.path.join(md_dir,'diagnostic_files'))
        #~ except OSError:
        #~ pass

        m = 'w'
        with open(os.path.join(md_dir, 'preprocessor_metadata.yaml'),
                  m) as meta:
            meta.write(
                yaml.dump(
                    {
                        'upadted':
                        datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S"),
                        'data version':
                        ver
                    },
                    default_flow_style=False))

        self.diagnostics.save_messages(os.path.join(md_dir, 'log.csv'))

        data_version_file = os.path.join(self.data_dir, 'VERSION')
        with open(data_version_file, 'r') as fd:
            ver = fd.read().replace("\n", "")

        z = zipfile.ZipFile(os.path.join(md_dir, "raw_data.zip"), "w")
        for raw in [f for f in os.listdir(self.data_dir) if '.csv' in f]:
            z.write(os.path.join(self.data_dir, raw), raw)
        z.write(os.path.join(data_version_file), 'VERSION')

    def load_communities(self):
        """ Function doc """
        data = read_csv(
            os.path.join(self.model_root, self.tag, 'config',
                         '__community_list.csv'))

        self.communities = [c for c in data['Community'].values]

    def setup(self, force=False, ng_coms=[], make_globals=False):
        """
        run the setup functionality
        
        inputs:
            force: (optional) overwrirte existing files <boolean>
            
        outputs:
            model structure is setup
        """
        setup_path = os.path.join(self.model_root, self.tag)
        if os.path.exists(setup_path) and force == False:
            return False

        self.setup_directories()
        self.setup_community_list()

        if self.communities is None:
            self.load_communities()

        f_path = os.path.join(self.model_root, self.tag, 'config')
        for community in self.communities:
            #~ print community
            #~ f_path = os.path.join(self.model_root, self.tag, 'config')
            preprocessor = Preprocessor(community,
                                        self.data_dir,
                                        diag=self.diagnostics,
                                        process_intertie=False)
            self.diagnostics.add_note('Preprocessing ' + community,
                                      '---------')
            if community in ng_coms:
                preprocessor.run(show=True, ng_com=True)
            else:
                preprocessor.run(show=True)

            if make_globals:
                keys_to_split = KEYS_FOR_GLOBAL
                preprocessor.save_config(f_path, keys_to_split)
                f_name = os.path.join(f_path, '__global_config.yaml')
                if not os.path.exists(f_name):
                    preprocessor.save_global_congfig(f_name, keys_to_split)
            else:
                preprocessor.save_config(f_path)
            ## the intertie, if it exists
            try:
                preprocessor = Preprocessor(community,
                                            self.data_dir,
                                            diag=self.diagnostics,
                                            process_intertie=True)
                self.diagnostics.add_note('Preprocessing ' + community,
                                          '---------')
                preprocessor.run(show=True)
                if make_globals:
                    keys_to_split = KEYS_FOR_GLOBAL
                    preprocessor.save_config(f_path, keys_to_split)
                else:
                    preprocessor.save_config(f_path)
            except PreprocessorError:
                pass

        #~ self.setup_global_config()
        #~ ids = self.setup_input_files()
        #~ self.setup_community_configs()

        #~ self.setup_construction_multipliers()
        #~ self.setup_goals()
        self.write_preprocessor_metadata(f_path)
        return True
    def run(self,
            community_config,
            global_config=None,
            tag='',
            scalers=None,
            alt_save_name=None):
        """
        run the model for a community
        
        inputs:
            community: the community <string>
            c_config: (optional, default: none) alternate community config 
                file <string>
            g_config: (optional, default: none) alternat global confing 
                file <string>
            tag: (optional) tag for results dir <string>
            
        outputs:
            the model is run for a community/project/assigned 'name'
            
        preconditions:
            see invariants
            
        postconditions:
            None
        """
        #~ print community_config

        if scalers is None:
            scalers = default_scalers

        #~ if name is None:
        #~ name = community

        temp = tag
        #~ if img_dir is None:
        #~ if temp != '':
        #~ temp = '_' + tag
        #~ img_dir = os.path.join(self.model_root, 'results' + temp, 'plots')

        #~ cd, fc, diag = self.setup_community(community, i_dir, c_config,
        #~ g_config, c_mult, scalers)

        diagnostics = Diagnostics()
        community_data = CommunityData(community_config, global_config,
                                       diagnostics, scalers)
        name = community_data.get_item('community', 'file id')
        #~ print name
        forecast = Forecast(community_data, diagnostics, scalers)

        comps_used = self.run_components(community_data, forecast, diagnostics,
                                         scalers)

        #~ name = community_data.get_item('community', 'file id')
        self.save_components_output(comps_used,
                                    name,
                                    forecast,
                                    tag,
                                    alt_name=alt_save_name)
        #~ self.save_forecast_output(forecast, name, img_dir, plot, tag)
        self.save_input_files(community_data,
                              name,
                              tag,
                              alt_name=alt_save_name)
        self.save_diagnostics(diagnostics, name, tag, alt_name=alt_save_name)

        comps_used['community data'] = community_data
        comps_used['forecast'] = forecast

        #~ print name
        #~ print 'rb', alt_save_name
        self.store_results(comps_used, tag, name=alt_save_name)
    def __init__(self,
                 community_data,
                 diag=None,
                 scalers={'kWh consumption': 1.0}):
        """
        pre:
            self.cd is a community_data instance. 
        post:
            self.start_year < self.end_year are years(ints) 
            self.cd is a community_data instance. 
        """
        self.diagnostics = diag
        if self.diagnostics == None:
            self.diagnostics = Diagnostics()
        self.cd = community_data

        self.forecast_population()

        #~ ## test block
        #~ self.forecast_consumption(scalers['kWh consumption'])
        #~ self.forecast_generation()
        #~ return

        if self.cd.get_item("community", "model electricity") is False:
            pass
        else:
            try:
                self.forecast_consumption(scalers['kWh consumption'])
                self.forecast_generation()

                self.forecast_average_kW()
            except RuntimeError:
                self.cd.set_item("community", "model financial", False)
                self.cd.set_item("community", "model electricity", False)
                self.diagnostics.add_warning(
                    'Forecast',
                    ('Electricity data was insufficient to model'
                     ' electricity or financial values. In community data'
                     ' model financial, and model electricity have been'
                     ' set to false.'))

        self.calc_average_diesel_load()

        self.cpi = np.array(self.cd.get_item('community', 'cpi multipliers'))
        #~ DataFrame([
        #~ 1, 0.9765625, 0.953674316, 0.931322575, 0.909494702, 0.88817842,
        #~ 0.867361738, 0.847032947, 0.827180613, 0.807793567, 0.788860905,
        #~ 0.770371978, 0.752316385, 0.734683969, 0.717464814, 0.700649232,
        #~ 0.684227766, 0.668191178, 0.652530447, 0.637236765, 0.622301528,
        #~ 0.607716336, 0.593472984, 0.579563461, 0.565979942, 0.552714788,
        #~ 0.539760535, 0.527109897, 0.514755759, 0.502691171, 0.490909347,
        #~ 0.479403659, 0.468167636, 0.457194957, 0.44647945, 0.436015088,
        #~ 0.425795984, 0.415816391, 0.406070694, 0.396553412, 0.387259192,
        #~ 0.378182804, 0.369319145, 0.360663227, 0.352210183, 0.343955257,
        #~ 0.335893805, 0.328021294, 0.320333295, 0.312825484, 0.305493636,
        #~ 0.298333629, 0.291341435, 0.28451312, 0.277844844, 0.271332855,
        #~ 0.264973491, 0.258763175, 0.252698413, 0.246775794, 0.240991987],
        #~ )
        #~ last_cpi_year = self.cpi.index.tolist()[-1]
        #~ if self.end_year < last_cpi_year:
        #~ self.cpi = self.cpi[:self.end_year]
        #~ elif self.end_year > last_cpi_year:
        #~ self.diagnostics.add_note("Forecast",
        #~ "extending cpi past avaiable data")
        #~ last_cpi = float(self.cpi.ix[last_cpi_year])
        #~ for i in range(last_cpi_year,self.end_year+1):
        #~ self.cpi.ix[i] = last_cpi

        #~ self.forecast_households()
        self.heat_demand_cols = []
        self.heating_fuel_cols = []
        self.electric_columns = []
class Forecast(object):
    """ Class doc """
    def __init__(self,
                 community_data,
                 diag=None,
                 scalers={'kWh consumption': 1.0}):
        """
        pre:
            self.cd is a community_data instance. 
        post:
            self.start_year < self.end_year are years(ints) 
            self.cd is a community_data instance. 
        """
        self.diagnostics = diag
        if self.diagnostics == None:
            self.diagnostics = Diagnostics()
        self.cd = community_data

        self.forecast_population()

        #~ ## test block
        #~ self.forecast_consumption(scalers['kWh consumption'])
        #~ self.forecast_generation()
        #~ return

        if self.cd.get_item("community", "model electricity") is False:
            pass
        else:
            try:
                self.forecast_consumption(scalers['kWh consumption'])
                self.forecast_generation()

                self.forecast_average_kW()
            except RuntimeError:
                self.cd.set_item("community", "model financial", False)
                self.cd.set_item("community", "model electricity", False)
                self.diagnostics.add_warning(
                    'Forecast',
                    ('Electricity data was insufficient to model'
                     ' electricity or financial values. In community data'
                     ' model financial, and model electricity have been'
                     ' set to false.'))

        self.calc_average_diesel_load()

        self.cpi = np.array(self.cd.get_item('community', 'cpi multipliers'))
        #~ DataFrame([
        #~ 1, 0.9765625, 0.953674316, 0.931322575, 0.909494702, 0.88817842,
        #~ 0.867361738, 0.847032947, 0.827180613, 0.807793567, 0.788860905,
        #~ 0.770371978, 0.752316385, 0.734683969, 0.717464814, 0.700649232,
        #~ 0.684227766, 0.668191178, 0.652530447, 0.637236765, 0.622301528,
        #~ 0.607716336, 0.593472984, 0.579563461, 0.565979942, 0.552714788,
        #~ 0.539760535, 0.527109897, 0.514755759, 0.502691171, 0.490909347,
        #~ 0.479403659, 0.468167636, 0.457194957, 0.44647945, 0.436015088,
        #~ 0.425795984, 0.415816391, 0.406070694, 0.396553412, 0.387259192,
        #~ 0.378182804, 0.369319145, 0.360663227, 0.352210183, 0.343955257,
        #~ 0.335893805, 0.328021294, 0.320333295, 0.312825484, 0.305493636,
        #~ 0.298333629, 0.291341435, 0.28451312, 0.277844844, 0.271332855,
        #~ 0.264973491, 0.258763175, 0.252698413, 0.246775794, 0.240991987],
        #~ )
        #~ last_cpi_year = self.cpi.index.tolist()[-1]
        #~ if self.end_year < last_cpi_year:
        #~ self.cpi = self.cpi[:self.end_year]
        #~ elif self.end_year > last_cpi_year:
        #~ self.diagnostics.add_note("Forecast",
        #~ "extending cpi past avaiable data")
        #~ last_cpi = float(self.cpi.ix[last_cpi_year])
        #~ for i in range(last_cpi_year,self.end_year+1):
        #~ self.cpi.ix[i] = last_cpi

        #~ self.forecast_households()
        self.heat_demand_cols = []
        self.heating_fuel_cols = []
        self.electric_columns = []

    #~ def calc_electricity_values (self):
    #~ """
    #~ pre:
    #~ 'fc_electricity_used' should contain the kWh used for each key type
    #~ post:
    #~ self.electricity_totals is a array of yearly values of total kWh used
    #~ """
    #~ kWh = self.fc_specs["electricity"]
    #~ print kWh
    #~ years = kWh.T.keys().values
    #~ self.yearly_res_kWh = DataFrame({"year":years,
    #~ "total":kWh['consumption residential'].values}).set_index("year")
    #~ self.yearly_total_kWh = DataFrame({"year":years,
    #~ "total":kWh['consumption'].values}).set_index("year")
    #~ self.average_nr_kWh = kWh['consumption non-residential'].values[-3:].mean()
    #~ if np.isnan(self.average_nr_kWh):
    #~ temp = kWh['consumption non-residential']
    #~ self.average_nr_kWh = temp[ np.logical_not(np.isnan(temp))].mean()
    #~ self.yearly_nr_kWh = DataFrame({"year":years,
    #~ "total":kWh['consumption non-residential'].values}).set_index("year")
    #~ print self.average_nr_kWh

    def forecast_population(self):
        """
        pre:
            tbd.
        post:
            self.population is a array of estimated populations for each 
        year between start and end
        """
        # pop forecast is preprocessed now
        population = self.cd.get_item('community', "population")

        self.p_map = DataFrame(population["population_qualifier"])
        self.population = DataFrame(population["population"])

        #~ last_pop_year = self.population.index.tolist()[-1]
        #~ if self.end_year < last_pop_year:
        #~ self.population = self.population[:self.end_year]
        #~ elif self.end_year > last_pop_year:
        #~ self.diagnostics.add_note("Forecast",
        #~ "extending popultation past avaiable data")
        #~ last_pop = int(round(self.population.ix[last_pop_year]))
        #~ for i in range(last_pop_year,self.end_year+1):
        #~ self.population.ix[i] = last_pop

        if len(self.p_map[self.p_map == "M"]) < 10:
            msg = "the data range is < 10 for input population "\
                  "check population.csv in the models data directory"
            self.diagnostics.add_warning("forecast", msg)

    def forecast_consumption(self, consumption_scaler=1.0):
        """
        pre:
            tbd.
        post:
            self.consumption is a array of estimated kWh consumption for each 
        year between start and end
        """
        total_consumption = \
            self.cd.get_item('community','utility info')['consumption'] \
            * consumption_scaler
        residential_consumption = \
            self.cd.get_item('community','utility info')\
            ['consumption residential'] * consumption_scaler
        non_residential_consumption = \
            self.cd.get_item('community','utility info')\
            ['consumption non-residential'] * consumption_scaler

        #~ print self.cd.get_item('community','utility info')

        index = list(residential_consumption.index)

        if len(residential_consumption) < 10:
            msg = "the data range is < 10 for input consumption "\
                  "check electricity.csv in the models data directory"
            self.diagnostics.add_warning("Forecast: consumption", msg)

        population = self.population.ix[index]
        if any(population.isnull()):
            v = population.isnull().values.T.tolist()[0]
            for year in population[v].index.values.tolist():
                index.remove(year)

        if any(residential_consumption.isnull()):
            to_remove = \
                residential_consumption[residential_consumption.isnull()].index
            for year in to_remove:
                #~ print year
                index.remove(year)

        self.diagnostics.add_note("forecast",
           "years with measured consumption and population " + str(index) +\
        ". List used to generate fit function")
        population = self.population.ix[index]['population'].values

        #~ self.measured_consumption = self.yearly_total_kWh.ix[idx]
        consumption = residential_consumption.ix[index].values

        if len(population) < 10:
            self.diagnostics.add_warning("forecast",
                  "the data range is < 10 matching years for "\
                  "population and consumption "\
                  "check population.csv and electricity.csv "\
                  "in the models data directory")

        try:
            #~ print population,consumption
            m, b = np.polyfit(population, consumption, 1)
        except TypeError:
            raise RuntimeError, "Known population & consumption do not overlap"

        forcasted_consumption = (m * self.population + b
                                 )  #+ self.average_nr_kWh
        forcasted_consumption.columns = ['consumption residential']

        forcasted_consumption['consumption_qualifier'] = "P"

        forcasted_consumption['consumption residential']\
            [residential_consumption.ix[index].index] = \
            residential_consumption.ix[index]

        if consumption_scaler == 1.0:
            ## if the scaler is not 1 then none of the values are really
            ## measured
            forcasted_consumption['consumption_qualifier'].ix[index] = \
                (residential_consumption * np.nan).fillna("M")

        mean_non_res_con = non_residential_consumption.values[-3:].mean()
        if np.isnan(mean_non_res_con):
            self.diagnostics.add_note(
                'Forecast', ('Average of last 3 known years was null,'
                             ' averaging all non null years'))
            mean_non_res_con = \
                non_residential_consumption\
                [ ~non_residential_consumption.isnull()].mean()

        #~ print non_residential_consumption.values
        #~ print mean_non_res_con

        forcasted_consumption['consumption non-residential'] = mean_non_res_con
        #~ print forcasted_consumption
        forcasted_consumption['consumption non-residential']\
            [non_residential_consumption.ix[index].index] = \
            non_residential_consumption.ix[index]


        forcasted_consumption['consumption'] = \
            forcasted_consumption['consumption non-residential'] + \
            forcasted_consumption['consumption residential']

        #~ forcasted_consumption['consumption non-residential']\
        #~ [total_consumption.ix[index].index] = total_consumption.ix[index]

        ## don't forecast backwards
        forcasted_consumption = \
            forcasted_consumption.ix[residential_consumption.index[0]:]
        self.consumption = forcasted_consumption
        #~ print self.consumption

    def forecast_generation(self):
        """
        forecast generation
        pre:
            tbd.
            self.consumption should be a float array of kWh/yr values
        post:
            self.generation is a array of estimated kWh generation for each 
        year between start and end
        """
        measured_generation = \
            self.cd.get_item('community','utility info')['net generation']
        generation = self.consumption['consumption']/\
                    (1.0-self.cd.get_item('community','line losses')/100.0)
        generation = DataFrame(generation)
        generation.columns = ['generation']
        generation = generation.round(-3)

        index = set(measured_generation.index) & set(generation.index)

        generation['generation'][index] = measured_generation.ix[index]

        types = [
            u'generation diesel', u'generation hydro',
            u'generation natural gas', u'generation wind', u'generation solar',
            u'generation biomass'
        ]
        generation_types = self.cd.get_item('community', 'utility info')[types]

        backup_type = 'generation diesel'
        if (generation_types[['generation natural gas']].fillna(0)\
            != 0).any().bool():
            backup_type = 'generation natural gas'

        ## forecast each type
        for fuel in types:
            current_fuel = generation_types[fuel]
            #~ print current_rfuel
            name = fuel.replace('generation ', '')
            ## get the hypothetical limit
            try:
                current_generation = self.cd.get_item(
                    'community', name + ' generation limit')
            except KeyError:
                current_generation = 0
            ## is there a non-zero limit defined
            if current_generation != 0:

                if fuel == 'generation wind':
                    measured = \
                        current_fuel[current_fuel.notnull()].values[-3:].mean()
                    max_wind_percent = .2
                    if measured > (current_generation * max_wind_percent):
                        pass
                    else:
                        generation[
                            fuel] = current_generation * max_wind_percent
                        generation[fuel].ix[index] = current_fuel.ix[index]
                        continue
                else:
                    generation[fuel] = current_generation
                    generation[fuel].ix[index] = current_fuel.ix[index]
                    continue
            else:
                current_generation = \
                    current_fuel[current_fuel.notnull()].values[-3:].mean()

            ## no generation found
            if current_generation == 0:
                generation[fuel] = current_generation
                continue

            if fuel in [u'generation wind', u'generation hydro']:
                generation[fuel] = current_generation
                generation[fuel].ix[index] = current_fuel.ix[index]
                for i in generation[fuel].ix[current_fuel.index[-1] +
                                             1:].index:
                    generation[fuel].ix[i] = \
                       generation[fuel].ix[i-3:i-1].mean()
            else:
                generation[fuel] = current_generation
                generation[fuel].ix[index] = current_fuel.ix[index]

            #~ print fuel, current_generation
        #~ print generation['generation hydro']
        ## scale back any excess
        for fuel in [
                'generation wind', 'generation hydro', 'generation biomass'
        ]:
            current_fuel = generation_types[fuel]
            fuel_sums = generation[types].sum(1)
            #~ print fuel_sums
            if any(generation['generation'] < fuel_sums):
                #~ print 'a'
                if any(generation['generation'] < generation[fuel]):
                    #~ print 'b'
                    msg = "scaling generation(" + fuel + ") where geneation(" + \
                            fuel + ") > total generation"
                    self.diagnostics.add_note('forecast', msg)

                    generation[fuel] = generation[fuel] -\
                            (fuel_sums - generation['generation'])

                    generation[fuel][generation[fuel] < 0] = 0

                    generation[fuel].ix[index] = current_fuel.ix[index]

        corrected_backup_type = generation['generation'] - \
            (generation[types].sum(1) - generation[backup_type])
        #~ print corrected_backup_type
        corrected_backup_type[corrected_backup_type < 0] = 0

        generation[backup_type] = corrected_backup_type
        generation[backup_type].ix[index] = generation_types[backup_type].ix[
            index]

        self.generation = generation

    def forecast_average_kW(self):
        """
        forecast the average kW used per community per year
        pre:
        post:
        """
        self.average_kW = (self.consumption['consumption']/ 8760.0)\
                                /(1-self.cd.get_item('community','line losses'))

        self.average_kW.columns = ["average load"]

    def forecast_households(self):
        """
        forcast # of houselholds
        pre:
        post:
        """
        peeps_per_house = float(self.base_pop) / \
            self.cd.get_item('Residential Buildings',
                'data').ix['total_occupied']
        self.households = \
            np.round(self.population / np.float64(peeps_per_house))
        self.households.columns = ["households"]

    def get_population(self, start, end=None):
        """
        get population values from the population forecast. 
        
        pre:
            start is a year where start >= self.start_year
            end(if provided) is a year where start <= end < self.end_year
        post:
            returns a float or list of floats
        """
        if end is None:
            return self.population.ix[start].T.values[0]

        ## dynamic extension
        existing_len = len(self.population.ix[start:end])
        extend_by = (end + 1) - start - existing_len
        if extend_by > 0:
            extend = DataFrame(index=range(
                self.population.index[-1] + 1,
                self.population.index[-1] + 1 + extend_by),
                               columns=['population'])
            extend['population'] = self.population.iloc[-1]['population']
            population = self.population.ix[start:end].append(extend)

        else:
            # -1 to ensure same behavour
            population = self.population.ix[start:end]
        return population['population'].values

    def get_consumption(self, start, end=None):
        """
        get consumption values from the consumption forecast. 
        
        pre:
            start is a year where start >= self.start_year
            end(if provided) is a year where start <= end < self.end_year
        post:
            returns a float or list of floats
        """
        if end is None:
            return self.consumption.ix[start]['consumption']

        ## dynamic extension
        existing_len = len(self.consumption.ix[start:end])
        extend_by = (end + 1) - start - existing_len
        if extend_by > 0:
            extend = DataFrame(index=range(
                self.consumption.index[-1] + 1,
                self.consumption.index[-1] + extend_by + 1),
                               columns=['consumption'])
            extend['consumption'] = self.consumption.iloc[-1]['consumption']
            consumption = \
                DataFrame(self.consumption.ix[start:end]['consumption']).\
                append(extend)

        else:
            #  -1 to ensure same behavour
            consumption = \
                DataFrame(self.consumption['consumption'].ix[start:end])
        return consumption['consumption'].values

    def get_generation(self, start, end=None):
        """
        get population values from the population forecast. 
        
        pre:
            start is a year where start >= self.start_year
            end(if provided) is a year where start <= end < self.end_year
        post:
            returns a float or list of floats
        """
        if end is None:
            return self.generation.ix[start]['generation']

        ## dynamic extension
        existing_len = len(self.generation.ix[start:end])
        extend_by = (end + 1) - start - existing_len
        if extend_by > 0:
            extend = DataFrame(index=range(
                self.generation.index[-1] + 1,
                self.generation.index[-1] + extend_by + 1),
                               columns=['generation'])
            extend['generation'] = self.generation.iloc[-1]['generation']
            generation = \
                DataFrame(self.generation.ix[start:end]['generation']).\
                append(extend)

        else:
            generation = \
                DataFrame(self.generation['generation'].ix[start:end])
        return generation['generation'].values

    def get_households(self, start, end=None):
        """
        get households values from the households forecast. 
        
        pre:
            start is a year where start >= self.start_year
            end(if provided) is a year where start <= end < self.end_year
        post:
            returns a float or list of floats
        """
        if end is None:
            return self.households.ix[start].T.values[0]

        ## dynamic extension
        existing_len = len(self.households.ix[start:end])
        extend_by = (end + 1) - start - existing_len
        if extend_by > 0:
            extend = DataFrame(index=range(
                self.households.index[-1] + 1,
                self.households.index[-1] + extend_by + 1),
                               columns=['households'])
            extend['households'] = self.households.iloc[-1]['households']
            households = self.households.ix[start:end].append(extend)

        else:
            # -1 to ensure same behavour
            households = self.households.ix[start:end]
        return households['households'].values

    #~ def save_forecast (self, path, png_path = None, do_plots = False):
    #~ """
    #~ save the forecast to a csv
    #~ pre:
    #~ everything needs to be foretasted
    #~ path: path to each communities output dir
    #~ png_path: altenat location for pngs
    #~ post:
    #~ saves 3 files
    #~ """
    #~ tag = self.cd.get_item("community", "name").replace(" ", "_") + "_"
    #~ pathrt = os.path.join(path, tag)
    #~ if png_path is None:
    #~ if do_plots:
    #~ os.makedirs(os.path.join(path,"images"))
    #~ png_path = os.path.join(path, 'images',tag)
    #~ epng_path = png_path
    #~ hdpng_path = png_path
    #~ hfpng_path = png_path;
    #~ gpng_path = png_path
    #~ else:
    #~ epng_path = os.path.join(png_path,'electric_forecast',tag)
    #~ try:
    #~ if do_plots:
    #~ os.makedirs(os.path.join(png_path,'electric_forecast'))
    #~ except OSError:
    #~ pass
    #~ gpng_path = os.path.join(png_path,'generation_forecast',tag)
    #~ try:
    #~ if do_plots:
    #~ os.makedirs(os.path.join(png_path,'generation_forecast'))
    #~ except OSError:
    #~ pass
    #~ hdpng_path = os.path.join(png_path,'heat_demand_forecast',tag)
    #~ try:
    #~ if do_plots:
    #~ os.makedirs(os.path.join(png_path,'heat_demand_forecast'))
    #~ except OSError:
    #~ pass
    #~ hfpng_path = os.path.join(png_path,'heating_fuel_forecast',tag)
    #~ try:
    #~ if do_plots:
    #~ os.makedirs(os.path.join(png_path,'heating_fuel_forecast'))
    #~ except OSError:
    #~ pass
    #~ if self.cd.intertie != 'child' and \
    #~ self.cd.get_item("community","model electricity"):

    ##start = datetime.now()
    #~ self.save_electric(pathrt, epng_path, do_plots)
    #~ self.save_generation_forecast(pathrt, gpng_path, do_plots)
    ##print "saving electric:" + str(datetime.now() - start)
    #~ if self.cd.intertie != 'parent' and \
    #~ self.cd.get_item("community","model heating fuel"):

    ##start = datetime.now()
    #~ self.save_heat_demand(pathrt, hdpng_path, do_plots)
    ##print "saving heat demand:" + str(datetime.now() - start)

    ##start = datetime.now()
    #~ self.save_heating_fuel(pathrt, hfpng_path, do_plots)
    ##print "saving heating fuel:" + str( datetime.now() - start)

    #~ def add_heat_demand_column (self, key, year_col, data_col):
    #~ """
    #~ add a column to be saved with the heat demand forecast
    #~ pre:
    #~ Key is a string. year_col is a numpy array of values, date_col
    #~ is a corresponding array of years
    #~ post:
    #~ a dataframe is added to self.heat_demand_cols
    #~ """
    #~ self.heat_demand_cols.append(DataFrame({"year":year_col,
    #~ key:data_col}).set_index("year"))

    #~ def add_heating_fuel_column (self, key, year_col, data_col):
    #~ """
    #~ add a column to be saved with the heating fuel forecast
    #~ pre:
    #~ Key is a string. year_col is a numpy array of values, date_col
    #~ is a corresponding array of years
    #~ post:
    #~ a dataframe is added to self.heating_fuel_cols
    #~ """
    #~ self.heating_fuel_cols.append(DataFrame({"year":year_col,
    #~ key:data_col}).set_index("year"))

    #~ def generate_electric_output_dataframe (self):
    #~ """
    #~ """
    #~ from copy import deepcopy
    #~ kWh_con = deepcopy(self.consumption)
    #~ kWh_con.columns = ["total_electricity_consumed [kWh/year]",
    #~ 'residential_electricity_consumed [kWh/year]',
    #~ 'non-residential_electricity_consumed [kWh/year]']
    #~ c_map = copy.deepcopy(self.c_map)
    #~ c_map.columns = ["total_electricity_consumed_qualifier"]

    #~ kWh_gen = self.generation
    #~ kWh_gen.columns = ["total_electricity_generation [kWh/year]"]
    #~ g_map = copy.deepcopy(c_map)
    #~ g_map.columns = ["total_electricity_generation_qualifier"]

    #~ community = copy.deepcopy(self.c_map)
    #~ community.columns = ["community"]
    #~ community.ix[:] = self.cd.get_item("community","name")
    #~ try:
    #~ self.electric_dataframe = \
    #~ concat([community,self.population.round().astype(int),
    #~ self.p_map, kWh_con.round().astype(int), c_map,
    #~ kWh_gen.round().astype(int), g_map] ,axis=1)
    #~ except ValueError:
    #~ self.electric_dataframe = None

    #~ def save_electric (self, csv_path, png_path, do_plots):
    #~ """
    #~ save the electric forecast

    #~ pre:
    #~ self.population, consumption, and generation. should be dataframes
    #~ as calculated in this class. path should be the path to the output
    #~ directory.
    #~ post:
    #~ a file, electricity_forecast.csv, is save in the output directory.
    #~ """
    #~ self.generate_electric_output_dataframe()
    #~ self.save_electric_csv(csv_path)
    #~ if do_plots:
    #~ self.save_electric_png(png_path)

    #~ def save_electric_csv (self, path):
    #~ """
    #~ save the electric forecast

    #~ pre:
    #~ self.population, consumption, and generation. should be dataframes
    #~ as calculated in this class. path should be the path to the output
    #~ directory.
    #~ post:
    #~ a file, electricity_forecast.csv, is save in the output directory.
    #~ """

    #~ if self.electric_dataframe is None:
    #~ self.diagnostics.add_warning("Forecast",
    #~ ("when saving null values were found in "
    #~ "generation/consumption data. Electricity "
    #~ "forecast is suspect not saving summary(csv)"))
    #~ return
    #~ data = self.electric_dataframe
    #~ #not joining path adding file name
    #~ f_name = path + "electricity_forecast.csv"

    #~ fd = open(f_name ,"w")
    #~ fd.write("# Electricity Forecast for " + \
    #~ self.cd.get_item("community","name") + "\n")
    #~ fd.write("# Qualifier info: \n")
    #~ fd.write("#   M indicates a measured value \n")
    #~ fd.write("#   P indicates a projected value \n")
    #~ fd.write("#   I indicates a value carried over from the input data. May be projected or measured see input data metadata.\n")

    #~ fd.close()

    #~ data.index = data.index.values.astype(int)
    #~ data.to_csv(f_name, index_label="year", mode = 'a')

    #~ def save_electric_png (self, path):
    #~ """ Function doc """
    #~ if self.electric_dataframe is None:
    #~ self.diagnostics.add_warning("Forecast",
    #~ ("when saving null values were found in "
    #~ "generation/consumption data. Electricity "
    #~ "forecast is suspect not saving summary(png)"))
    #~ return

    #~ path = path+ "electricity_forecast.png"
    #~ start = self.c_map[self.c_map['consumption_qualifier'] == 'P'].index[0]
    #~ df2 = self.electric_dataframe[['population',
    #~ 'total_electricity_consumed [kWh/year]',
    #~ 'total_electricity_generation [kWh/year]',
    #~ 'residential_electricity_consumed [kWh/year]',
    #~ 'non-residential_electricity_consumed [kWh/year]']]
    #~ name = self.cd.get_item("community","name") + ' Electricity Forecast'

    #~ fig, ax = plot.setup_fig(name ,'years','population')
    #~ ax1 = plot.add_yaxis(fig,'kWh')

    #~ c_dict = {'population': [min(1,r*4) for r in colors.red],
    #~ 'total_electricity_consumed [kWh/year]':colors.orange,
    #~ 'total_electricity_generation [kWh/year]':colors.green,
    #~ 'residential_electricity_consumed [kWh/year]':colors.yellow,
    #~ 'non-residential_electricity_consumed [kWh/year]':colors.violet
    #~ }

    #~ plot.plot_dataframe(ax1,df2,ax,['population'],
    #~ {'population':'population',
    #~ 'total_electricity_consumed [kWh/year]':'consumption',
    #~ 'total_electricity_generation [kWh/year]':'generation',
    #~ 'residential_electricity_consumed [kWh/year]':'residential consumption',
    #~ 'non-residential_electricity_consumed [kWh/year]':'non residential consumption'},
    #~ c_dict)
    #~ fig.subplots_adjust(right=.85)
    #~ fig.subplots_adjust(left=.12)
    #~ plot.add_vertical_line(ax,start, 'forecasting starts' )
    #~ plot.add_horizontal_line(ax1,0)

    #~ plot.create_legend(fig,.20)
    #~ plot.save(fig,path)
    #~ plot.clear(fig)

    #~ def generate_generation_forecast_dataframe(self):
    #~ """
    #~ """
    #~ data = copy.deepcopy(self.generation)
    #~ g_map = copy.deepcopy(self.c_map)
    #~ g_map.columns = ["generation_qualifier"]

    #~ order = []
    #~ try:
    #~ for col in ['generation total', 'generation diesel',
    #~ 'generation hydro',
    #~ 'generation natural gas', 'generation wind',
    #~ 'generation solar', 'generation biomass']:
    #~ if col == 'generation total' and \
    #~ any(np.isnan(data[col].ix[self.start_year:])):
    #~ raise ValueError, "nans found"

    #~ kWh = col.replace(" ", "_") + " [kWh/year]"
    #~ mmbtu = col.replace(" ", "_") + " [mmbtu/year]"
    #~ data[kWh] = data[col].fillna(0).round().astype(int)
    #~ data[mmbtu] = (data[col] / constants.mmbtu_to_kWh)\
    #~ .fillna(0).round().astype(int)
    #~ del data[col]
    #~ order += [kWh, mmbtu]
    #~ data = concat([self.population.round().astype(int),
    #~ self.p_map,g_map,data],axis=1)
    #~ data["community"] = self.cd.get_item("community","name")

    #~ self.generation_forecast_dataframe = \
    #~ data[[data.columns[-1]] + data.columns[:-1].tolist()]
    #~ del data
    #~ except ValueError:
    #~ self.generation_forecast_dataframe = None

    #~ def save_generation_forecast (self, csv_path, png_path, do_plots):

    #~ self.generate_generation_forecast_dataframe()

    #~ if self.generation_forecast_dataframe is None:
    #~ self.diagnostics.add_warning("Forecast",
    #~ ("when saving null values were found in "
    #~ "generation data. generation generation "
    #~ "forecast is suspect not saving summary(csv and png)"))
    #~ else:
    #~ self.save_generation_forecast_csv(csv_path)
    #~ if do_plots:
    #~ self.save_generation_forecast_png(png_path)

    #~ def save_generation_forecast_csv (self, path):
    #~ """
    #~ """
    #~ out_file = path + "generation_forecast.csv"
    #~ fd = open(out_file, 'w')
    #~ fd.write("# Generation forecast\n")
    #~ fd.write(("# The column 'generation_qualifier' applies to all "
    #~ "of the following generation columns"))
    #~ fd.write("# Qualifier info: \n")
    #~ fd.write("#   M indicates a measured value \n")
    #~ fd.write("#   P indicates a projected value \n")
    #~ fd.write("#   I indicates a value carried over from the input data. May be projected or measured see input data metadata.\n")
    #~ fd.close()
    #~ self.generation_forecast_dataframe.to_csv(out_file,
    #~ index_label="year", mode = 'a')

    #~ def save_generation_forecast_png (self, path):
    #~ """ Function doc """

    #~ path = path + "generation_forecast.png"

    #~ png_list = ['population',
    #~ 'generation_total [kWh/year]',
    #~ 'generation_diesel [kWh/year]',
    #~ 'generation_hydro [kWh/year]',
    #~ 'generation_natural_gas [kWh/year]',
    #~ 'generation_wind [kWh/year]',
    #~ 'generation_solar [kWh/year]',
    #~ 'generation_biomass [kWh/year]']

    #~ png_dict = {'population':'population',
    #~ 'generation_total [kWh/year]':'total',
    #~ 'generation_diesel [kWh/year]':'diesel',
    #~ 'generation_hydro [kWh/year]':'hydro',
    #~ 'generation_natural_gas [kWh/year]':'natural gas',
    #~ 'generation_wind [kWh/year]':'wind',
    #~ 'generation_solar [kWh/year]':'solar',
    #~ 'generation_biomass [kWh/year]':'biomass'}

    #~ c_dict = {'population': [min(1,r*4) for r in colors.red],
    #~ 'generation_total [kWh/year]':colors.jet,
    #~ 'generation_diesel [kWh/year]':colors.red,
    #~ 'generation_hydro [kWh/year]':colors.blue,
    #~ 'generation_natural_gas [kWh/year]':colors.violet,
    #~ 'generation_wind [kWh/year]':[ min(1,r*2) for r in colors.blue],
    #~ 'generation_solar [kWh/year]':colors.orange,
    #~ 'generation_biomass [kWh/year]':colors.avacado}

    #~ temp = []
    #~ for i in png_list:
    #~ if  all(self.generation_forecast_dataframe[i].fillna(0) == 0):
    #~ continue
    #~ temp.append(i)
    #~ png_list = temp
    #~ if len(png_list) == 3:
    #~ png_list = list(set(png_list).\
    #~ difference(['generation_total [kWh/year]']))

    #~ df2 = self.generation_forecast_dataframe[png_list]
    #~ plot_name = self.cd.get_item("community","name") +' Generation Forecast'

    #~ fig, ax = plot.setup_fig(plot_name ,'years','population')
    #~ ax1 = plot.add_yaxis(fig,'Generation kWh')

    #~ plot.plot_dataframe(ax1,df2,ax,['population'],png_dict,c_dict)
    #~ fig.subplots_adjust(right=.85)
    #~ fig.subplots_adjust(left=.12)
    #~ start = self.c_map[self.c_map['consumption_qualifier'] == 'P'].index[0]
    #~ plot.add_vertical_line(ax,start, 'forecasting starts' )
    #~ plot.add_horizontal_line(ax1,0)
    #~ plot.create_legend(fig,.20)

    #~ plot.save(fig,path)
    #~ plot.clear(fig)

    #~ def generate_heat_demand_dataframe (self):
    #~ """
    #~ """
    #~ data = concat([self.population.round().astype(int), self.p_map] + \
    #~ self.heat_demand_cols,axis=1)

    #~ for key in data.keys():
    #~ try:
    #~ col = data[key]
    #~ idx = np.logical_not(col.apply(np.isnan))
    #~ lv = col[idx].tail(1).values[0]
    #~ yr = col[idx].tail(1).index[0]
    #~ col[col.index > yr] = lv
    #~ data[key] = col.round()
    #~ except (TypeError, IndexError, AttributeError):
    #~ pass

    #~ idx = data.keys()[['mmbtu' in s for s in data.keys()]]

    #~ data["heat_energy_demand_total [mmbtu/year]"] = data[idx].sum(1).round()
    #~ data["community"] = self.cd.get_item("community","name")

    #~ self.heat_demand_dataframe = data[[data.columns[-1]] + data.columns[:-1].tolist()]
    #~ del data

    #~ def save_heat_demand (self,csv_path, png_path, do_plots):
    #~ """ Function doc """
    #~ self.generate_heat_demand_dataframe()
    #~ self.save_heat_demand_csv(csv_path)
    #~ if do_plots:
    #~ self.save_heat_demand_png(png_path)

    #~ def save_heat_demand_csv (self, path):
    #~ """
    #~ save the heat demand forecast

    #~ pre:
    #~ self.population should be a dataframe. self.heat_demad_cols should
    #~ be populated using add_heat_demad_column. path should be the path to the
    #~ output directory.
    #~ post:
    #~ a file, heat_demand_forecast.csv, is save in the output directory.
    #~ """
    #~ f_name = path + "heat_demand_forecast.csv"
    #~ data =  self.heat_demand_dataframe
    #~ fd = open(f_name ,"w")
    #~ fd.write("# Heat Demand Forecast for " + \
    #~ self.cd.get_item("community","name") + "\n")
    #~ fd.write("# Qualifier info: \n")
    #~ fd.write("#   M indicates a measured value \n")
    #~ fd.write("#   P indicates a projected value \n")
    #~ fd.write("#   I indicates a value carried over from the input data. May be projected or measured see input data metadata.\n")
    #~ fd.close()

    #~ data.index = data.index.values.astype(int)
    #~ data.to_csv(f_name, index_label="year", mode = 'a')

    #~ def save_heat_demand_png(self,path):
    #~ """
    #~ """
    #~ path = path+ "heat_demand_forecast.png"
    ##start = self.c_map[self.c_map['consumption_qualifier'] == 'P'].index[0]

    #~ cols = ['population']
    #~ for col in self.heat_demand_dataframe.columns:
    #~ if "[mmbtu/year]" in col:
    #~ cols += [col]

    #~ df2 = self.heat_demand_dataframe[cols]
    #~ name = self.cd.get_item("community","name") + ' Heat Demand Forecast'

    #~ fig, ax = plot.setup_fig(name ,'years','population')
    #~ ax1 = plot.add_yaxis(fig,'Heat Demand MMBtu')

    #~ c_dict = {'population': [min(1,r*4) for r in colors.red],
    #~ 'heat_energy_demand_residential [mmbtu/year]':colors.yellow,
    #~ 'heat_energy_demand_water-wastewater [mmbtu/year]':colors.cobalt,
    #~ 'heat_energy_demand_non-residential [mmbtu/year]':colors.green,
    #~ 'heat_energy_demand_total [mmbtu/year]':colors.jet}

    #~ plot.plot_dataframe(ax1,df2,ax,['population'],
    #~ {'population':'population',
    #~ 'heat_energy_demand_residential [mmbtu/year]':'residential',
    #~ 'heat_energy_demand_water-wastewater [mmbtu/year]':'water & wastewater',
    #~ 'heat_energy_demand_non-residential [mmbtu/year]':'non-residential',
    #~ 'heat_energy_demand_total [mmbtu/year]':'total'},
    #~ c_dict)
    #~ fig.subplots_adjust(right=.85)
    #~ fig.subplots_adjust(left=.12)
    ##plot.add_vertical_line(ax,start, 'forecasting starts' )
    #~ plot.add_horizontal_line(ax1,0)

    #~ plot.create_legend(fig,.2)
    #~ plot.save(fig,path)
    #~ plot.clear(fig)

    #~ def generate_heating_fuel_dataframe(self):
    #~ """ """
    #~ data = concat([self.population.round().astype(int), self.p_map] + \
    #~ self.heating_fuel_cols, axis=1)

    #~ for key in data.keys():
    #~ try:
    #~ col = data[key]
    #~ idx = np.logical_not(col.apply(np.isnan))
    #~ lv = col[idx].tail(1).values[0]
    #~ yr = col[idx].tail(1).index[0]
    #~ col[col.index > yr] = lv
    #~ data[key] = col.round()
    #~ except (TypeError,IndexError, AttributeError):
    #~ pass

    #~ hf_gal_idx = data.keys()[['gallons' in s for s in data.keys()]]
    #~ hf_gal_idx =  hf_gal_idx[['heating_fuel' in s for s in hf_gal_idx]]

    #~ hf_btu_idx = data.keys()[['mmbtu' in s for s in data.keys()]]
    #~ hf_btu_idx =  hf_btu_idx[['heating_fuel' in s for s in hf_btu_idx]]

    #~ data["heating_fuel_total_consumed [gallons/year]"] = \
    #~ data[hf_gal_idx].sum(1).round()
    #~ data["heating_fuel_total_consumed [mmbtu/year]"]= \
    #~ data[hf_btu_idx].sum(1).round()
    #~ data["community"] = self.cd.get_item("community","name")

    #~ self.heating_fuel_dataframe = \
    #~ data[[data.columns[-1]] + data.columns[:-1].tolist()]
    #~ del data

    #~ def save_heating_fuel(self, csv_path, png_path, do_plots):
    #~ """ """
    ##start = datetime.now()
    #~ self.generate_heating_fuel_dataframe()
    ##print "saving heating fuel *1:" + str( datetime.now() - start)
    ##start = datetime.now()
    #~ self.save_heating_fuel_csv(csv_path)
    ##print "saving heating fuel *2:" + str( datetime.now() - start)
    ##start = datetime.now()
    #~ if do_plots:
    #~ self.save_heating_fuel_png(png_path)
    ##print "saving heating fuel *3:" + str( datetime.now() - start)

    #~ def save_heating_fuel_csv (self, path):
    #~ """
    #~ save the heating fuel

    #~ pre:
    #~ self.population should be a dataframe. self.heat_demad_cols should
    #~ be populated using add_heating_fuel_column. path should be the path to
    #~ the output directory.
    #~ post:
    #~ a file, heating_fuel_forecast.csv, is save in the output directory.
    #~ """
    #~ f_name = path + "heating_fuel_forecast.csv"
    #~ data = self.heating_fuel_dataframe

    #~ fd = open(f_name ,"w")
    #~ fd.write("# Heating Fuel Forecast for " + \
    #~ self.cd.get_item("community","name") + "\n")
    #~ fd.write("# Qualifier info: \n")
    #~ fd.write("#   M indicates a measured value \n")
    #~ fd.write("#   P indicates a projected value \n")
    #~ fd.write("#   I indicates a value carried over from the input data. May be projected or measured see input data metadata.\n")
    #~ fd.close()

    #~ data.index = data.index.values.astype(int)
    #~ data.to_csv(f_name, index_label="year", mode = 'a')

    #~ def save_heating_fuel_png (self, path):
    #~ """"""
    #~ path = path+ "heating_fuel_forecast.png"
    ##start = self.c_map[self.c_map['consumption_qualifier'] == 'P'].index[0]

    #~ cols = ['population']
    #~ for col in self.heating_fuel_dataframe.columns:
    #~ if "[mmbtu/year]" in col:
    #~ cols += [col]

    #~ df2 = self.heating_fuel_dataframe[cols]

    #~ name = self.cd.get_item("community","name") + ' Heating fuel Forecast'

    #~ fig, ax = plot.setup_fig(name ,'years','population')
    #~ ax1 = plot.add_yaxis(fig,'Heating fuel MMBtu')

    #~ c_dict = {'population': [min(1,r*4) for r in colors.red],
    #~ "heating_fuel_residential_consumed [mmbtu/year]":
    #~ colors.red,
    #~ "cords_wood_residential_consumed [mmbtu/year]":
    #~ colors.avacado,
    #~ "gas_residential_consumed [mmbtu/year]":
    #~ colors.violet,
    #~ "electric_residential_consumed [mmbtu/year]":
    #~ colors.goldenrod,
    #~ "propane_residential_consumed [mmbtu/year]":
    #~ colors.orange,
    #~ "heating_fuel_water-wastewater_consumed [mmbtu/year]":
    #~ colors.cobalt,
    #~ "heating_fuel_non-residential_consumed [mmbtu/year]":
    #~ colors.green,
    #~ "heating_fuel_total_consumed [mmbtu/year]":colors.jet,}

    #~ plot.plot_dataframe(ax1,df2,ax,['population'],
    #~ {"population":'population',
    #~ "heating_fuel_residential_consumed [mmbtu/year]":
    #~ 'heating oil - residential',
    #~ "cords_wood_residential_consumed [mmbtu/year]":
    #~ 'wood - residential',
    #~ "gas_residential_consumed [mmbtu/year]":
    #~ 'natural gas - residential',
    #~ "electric_residential_consumed [mmbtu/year]":
    #~ 'electric - residential',
    #~ "propane_residential_consumed [mmbtu/year]":
    #~ 'propane - residential',
    #~ "heating_fuel_water-wastewater_consumed [mmbtu/year]":
    #~ 'heating oil - water wastewater',
    #~ "heating_fuel_non-residential_consumed [mmbtu/year]":
    #~ 'heating fuel(all) - non - residential',
    #~ "heating_fuel_total_consumed [mmbtu/year]":'total'},
    #~ c_dict)
    #~ fig.subplots_adjust(right=.85)
    #~ fig.subplots_adjust(left=.12)
    ##plot.add_vertical_line(ax,start, 'forecasting starts' )
    #~ plot.add_horizontal_line(ax1,0)

    #~ plot.create_legend(fig,.30)
    #~ plot.save(fig,path)
    #~ plot.clear(fig)

    def calc_average_diesel_load(self):
        """
        """
        #~ self.average_diesel_load = 0
        try:
            generation = self.generation['generation diesel']
        except AttributeError:
            self.diagnostics.add_error('Forecast',
                                       ('no generation diesel to caclulate'
                                        ' yearly average load. setting to 0'))
            generation = 0.0

        self.yearly_average_diesel_load = generation / constants.hours_per_year
Exemplo n.º 21
0
from people import Customer

from logplot import LogPlot
from diagnostics import Diagnostics

C = Customer('Self', 1)

diagnostics = Diagnostics()
results = diagnostics.from_file(
    'C:\\Users\\Imraan\\PycharmProjects\\crossLang\\corellian_74G.log')
LogPlot().plot(results, './results')

A = Customer('Imraan', 30)

C.deposit(50)
Exemplo n.º 22
0
 def diagnostics(self):
     """
     >>> Product('DGKFL06JDHJP').diagnostics()
     """
     diags = Diagnostics(serialNumber=self.serialNumber)
     return diags.fetch()
Exemplo n.º 23
0
class MainScreen(App):

    ser_bytes = NumericProperty()
    ser = serial.Serial(baudrate=115200)
    data = ListProperty([0,0,0,0,0,0])
    info = ObjectProperty()
    errors = []
    sm = ScreenManager()
    main = Main(name='main')
    data_screen = Data_Screen(name='data')
    diagnostics = Diagnostics(name='diagnostics')
    sm.add_widget(main)
    sm.add_widget(data_screen)
    sm.add_widget(diagnostics)


    def build(self):
        Window.clearcolor = (0, 0, 0, 1)
        Window.fullscreen = 'auto'
        Window.bind(on_key_down=self.press)

        #create data dictionary
        self.info = extract('telemetry2.xlsx')
        self.main.info = self.info
        self.diagnostics.info = self.info
        self.diagnostics.tabs.info = self.info 

        #Clock.schedule_interval(self.readserial, 0.05)
        Clock.schedule_interval(self.noserial, 0.5)
        return self.sm
    
    #keyboard inputs
    def press(self, keyboard, keycode, text, modifiers, type):
        if keycode == 275: # ascii code for left
            if self.sm.current == 'main':
                self.sm.transition = SlideTransition(direction="left")
                self.sm.current = 'data'
            elif self.sm.current == 'diagnostics':
                self.sm.transition = SlideTransition(direction="left")
                self.sm.current = 'main'
        elif keycode == 276: # ascii code for right 
            if self.sm.current == 'main':
                self.sm.transition = SlideTransition(direction="right")
                self.sm.current = 'diagnostics'
            elif self.sm.current == 'data':
                self.sm.transition = SlideTransition(direction="right")
                self.sm.current = 'main'
        elif keycode == 27: # ascii code for ESC
            App.stop(self)
        elif keycode == 273: #up
            if self.sm.current == 'diagnostics':
                if self.diagnostics.tabs.opacity == 0: # reset
                    self.diagnostics.rv.pos_hint = {"x": 0.02, "y": 0.02}
                    self.diagnostics.rv.size_hint = (0.7, 0.2)
                    self.diagnostics.tabs.pos_hint = {"x": 0.02, "y": 0.25}
                    self.diagnostics.tabs.size_hint = (0.7, 0.65)
                    self.diagnostics.tabs.opacity = 1
                else:
                    self.diagnostics.rv.pos_hint= {"x": 0.02, "y": 0.02}
                    self.diagnostics.rv.size_hint = (0.7, 0.05)
                    self.diagnostics.tabs.pos_hint = {"x": 0.02, "y": 0.08}
                    self.diagnostics.tabs.size_hint = (0.7, 0.9)
        elif keycode == 274: #down
            if self.sm.current == 'diagnostics':
                if self.diagnostics.tabs.size_hint[1] == 0.9: # reset
                    self.diagnostics.rv.pos_hint = {"x": 0.02, "y": 0.02}
                    self.diagnostics.rv.size_hint = (0.7, 0.2)
                    self.diagnostics.tabs.pos_hint = {"x": 0.02, "y": 0.25}
                    self.diagnostics.tabs.size_hint = (0.7, 0.65)
                else:
                    self.diagnostics.rv.pos_hint = {"x": 0.02, "y": 0.02}
                    self.diagnostics.rv.size_hint = (0.7, 0.95)
                    self.diagnostics.tabs.pos_hint = {"x": 0.02, "y": 0.06}
                    self.diagnostics.tabs.size_hint = (0.7, 0)
                    self.diagnostics.tabs.opacity = 0
        return True

    
    def readserial(self, dt):
        try:
            global ser
            if ser.in_waiting:
                #print(ser.in_waiting)
                #length = ser.in_waiting
                #temp = ser.readline()
                self.data = ser.readline().split()
                ser.reset_input_buffer()
                #print(len(self.data))
                #print(self.data[23])
                self.main.data = self.data
                #print(len(self.main.data))
                #print(str(int(self.data[28])))
                #self.diagnostics.data = self.data
                #self.data_screen.data = self.data
        except Exception as e:
            print(e)
            self.main.pc_status.serial_status = False
            try:
                ser = serial.Serial(
                    baudrate='115200',
                    timeout=1,
                    port=str(serial.tools.list_ports.comports()[0]).split()[0]
                    #port="/dev/ttyUSB0"
                )
                self.main.pc_status.serial_status = True
            except Exception:
                pass

    def noserial(self, dt):
        self.data = [random.randint(0,255) for i in range(60)] #instead of readserial

        #passes all the values 
        self.main.data = self.data
        self.diagnostics.update(self.data)
        #self.data_screen.data = self.data




    def update_errors2(self):
        self.errors = []
        for i in range(len(self.data)):
            self.errors.append(0)
        if int(self.data[7]) > 80:
            self.errors[7] = 2


    def update_errors(self):
        #stringA
        if self.data[0] == 0:
            for i in range(1,250):
                if self.data[i] > 250 or self.data[i] < 5:
                    self.errors.append({'datapos':i, 'value': self.data[i], 'timestamp': datetime.now().strftime("%H:%M:%S")})
        #stringB
        elif self.data[0] == 1:
            pass
        #stringC
        elif self.data[0] == 2:
            pass
        #log and remove the errors  if they bunch up
        if len(self.errors) > 20:
            self.errors = []
Exemplo n.º 24
0
                        type=int,
                        default=1,
                        help='plot every i\'th frame')
    parser.add_argument('-ntmax',
                        metavar='i',
                        type=int,
                        default=0,
                        help='limit to i points in time')

    args = parser.parse_args()

    print
    print("Replay run with " + args.hdf5_file)
    print

    diagnostics = Diagnostics(args.hdf5_file)

    ntMax = args.ntmax
    nPlot = args.np

    if ntMax > 0 and ntMax < diagnostics.nt:
        nt = ntMax
    else:
        nt = diagnostics.nt

    plot = PlotEnergy(diagnostics, args.hdf5_file.replace(".hdf5", ""), nt,
                      nPlot)

    print
    print("Replay finished.")
    print
Exemplo n.º 25
0
    def birnn_dynamic(x, config, training, attention=False):
        # reshape outputs to [batch_size, max_time_steps, n_features]
        #max_time = tf.shape(x)[1]
        #rnn_inputs = tf.reshape(x, [-1, max_time, config.embedding_dim])
        #rnn_inputs = x
        rnn_inputs = tf.reshape(x, [-1, config.max_seq_len, config.n_features])

        sequence_lengths = Diagnostics.length(rnn_inputs)
        init = tf.contrib.layers.xavier_initializer()

        # Choose rnn cell type
        if config.rnn_cell == 'lstm':
            args = {
                'num_units': config.hidden_units,
                'forget_bias': 1.0,
                'state_is_tuple': True
            }
            base_cell = tf.nn.rnn_cell.LSTMCell
        elif config.rnn_cell == 'gru':
            args = {'num_units': config.hidden_units}
            base_cell = tf.nn.rnn_cell.GRUCell
        elif config.rnn_cell == 'layer_norm':
            args = {
                'num_units': config.hidden_units,
                'forget_bias': 1.0,
                'dropout_keep_prob': config.recurrent_keep_prob
            }
            base_cell = tf.contrib.rnn.LayerNormBasicLSTMCell

        cell = base_cell

        if config.output_keep_prob < 1:
            # rnn_inputs = tf.nn.dropout(rnn_inputs, self.keep_prob)
            fwd_cells = [
                tf.nn.rnn_cell.DropoutWrapper(
                    cell(**args),
                    output_keep_prob=config.output_keep_prob,
                    variational_recurrent=True,
                    dtype=tf.float32) for _ in range(config.rnn_layers)
            ]
            bwd_cells = [
                tf.nn.rnn_cell.DropoutWrapper(
                    cell(**args),
                    output_keep_prob=config.output_keep_prob,
                    variational_recurrent=True,
                    dtype=tf.float32) for _ in range(config.rnn_layers)
            ]
        else:
            fwd_cells = [cell(**args) for _ in range(config.rnn_layers)]
            bwd_cells = [cell(**args) for _ in range(config.rnn_layers)]

        fwd_cells = tf.contrib.rnn.MultiRNNCell(fwd_cells)
        bwd_cells = tf.contrib.rnn.MultiRNNCell(bwd_cells)

        outputs, output_states = tf.nn.bidirectional_dynamic_rnn(
            cell_fw=fwd_cells,
            cell_bw=bwd_cells,
            inputs=rnn_inputs,
            sequence_length=sequence_lengths,
            dtype=tf.float32,
            parallel_iterations=128)

        birnn_output = tf.concat(outputs, 2)

        if attention:  # invoke soft attention mechanism - attend to different particles
            summary_vector = attention_A(birnn_output,
                                         config.attention_dim,
                                         my_method=False)
        else:  # Select last relevant output
            summary_vector = Diagnostics.last_relevant(birnn_output,
                                                       sequence_lengths)

        print(summary_vector.get_shape().as_list())
        # Fully connected layer for classification
        with tf.variable_scope("fc"):
            logits_RNN = tf.layers.dense(summary_vector,
                                         units=config.n_classes,
                                         kernel_initializer=init)

        return logits_RNN