Ejemplo n.º 1
0
    def __init__(self, url=None, session_id=None):

        self.json_net = None

        self.warnings = []
        self.files = []
        self.template = None
        self.attributes = []
        self.attr_name_map = {}

        self.nodes = {}
        self.links = {}
        self.groups = {}

        self.node_id = PluginLib.temp_ids()
        self.link_id = PluginLib.temp_ids()
        self.group_id = PluginLib.temp_ids()

        self.connection = JsonConnection(url)
        if session_id is not None:
            write_output("Using existing session %s" % session_id)
            self.connection.session_id = session_id
        else:
            self.connection.login()

        #3 steps: start, read, save
        self.num_steps = 3
Ejemplo n.º 2
0
    def __init__(self, url=None, session_id=None):

        #Record the names of the files created by the plugin so we can
        #display them to the user.
        self.files    = []

        #A mapping from attr ID to attr object. Makes searching for attributes
        #easier 
        self.attr_id_map = {}

        self.connection = JsonConnection(url)
        write_output("Connecting...")
        if session_id is not None:
            write_output("Using existing session %s"% session_id)
            self.connection.session_id=session_id
        else:
            self.connection.login()

        #Dictionaries, keyed on their name, where all the nodes, links & groups will go
        self.hydra_nodes = {}
        self.hydra_links = {}
        self.hydra_groups = {}
        self.hydra_group_members = {}

        #Generators of NEGATIVE IDS for the nodes & links. 
        #Negative IDS are used as temporary client-side IDS until the network
        #is saved, at which point the resources will be given permanent, positive IDS.
        self.node_ids  = temp_ids()
        self.link_ids  = temp_ids()
        self.group_ids = temp_ids() # A group is an institution.
    
        #A mapping from the name of a type to the type itself.
        self.type_name_map = {}

        self.num_steps = 3
Ejemplo n.º 3
0
    def __init__(self, url=None, session_id=None):

        self.json_net = None

        self.warnings = []
        self.files    = []
        self.template = None
        self.attributes = []
        self.attr_name_map = {}

        self.nodes = {}
        self.links = {}
        self.groups = {}
        
        self.node_id  = PluginLib.temp_ids()
        self.link_id  = PluginLib.temp_ids()
        self.group_id  = PluginLib.temp_ids()

        self.connection = JsonConnection(url)
        if session_id is not None:
            write_output("Using existing session %s"% session_id)
            self.connection.session_id=session_id
        else:
            self.connection.login()

        #3 steps: start, read, save 
        self.num_steps = 3
Ejemplo n.º 4
0
    def __init__(self, url=None, username=None, password=None):
        self.conn = JsonConnection(url)
        if username is not None and password is not None:
            self.session_id = self.conn.login(username=username,
                                              password=password)
        else:
            self.session_id = self.conn.login()

        self.projects = dict()
Ejemplo n.º 5
0
class HydraNetworkTree(object):

    def __init__(self, url=None, username=None, password=None):
        self.conn = JsonConnection(url)
        if username is not None and password is not None:
            self.session_id = self.conn.login(username=username,
                                              password=password)
        else:
            self.session_id = self.conn.login()

        self.projects = dict()

    def get_tree(self):
        project_list = self.conn.call('get_projects', {})
        for project in project_list:
            networks = self.conn.call('get_networks',
                                      {'project_id': project['id']})
            project['networks'] = networks
            self.projects.update({project['id']: project})

    def print_tree(self, color=True):
        if color:
            pr_col = '\033[1m'
            ne_col = '\033[92m'
            sc_col = '\033[94m'
            endtag = '\033[0m'
        else:
            pr_col = ''
            ne_col = ''
            sc_col = ''
            endtag = ''
        for pid, project in self.projects.iteritems():
            print('%sP %3d %s%s' %
                  (pr_col, pid, project['name'], endtag))
            for network in project['networks']:
                print(u'%sN%s   \u2514\u2500%s%2d %s%s' %
                      (ne_col, endtag, ne_col, network['id'], network['name'],
                       endtag))
                for scenario in network['scenarios']:
                    print(u'%sS%s      \u2514\u2500%s%2d %s%s' %
                          (sc_col, endtag, sc_col, scenario['id'],
                           scenario['name'], endtag))
Ejemplo n.º 6
0
 def __init__(self,
              vars,
              objs,
              actual_time_steps,
              url=None,
              session_id=None):
     self.vars = vars
     self.objs = objs
     self.actual_time_steps = actual_time_steps
     self.network_id = None
     self.scenario_id = None
     self.network = None
     self.res_scenario = None
     self.connection = JsonConnection(url)
     self.attrs = dict()
     self.time_axis = dict()
     if session_id is not None:
         log.info("Using existing session %s", session_id)
         self.connection.session_id = session_id
     else:
         self.connection.login()
Ejemplo n.º 7
0
    def __init__(self, url=None, session_id=None):

        #Record the names of the files created by the plugin so we can
        #display them to the user.
        self.files = []

        #A mapping from attr ID to attr object. Makes searching for attributes
        #easier
        self.attr_id_map = {}

        self.connection = JsonConnection(url)
        write_output("Connecting...")
        if session_id is not None:
            write_output("Using existing session %s" % session_id)
            self.connection.session_id = session_id
        else:
            self.connection.login()

        self.network = None

        self.num_steps = 3
Ejemplo n.º 8
0
 def __init__(self, vars, objs, actual_time_steps, url=None, session_id=None):
     self.vars=vars
     self.objs=objs
     self.actual_time_steps=actual_time_steps
     self.network_id = None
     self.scenario_id = None
     self.network = None
     self.res_scenario=None
     self.connection = JsonConnection(url)
     self.attrs = dict()
     self.time_axis = dict()
     if session_id is not None:
         log.info("Using existing session %s", session_id)
         self.connection.session_id = session_id
     else:
         self.connection.login()
Ejemplo n.º 9
0
    def __init__(self, url=None, username=None, password=None):
        super(HydraNetwork, self).__init__()
        self.conn = JsonConnection(url=url, app_name='ShapefileApp')
        self.url = url
        self.username = username
        self.password = password
        self.session_id = None
        self.project = None
        self.hydra_network = None
        self.hydra_scenario = None
        self.hydra_attributes = None

        self.attrs = dict()
        self.attr_ids = dict()
        self.epsg = None
        self.nodes = dict()
        self.links = []

        self.link_names = dict()
        self.node_names = dict()

        self._node_coord_index = dict()
Ejemplo n.º 10
0
class Importer(object):
    def __init__(self,
                 vars,
                 objs,
                 actual_time_steps,
                 url=None,
                 session_id=None):
        self.vars = vars
        self.objs = objs
        self.actual_time_steps = actual_time_steps
        self.network_id = None
        self.scenario_id = None
        self.network = None
        self.res_scenario = None
        self.connection = JsonConnection(url)
        self.attrs = dict()
        self.time_axis = dict()
        if session_id is not None:
            log.info("Using existing session %s", session_id)
            self.connection.session_id = session_id
        else:
            self.connection.login()

    def load_network(self, network_id, scenario_id):
        """
         Load network and scenario from the server.
        """

        try:
            network_id = int(network_id)
        except (TypeError, ValueError):
            network_id = self.network_id
        if network_id is None:
            raise HydraPluginError("No network specified.")

        try:
            scenario_id = int(scenario_id)
        except (TypeError, ValueError):
            scenario_id = self.scenario_id
        if scenario_id is None:
            raise HydraPluginError("No scenario specified.")

        self.network = self.connection.call(
            'get_network', {
                'network_id': int(network_id),
                'include_data': 'Y',
                'scenario_ids': [int(scenario_id)],
                'template_id': None
            })
        self.res_scenario = self.network.scenarios[0].resourcescenarios
        attrslist = self.connection.call('get_all_attributes', {})
        for attr in attrslist:
            self.attrs.update({attr.id: attr.name})

    #####################################################
    def set_network(self, network):
        """
           Load network and scenario from the server.
        """
        self.network = network
        #self.res_scenario = self.network.scenarios[0].resourcescenarios
        attrslist = self.connection.call('get_attributes', {})
        for attr in attrslist:
            self.attrs.update({attr.id: attr.name})

    #####################################################

    def create_timeseries(self, data):
        timeseries = {'0': {}}
        counter = 0
        for time_s in self.actual_time_steps:
            timeseries['0'][time_s] = json.dumps(data[counter])
            counter += 1
        return json.dumps(timeseries)

    '''
    def create_timeseries(self, data):
        timeseries = {'0': {}}
        index=0
        for i, idx in enumerate(index):
            timeseries['0'][self.time_axis[int(idx)]] = float(data[i])
            index+=1
        return json.dumps(timeseries)
    '''

    def create_scalar(self, value):
        return json.dumps(value)

    def create_array(self, index, data):
        pass

    def create_descriptor(self, value):
        descriptor = json.dumps(value)
        return descriptor

    def save(self):
        self.network.scenarios[0].resourcescenarios = self.res_scenario
        self.connection.call('update_scenario',
                             {'scen': self.network.scenarios[0]})

    def import_res(self):
        self.import_vars()
        self.import_objs()

    def import_vars(self):
        '''
        import varaible to Hydra
        '''
        nodes = dict()
        for node in self.network.nodes:
            nodes.update({node.id: node.name})
        for var in self.vars.keys():
            for varModel in self.vars[var]:
                for link in self.network.links:
                    #print "Owner: ", type(varModel.owner)
                    fromnode = nodes[link.node_1_id]
                    tonode = nodes[link.node_2_id]
                    if len(varModel.owner) == 2 and fromnode == varModel.owner[
                            0] and tonode == varModel.owner[1]:
                        # "It is here, link Var: ",var,": ",varModel.owner, ": ", varModel.data_set
                        for attr in link.attributes:
                            if self.attrs[attr.attr_id] == varModel.name:
                                if attr.attr_is_var == 'Y':
                                    dataset = dict(name = 'Pyomo import - ' + link.name + ' ' \
                                            + varModel.name)
                                    dataset['unit'] = varModel.unit
                                    if len(varModel.data_set) > 1:
                                        dataset['type'] = 'timeseries'
                                        dataset[
                                            'value'] = self.create_timeseries(
                                                varModel.data_set)
                                    elif len(varModel.data_set) == 1:
                                        try:
                                            data = float(varModel.data_set[0])
                                            dataset['type'] = 'scalar'
                                            dataset['value'] = \
                                                self.create_scalar(data)
                                        except ValueError:
                                            dataset['type'] = 'descriptor'
                                            dataset[
                                                'value'] = self.create_descriptor(
                                                    varModel.data_set[0])
                                    #print "link attr is added"
                                    metadata = {}
                                    dataset['metadata'] = json.dumps(metadata)
                                    dataset[
                                        'dimension'] = attr.resourcescenario.value.dimension
                                    res_scen = dict(resource_attr_id=attr.id,
                                                    attr_id=attr.attr_id,
                                                    value=dataset)
                                    self.res_scenario.append(
                                        res_scen)  #dataset

                for node in self.network.nodes:
                    if node.name in varModel.owner and len(
                            varModel.owner) == 1:
                        # "node Var: ", var, ": ", varModel.owner,": ", varModel.data_set
                        for attr in node.attributes:
                            if self.attrs[attr.attr_id] == varModel.name:
                                if attr.attr_is_var == 'Y':
                                    dataset = dict(name = 'Pyomo import - ' + node.name + ' ' \
                                        + varModel.name)
                                    dataset['unit'] = varModel.unit
                                    if len(varModel.data_set) > 1:
                                        dataset['type'] = 'timeseries'
                                        dataset[
                                            'value'] = self.create_timeseries(
                                                varModel.data_set)
                                    elif len(varModel.data_set) == 1:
                                        try:
                                            data = float(varModel.data_set[0])
                                            dataset['type'] = 'scalar'
                                            dataset['value'] = \
                                            self.create_scalar(data)
                                        except ValueError:
                                            dataset['type'] = 'descriptor'
                                            dataset[
                                                'value'] = self.create_descriptor(
                                                    varModel.data_set[0])
                                    #print "node att is added"
                                    '''
                                    '''
                                    metadata = {}
                                    dataset['metadata'] = json.dumps(metadata)
                                    dataset[
                                        'dimension'] = attr.resourcescenario.value.dimension
                                    res_scen = dict(resource_attr_id=attr.id,
                                                    attr_id=attr.attr_id,
                                                    value=dataset)
                                    self.res_scenario.append(res_scen)
                                    # "Attr is added ...."

    def import_objs(self):
        '''
        import objects values to Hydra
        '''
        for var in self.objs.keys():
            for varModel in self.objs[var]:
                for attr in self.network.attributes:
                    if attr.attr_is_var == 'Y':
                        if self.attrs[attr.attr_id] == varModel.name:
                            dataset = dict(name = 'Pyomo import - ' + self.network.name + ' ' \
                                            + varModel.name)
                            dataset['unit'] = varModel.unit
                            if len(varModel.data_set) > 1:
                                dataset['type'] = 'timeseries'

                                dataset['value'] = self.create_timeseries(
                                    varModel.data_set)
                            elif len(varModel.data_set) == 1:
                                try:
                                    data = float(varModel.data_set[0])
                                    dataset['type'] = 'scalar'
                                    dataset['value'] = \
                                    self.create_scalar(data)
                                except ValueError:
                                    dataset['type'] = 'descriptor'
                                    dataset['value'] = self.create_descriptor(
                                        varModel.data_set[0])
                            '''
                            '''
                            res_scen = dict(resource_attr_id=attr.id,
                                            attr_id=attr.attr_id,
                                            value=dataset)
                            metadata = {}
                            dataset['metadata'] = json.dumps(metadata)
                            dataset[
                                'dimension'] = attr.resourcescenario.value.dimension
                            self.res_scenario.append(res_scen)
Ejemplo n.º 11
0
class ModelRunner(object):
    """
       Exporter of Hydra networks to JSON or XML files.
    """
    def __init__(self, url=None, session_id=None):

        #Record the names of the files created by the plugin so we can
        #display them to the user.
        self.files = []

        #A mapping from attr ID to attr object. Makes searching for attributes
        #easier
        self.attr_id_map = {}

        self.connection = JsonConnection(url)
        write_output("Connecting...")
        if session_id is not None:
            write_output("Using existing session %s" % session_id)
            self.connection.session_id = session_id
        else:
            self.connection.login()

        self.network = None

        self.num_steps = 3

    def get_attributes(self, template_id):
        if template_id is not None:
            #TODO: Find out why get_all_template_attributes isnt here.
            attributes = self.connection.call('get_all_attributes',
                                              {'template_id': template_id})
        else:
            attributes = self.connection.call('get_all_attributes', {})

        attr_id_map = {}
        for a in attributes:
            attr_id_map[a.id] = a
        self.attr_id_map = attr_id_map

    def get_network_data(self, network_id, scenario_id):
        """
            Retrieve the network, identify the parameters to set, 
            set them and run the model. Then identify the results
            and set them back on the network.
        """

        write_output("Retrieving Network")
        write_progress(2, self.num_steps)
        if network_id is not None:

            if scenario_id is None:
                raise HydraPluginError("A scenario ID must be specified.")

            #The network ID can be specified to get the network...
            try:
                network_id = int(network_id)
                network = self.connection.call(
                    'get_network', {
                        'network_id': network_id,
                        'scenario_ids': [int(scenario_id)]
                    })

                write_output("Network retrieved")
            except Exception, e:
                log.exception(e)
                raise HydraPluginError("Network %s not found." % network_id)

        else:
Ejemplo n.º 12
0
class HobbesImporter(object):
    """
       Importer of JSON files into Hydra. Also accepts XML files.
    """
    def __init__(self, url=None, session_id=None):

        self.json_net = None

        self.warnings = []
        self.files = []
        self.template = None
        self.attributes = []
        self.attr_name_map = {}

        self.nodes = {}
        self.links = {}
        self.groups = {}

        self.node_id = PluginLib.temp_ids()
        self.link_id = PluginLib.temp_ids()
        self.group_id = PluginLib.temp_ids()

        self.connection = JsonConnection(url)
        if session_id is not None:
            write_output("Using existing session %s" % session_id)
            self.connection.session_id = session_id
        else:
            self.connection.login()

        #3 steps: start, read, save
        self.num_steps = 3

    def fetch_project(self, project_id):
        """
            If a project ID is not specified, a new one
            must be created to hold the incoming network. 
            If an ID is specified, we must retrieve the project to make sure
            it exists. If it does not exist, then throw an error.

            Returns the project object so that the network can access it's ID.
        """
        if project_id is not None:
            try:
                project = self.connection.call('get_project',
                                               {'project_id': int(project_id)})
                log.info('Loading existing project (ID=%s)' % project_id)
                return project
            except RequestError:
                raise HydraPluginError("Project with ID %s not found" %
                                       project_id)

        #Using 'datetime.now()' in the name guarantees a unique project name.
        new_project = dict(
            name = "Hobbes Project created at %s" % (datetime.now()),
            description = \
            "Default project created by the %s plug-in." % \
                (self.__class__.__name__),
        )

        saved_project = self.connection.call('add_project',
                                             {'project': new_project})

        return saved_project

    def fetch_template(self, template_id):
        self.template = self.connection.call('get_template',
                                             {'template_id': int(template_id)})

        self.attributes = self.connection.call(
            'get_template_attributes', {'template_id': int(template_id)})

        #Build a lookup dict of attributes by name
        for a in self.attributes:
            self.attr_name_map[a.name] = a

    def fetch_remote_network(self):
        """
            Request the hobbes network from the hobbes server
        """
        write_output("Fetching Network")
        write_progress(2, self.num_steps)

        net_response = requests.get(
            "http://cwn.casil.ucdavis.edu/network/get")  #JSON Network
        #http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE    #XLS

        if net_response.status_code != 200:
            raise HydraPluginError(
                "A connection error has occurred with status code: %s" %
                net_response.status_code)

        self.json_net = json.loads(net_response.content)

    def upload_template(self):
        """
            Upload the template file found in ./template.xml
        """
        file_ = os.path.join(__location__, '../', 'template', 'template.xml')

        with open(file_) as f:
            xml_template = f.read()

        self.template = self.connection.call('upload_template_xml',
                                             {'template_xml': xml_template})

        self.attributes = self.connection.call(
            'get_template_attributes', {'template_id': self.template.id})

        #Build a lookup dict of attributes by name
        for a in self.attributes:
            self.attr_name_map[a.name] = a

    def import_network_topology(self, project_id=None):
        """
            Read the file containing the network data and send it to
            the server.
        """

        if self.json_net is None:
            self.fetch_remote_network()

        for node in self.json_net:
            props = node['properties']
            node_type = props['type']
            node_coords = node['geometry']['coordinates']

            tmp_node_id = self.node_id.next()

            #TODO: HACK. WHy are there 2 coordinates for the node?
            if isinstance(node_coords[0], list):
                #log.debug("Using 1st coords of %s (%s)", node_coords, props['type'])
                node_coords = node_coords[0]

            #log.info("X=%s, y=%s",node_coords[0], node_coords[1])
            node = dict(
                id=tmp_node_id,
                name=props['prmname'],
                x=str(
                    node_coords[0]),  #swap these if they are lat-long, not x-y
                y=str(node_coords[1]),
                description=props['description'],
                attributes=[],
                types=[{
                    'template_id': self.template.id,
                    'id': node_type
                }])
            self.nodes[props['prmname']] = node

            #ATTRIBUTES

            #Find the matching type
            for t in self.template.types:
                if t.name == node_type:
                    node['types'][0]['id'] = t.id
                    #Assign the attributes to the node
                    for tattr in t.typeattrs:
                        node['attributes'].append({'attr_id': tattr.attr_id})

            inlinks = [o['link_prmname'] for o in props.get('origins', [])]
            for linkname in inlinks:
                if linkname not in self.links:
                    link = dict(
                        id=self.link_id.next(),
                        name=linkname,
                        node_2_id=tmp_node_id,
                        attributes=[],
                        description="",
                    )
                    self.links[linkname] = link
                else:
                    link = self.links[linkname]
                    link['node_2_id'] = tmp_node_id

            outlinks = [o['link_prmname'] for o in props.get('terminals', [])]
            for linkname in outlinks:
                if linkname not in self.links:
                    link = dict(
                        id=self.link_id.next(),
                        name=linkname,
                        node_1_id=tmp_node_id,
                        attributes=[],
                        description="",
                    )
                    self.links[linkname] = link
                else:
                    link = self.links[linkname]
                    link['node_1_id'] = tmp_node_id

            node_groups = props['regions']

        project = self.fetch_project(project_id)
        project_id = project.id

        write_output("Saving Network")
        write_progress(3, self.num_steps)

        for t in self.template.types:
            if t.name == 'HobbesNetwork':
                network_type = t.id
        hydra_network = {
            'name': "HOBBES Network (%s)" % datetime.now(),
            'description':
            "Hobbes Network, imported directly from the web API",
            'nodes': self.nodes.values(),
            'links': self.links.values(),
            'project_id': project_id,
            'projection': 'EPSG:2229',
            'groups': [],
            'scenarios': [],
            'types': [{
                'template_id': self.template.id,
                'id': network_type
            }]
        }

        #The network ID can be specified to get the network...
        self.network = self.connection.call('add_network',
                                            {'net': hydra_network})
        return self.network

    def make_repo_dataset(self, json_repo):

        meta = {}
        for k, v in json_repo.items():
            meta[k] = str(v)

        dataset = {
            'dimension': 'dimensionless',
            'unit': None,
            'type': 'descriptor',
            'value': json_repo['tag'],
            'metadata': json.dumps(meta),
            'name': 'repo',
        }

        return dataset

    def import_data(self, include_timeseries=True):

        scenario = {
            "name": "Hobbes Import",
            "description": "Import from hobbes",
        }

        #List of parameters to ignore
        non_attributes = set([
            'origins', 'prmname', 'regions', 'terminals', 'description',
            'extras', 'type', 'repo', 'origin'
        ])

        #Make a map from node name to a list of attributes. Do this by first
        #making a node id map
        node_name_id_map = {}
        for n in self.network.nodes:
            node_name_id_map[n.name] = n.id

        node_attributes = self.connection.call('get_all_node_attributes',
                                               {'network_id': self.network.id})

        node_id_attr_map = {}
        for a in node_attributes:
            n_attrs = node_id_attr_map.get(a.ref_id, [])
            n_attrs.append(a)
            node_id_attr_map[a.ref_id] = n_attrs

        resource_scenarios = []
        #request data for first 2 nodes.
        for node in self.json_net[:10]:
            props = node['properties']
            name = props['prmname']
            node_id = node_name_id_map[name]

            #repo is a special case
            repo = self.make_repo_dataset(props['repo'])
            repo_attr_id = self.attr_name_map['repo'].id
            ra_id = None
            for a in node_id_attr_map[node_id]:
                if a.attr_id == repo_attr_id:
                    ra_id = a.id
                    break

            repo_rs = dict(
                resource_attr_id=ra_id,
                attr_id=repo_attr_id,
                is_var='N',
                value=repo,
            )
            resource_scenarios.append(repo_rs)

            for k, v in props.items():
                if k not in non_attributes:
                    if isinstance(v, float):
                        attr_id = self.attr_name_map[k].id
                        dataset = dict(
                            name=k,
                            value=str(v),
                            type='scalar',
                            dimension='dimensionless',
                            unit=None,
                        )

                        ra_id = None
                        for a in node_id_attr_map[node_id]:
                            if a.attr_id == attr_id:
                                ra_id = a.id
                                break

                        resource_scenario = dict(
                            resource_attr_id=ra_id,
                            attr_id=attr_id,
                            is_var='N',
                            value=dataset,
                        )

                        resource_scenarios.append(resource_scenario)

            #timeseries, requested from the hobbes server
            if include_timeseries is True:
                extras = props.get('extras', [])

                if extras is not None and len(extras) > 0:
                    attr_response = requests.get(
                        "http://cwn.casil.ucdavis.edu/network/extras?prmname=%s"
                        % props['prmname'])  #JSON attributes
                else:
                    continue

                if attr_response.status_code != 200:
                    raise HydraPluginError(
                        "A connection error has occurred with status code: %s"
                        % attr_response.status_code)

                extra_data = json.loads(attr_response.content)

                non_attrs = ['prmname', 'readme']

                for k, v in extra_data.items():
                    if k in non_attrs:
                        continue
                    else:
                        if len(v) < 2:
                            continue

                        ts = self.parse_timeseries(v)

                        attr_id = self.attr_name_map[k].id
                        dataset = dict(
                            name=k,
                            value=json.dumps(ts),
                            type='timeseries',
                            dimension='dimensionless',
                            unit=None,
                        )

                        ra_id = None
                        for a in node_id_attr_map[node_id]:
                            if a.attr_id == attr_id:
                                ra_id = a.id
                                break

                        resource_scenario = dict(
                            resource_attr_id=ra_id,
                            attr_id=attr_id,
                            is_var='N',
                            value=dataset,
                        )

                        resource_scenarios.append(resource_scenario)

        scenario['resourcescenarios'] = resource_scenarios

        new_scenario = self.connection.call('add_scenario', {
            'network_id': self.network.id,
            'scen': scenario
        })

        self.scenario = new_scenario
        return new_scenario

    def parse_timeseries(self, timeseries_value):
        """
            Convert a hobbes timeseries to a hydra timeseries
        """

        timeformat = config.get('DEFAULT', 'datetime_format')

        val = {}
        for timeval in timeseries_value[1:]:

            time = timeval[0]
            split = time.split('-')
            d = datetime(year=int(split[0]),
                         month=int(split[1]),
                         day=int(split[2]))

            tstime = datetime.strftime(d, timeformat)
            val[tstime] = float(timeval[1])

        return {"idx1": val}
Ejemplo n.º 13
0
class HobbesImporter(object):
    """
       Importer of JSON files into Hydra. Also accepts XML files.
    """
    def __init__(self, url=None, session_id=None):

        self.json_net = None

        self.warnings = []
        self.files    = []
        self.template = None
        self.attributes = []
        self.attr_name_map = {}

        self.nodes = {}
        self.links = {}
        self.groups = {}
        
        self.node_id  = PluginLib.temp_ids()
        self.link_id  = PluginLib.temp_ids()
        self.group_id  = PluginLib.temp_ids()

        self.connection = JsonConnection(url)
        if session_id is not None:
            write_output("Using existing session %s"% session_id)
            self.connection.session_id=session_id
        else:
            self.connection.login()

        #3 steps: start, read, save 
        self.num_steps = 3

    def fetch_project(self, project_id):
        """
            If a project ID is not specified, a new one
            must be created to hold the incoming network. 
            If an ID is specified, we must retrieve the project to make sure
            it exists. If it does not exist, then throw an error.

            Returns the project object so that the network can access it's ID.
        """
        if project_id is not None:
            try:
                project = self.connection.call('get_project', {'project_id':int(project_id)})
                log.info('Loading existing project (ID=%s)' % project_id)
                return project
            except RequestError:
                raise HydraPluginError("Project with ID %s not found"%project_id)

        #Using 'datetime.now()' in the name guarantees a unique project name.
        new_project = dict(
            name = "Hobbes Project created at %s" % (datetime.now()),
            description = \
            "Default project created by the %s plug-in." % \
                (self.__class__.__name__),
        )

        saved_project = self.connection.call('add_project', {'project':new_project})
        
        return saved_project 

    def fetch_template(self, template_id):
        self.template = self.connection.call('get_template',
                                             {'template_id':int(template_id)})
                                             
        self.attributes = self.connection.call('get_template_attributes',
                                               {'template_id':int(template_id)})

        #Build a lookup dict of attributes by name
        for a in self.attributes:
            self.attr_name_map[a.name] = a
            
    def fetch_remote_network(self):
        """
            Request the hobbes network from the hobbes server
        """
        write_output("Fetching Network") 
        write_progress(2, self.num_steps)

        net_response = requests.get("http://cwn.casil.ucdavis.edu/network/get") #JSON Network
        #http://cwn.casil.ucdavis.edu/excel/create?prmname=SR_CLE    #XLS

        if net_response.status_code != 200:
            raise HydraPluginError("A connection error has occurred with status code: %s"%net_response.status_code)

        self.json_net = json.loads(net_response.content)

    def upload_template(self):
        """
            Upload the template file found in ./template.xml
        """
        file_ = os.path.join(__location__, '../', 'template', 'template.xml')

        with open(file_) as f:
            xml_template = f.read()

        self.template = self.connection.call('upload_template_xml',
                                    {'template_xml':xml_template})

        self.attributes = self.connection.call('get_template_attributes',
                                               {'template_id':self.template.id})

        #Build a lookup dict of attributes by name
        for a in self.attributes:
            self.attr_name_map[a.name] = a
            
    def import_network_topology(self, project_id=None):
        """
            Read the file containing the network data and send it to
            the server.
        """

        if self.json_net is None:
            self.fetch_remote_network()

        for node in self.json_net:
            props = node['properties']
            node_type = props['type']
            node_coords = node['geometry']['coordinates']
            
            tmp_node_id = self.node_id.next()

            #TODO: HACK. WHy are there 2 coordinates for the node?
            if isinstance(node_coords[0], list):
                #log.debug("Using 1st coords of %s (%s)", node_coords, props['type'])
                node_coords = node_coords[0]

            #log.info("X=%s, y=%s",node_coords[0], node_coords[1]) 
            node = dict(
                id = tmp_node_id,
                name = props['prmname'],
                x = str(node_coords[0]), #swap these if they are lat-long, not x-y
                y = str(node_coords[1]),
                description = props['description'],
                attributes = [],
                types = [{'template_id':self.template.id,
                          'id':node_type}]
            )
            self.nodes[props['prmname']] = node

            #ATTRIBUTES

            #Find the matching type
            for t in self.template.types:
                if t.name == node_type:
                    node['types'][0]['id'] = t.id
                    #Assign the attributes to the node
                    for tattr in t.typeattrs:
                        node['attributes'].append(
                            {'attr_id':tattr.attr_id}
                        )

            inlinks = [o['link_prmname'] for o in props.get('origins', [])]
            for linkname in inlinks:
                if linkname not in self.links:
                    link = dict(
                        id=self.link_id.next(),
                        name = linkname,
                        node_2_id = tmp_node_id,
                        attributes = [],
                        description = "",
                    )
                    self.links[linkname] = link
                else:
                    link = self.links[linkname]
                    link['node_2_id'] = tmp_node_id


            outlinks = [o['link_prmname'] for o in props.get('terminals', [])]
            for linkname in outlinks:
                if linkname not in self.links:
                    link = dict(
                        id=self.link_id.next(),
                        name = linkname,
                        node_1_id = tmp_node_id,
                        attributes = [],
                        description = "",
                    )
                    self.links[linkname] = link
                else:
                    link = self.links[linkname]
                    link['node_1_id'] = tmp_node_id

            node_groups = props['regions']

        project = self.fetch_project(project_id)
        project_id = project.id

        write_output("Saving Network") 
        write_progress(3, self.num_steps) 

        for t in self.template.types:
            if t.name == 'HobbesNetwork':
                network_type = t.id
        hydra_network = {
            'name' : "HOBBES Network (%s)"%datetime.now(),
            'description' : "Hobbes Network, imported directly from the web API",
            'nodes': self.nodes.values(),
            'links': self.links.values(),
            'project_id' : project_id,
            'projection':'EPSG:2229',
            'groups'   : [],
            'scenarios': [],
            'types'    : [{'template_id':self.template.id,
                          'id':network_type}]
        }

        #The network ID can be specified to get the network...
        self.network = self.connection.call('add_network', {'net':hydra_network})
        return self.network
    
    def make_repo_dataset(self, json_repo):

        meta = {}
        for k, v in json_repo.items():
            meta[k] = str(v)

        dataset = {
            'dimension': 'dimensionless',
            'unit'     : None,
            'type'     : 'descriptor',
            'value'    : json_repo['tag'],
            'metadata' : json.dumps(meta),
            'name'     : 'repo',
        }

        return dataset

    def import_data(self, include_timeseries=True):

        scenario = {
            "name" : "Hobbes Import",
            "description" : "Import from hobbes",
        }

        #List of parameters to ignore
        non_attributes = set(['origins', 'prmname', 'regions', 'terminals', 'description', 'extras', 'type', 'repo', 'origin'])

        #Make a map from node name to a list of attributes. Do this by first
        #making a node id map
        node_name_id_map = {}
        for n in self.network.nodes:
            node_name_id_map[n.name] = n.id

        node_attributes = self.connection.call('get_all_node_attributes', {'network_id':self.network.id})

        node_id_attr_map = {}
        for a in node_attributes:
            n_attrs = node_id_attr_map.get(a.ref_id, [])
            n_attrs.append(a)
            node_id_attr_map[a.ref_id] = n_attrs

        resource_scenarios = []
        #request data for first 2 nodes.
        for node in self.json_net[:10]:
            props = node['properties']
            name  = props['prmname'] 
            node_id = node_name_id_map[name]

            #repo is a special case
            repo = self.make_repo_dataset(props['repo'])
            repo_attr_id = self.attr_name_map['repo'].id
            ra_id = None
            for a in node_id_attr_map[node_id]:
                if a.attr_id == repo_attr_id:
                    ra_id = a.id
                    break

            repo_rs = dict(
                resource_attr_id = ra_id,
                attr_id          = repo_attr_id,
                is_var           = 'N',
                value            = repo,
            )
            resource_scenarios.append(repo_rs)

            for k, v in props.items():
                if k not in non_attributes:
                    if isinstance(v, float):
                        attr_id = self.attr_name_map[k].id
                        dataset = dict(
                            name = k,
                            value = str(v),
                            type        = 'scalar',
                            dimension   = 'dimensionless',
                            unit        = None,
                        )

                        ra_id = None
                        for a in node_id_attr_map[node_id]:
                            if a.attr_id == attr_id:
                                ra_id = a.id
                                break

                        resource_scenario = dict(
                            resource_attr_id = ra_id,
                            attr_id          = attr_id,
                            is_var           = 'N',
                            value            = dataset,
                        )

                        resource_scenarios.append(resource_scenario)

            #timeseries, requested from the hobbes server
            if include_timeseries is True:
                extras = props.get('extras', [])
                
                if extras is not None and len(extras) > 0:
                    attr_response = requests.get("http://cwn.casil.ucdavis.edu/network/extras?prmname=%s"%props['prmname']) #JSON attributes
                else:
                    continue
                
                if attr_response.status_code != 200:
                    raise HydraPluginError("A connection error has occurred with status code: %s"%attr_response.status_code)

                extra_data = json.loads(attr_response.content)

                non_attrs = ['prmname', 'readme']

                for k, v in extra_data.items():
                    if k in non_attrs:
                        continue
                    else:
                        if len(v) < 2:
                            continue

                        ts = self.parse_timeseries(v)

                        attr_id = self.attr_name_map[k].id
                        dataset = dict(
                            name = k,
                            value = json.dumps(ts),
                            type        = 'timeseries',
                            dimension   = 'dimensionless',
                            unit        = None,
                        )

                        ra_id = None
                        for a in node_id_attr_map[node_id]:
                            if a.attr_id == attr_id:
                                ra_id = a.id
                                break

                        resource_scenario = dict(
                            resource_attr_id = ra_id,
                            attr_id          = attr_id,
                            is_var           = 'N',
                            value            = dataset,
                        )

                        resource_scenarios.append(resource_scenario)

                

        scenario['resourcescenarios'] = resource_scenarios
        
        new_scenario = self.connection.call('add_scenario', 
                                               {'network_id':self.network.id,
                                                'scen':scenario
                                               })

        self.scenario = new_scenario
        return new_scenario

    def parse_timeseries(self, timeseries_value):
        """
            Convert a hobbes timeseries to a hydra timeseries
        """

        timeformat = config.get('DEFAULT', 'datetime_format')

        val = {}
        for timeval in timeseries_value[1:]:
            
            time = timeval[0]
            split = time.split('-')
            d = datetime(year=int(split[0]), month=int(split[1]), day=int(split[2]))

            tstime = datetime.strftime(d, timeformat)
            val[tstime]  = float(timeval[1])

        return {"idx1": val}
Ejemplo n.º 14
0
class HydraNetwork(HydraResource):

    def __init__(self, url=None, username=None, password=None):
        super(HydraNetwork, self).__init__()
        self.conn = JsonConnection(url=url, app_name='ShapefileApp')
        self.url = url
        self.username = username
        self.password = password
        self.session_id = None
        self.project = None
        self.hydra_network = None
        self.hydra_scenario = None
        self.hydra_attributes = None

        self.attrs = dict()
        self.attr_ids = dict()
        self.epsg = None
        self.nodes = dict()
        self.links = []

        self.link_names = dict()
        self.node_names = dict()

        self._node_coord_index = dict()

    def login(self):
        if self.username is not None and self.password is not None:
            self.session_id = self.conn.login(username=self.username,
                                              password=self.password)
        else:
            self.session_id = self.conn.login()

    def load_attributes(self):
        self.hydra_attributes = self.conn.call('get_all_attributes', {})
        for attr in self.hydra_attributes:
            self.attrs[attr.id] = attr
            self.attr_ids[attr.name] = attr.id

    def load_network(self, network_id, scenario_id):
        """Load a network from HydraPlatform.
        """
        if self.project is None:
            self.load_project(network_id=network_id)

        self.hydra_network = self.conn.call('get_network',
                                            {'network_id': network_id,
                                             'include_data': 'Y'})

        self.load_attributes()

        for scenario in self.hydra_network['scenarios']:
            if scenario['id'] == scenario_id:
                self.hydra_scenario = scenario
                break

        res_scen_dict = dict()
        for res_scen in self.hydra_scenario['resourcescenarios']:
            res_scen_dict.update({res_scen['resource_attr_id']: res_scen})

        # Create node index
        node_dict = dict()
        for node in self.hydra_network['nodes']:
            node_dict.update({node['id']: node})

        self.name = self.hydra_network['name']
        self.description = self.hydra_network['description']

        if 'projection' in self.hydra_network.keys():
            try:
                self.epsg = \
                    int(self.hydra_network['projection'].split(':')[1])
            except ValueError:
                warnings.warn('Could not load EPSG code.')

        # Add network attributes
        for res_attr in self.hydra_network['attributes']:
            if res_scen_dict.get(res_attr['id']) is not None:
                res_scen = res_scen_dict[res_attr['id']]
                self.add_attribute(self.attrs[res_attr['attr_id']],
                                   res_attr, res_scen)
            else:
                self.add_attribute(self.attrs[res_attr['attr_id']],
                                   res_attr, None)

        # Add nodes and attributes
        for node in self.hydra_network['nodes']:
            n_node = HydraNode(x=float(node['x']), y=float(node['y']))
            n_node.name = node['name']
            n_node.layout = node['layout']
            n_node.id = node['id']
            n_node.types = node['types']
            for res_attr in node['attributes']:
                if res_scen_dict.get(res_attr['id']) is not None:
                    res_scen = res_scen_dict[res_attr['id']]
                    n_node.add_attribute(self.attrs[res_attr['attr_id']],
                                         res_attr, res_scen)
                else:
                    n_node.add_attribute(self.attrs[res_attr['attr_id']],
                                         res_attr, None)
            self.add_node(n_node)

        # Add segments and attributes
        for link in self.hydra_network['links']:
            n_link = HydraLink(start_node=self.nodes[link['node_1_id']],
                               end_node=self.nodes[link['node_2_id']])
            n_link.name = link['name']
            n_link.layout = link['layout']
            n_link.id = link['id']
            n_link.types = link['types']
            for res_attr in link['attributes']:
                if res_scen_dict.get(res_attr['id']) is not None:
                    res_scen = res_scen_dict[res_attr['id']]
                    n_link.add_attribute(self.attrs[res_attr['attr_id']],
                                         res_attr, res_scen)
                else:
                    n_link.add_attribute(self.attrs[res_attr['attr_id']],
                                         res_attr, None)

            self.add_link(n_link)

    def load_project(self, project_id=None, network_id=None):
        """Load a project by its ID or by a network ID.
        """
        if project_id is not None:
            self.project = self.conn.call('get_project',
                                          {'project_id': project_id})
        elif network_id is not None:
            self.project = self.conn.call('get_network_project',
                                          {'network_id': network_id})

    def create_project(self, name=None):
        self.project = dict()
        if name is not None:
            self.project['name'] = name
        else:
            self.project['name'] = "Shapefile import @ %s" % datetime.now()
        self.project['description'] = ''
        self.project['status'] = 'A'
        self.project = self.conn.call('add_project',
                                      {'project': self.project})

    def add_node(self, node):
        if self.node_names.get(node.name.lower()) is not None:
            add_string = ' (%s)' % self.node_names[node.name.lower()]
            self.node_names[node.name.lower()] += 1
            node.name += add_string
        else:
            self.node_names[node.name.lower()] = 1
        self.nodes[node.id] = node
        self._node_coord_index[(node.x, node.y)] = node.id

    def add_link(self, link):
        if self.link_names.get(link.name.lower()) is not None:
            add_string = ' (%s)' % self.link_names[link.name.lower()]
            self.link_names[link.name.lower()] += 1
            link.name += add_string
        else:
            self.link_names[link.name.lower()] = 1
        self.links.append(link)

    def save_network(self, network_name=None, project_name=None):
        """Save the network to HydraPlatform server.
        """
        if self.project is None:
            self.create_project(name=project_name)
        self.hydra_network = dict()
        self.hydra_scenario = dict()
        self.hydra_scenario['name'] = 'Scenario created by ShapefileApp'
        self.hydra_scenario['description'] = \
            'Standard scenario created by ShapefileApp.'
        self.hydra_scenario['id'] = -1
        self.hydra_scenario['resourcescenarios'] = []

        if network_name is not None:
            self.hydra_network['name'] = network_name
        else:
            self.hydra_network['name'] = \
                "Network imported from shapefile (%s)." % self.name
        self.hydra_network['description'] = \
            "Network imported from %s" % self.name
        if self.epsg is not None:
            self.hydra_network['projection'] = \
                'EPSG:%s' % self.epsg
        self.hydra_network['nodes'] = []
        self.hydra_network['links'] = []
        self.hydra_network['attributes'] = []
        self.hydra_network['scenarios'] = []
        self.hydra_network['project_id'] = self.project['id']

        for node in self.nodes.values():
            hydra_node = self.create_hydra_node(node)
            self.hydra_network['nodes'].append(hydra_node)

        for link in self.links:
            hydra_link = self.create_hydra_link(link)
            self.hydra_network['links'].append(hydra_link)

        self.hydra_network['scenarios'].append(self.hydra_scenario)
        net_summary = self.conn.call('add_network',
                                     {'net': self.hydra_network})

        return net_summary

    def create_hydra_node(self, node):
        """Build a node dict from a HydraNode object.
        """
        hydra_node = dict()
        hydra_node['id'] = node.id
        hydra_node['name'] = node.name
        hydra_node['description'] = ''
        hydra_node['attributes'] = []
        hydra_node['x'] = repr(node.x)
        hydra_node['y'] = repr(node.y)

        #TODO: Add attributes
        return hydra_node

    def create_hydra_link(self, link):
        """Build a link dict from a HydraLink object.
        """
        hydra_link = dict()
        #TODO: Finalise this function

        return hydra_link
Ejemplo n.º 15
0
# import
import pandas
from pandas.io.json import json_normalize
import sys
from os.path import expanduser
from os.path import join
import json

from HydraLib.PluginLib import JsonConnection

url = 'www.openaguadss.org'
conn = JsonConnection(url=url)
conn.login(username = '******', password = '')

def get_project_by_name(conn, project_name):
    return conn.call('get_project_by_name',
                     {'project_name':project_name})

def get_network_by_name(conn, project_id, network_name):
    return conn.call('get_network_by_name',
                     {'project_id':project_id, 'network_name':network_name})

# get project ID
project_name = 'Water Allocation Demo'
project = get_project_by_name(conn, project_name)
project_id = project.id

# get network ID
network_name = 'Water Allocation Network'
network = get_network_by_name(conn, project_id, network_name)
network_id = network.id
Ejemplo n.º 16
0
class NetworkImporter(object):
    """
       Exporter of Hydra networks to JSON or XML files.
    """

    def __init__(self, url=None, session_id=None):

        #Record the names of the files created by the plugin so we can
        #display them to the user.
        self.files    = []

        #A mapping from attr ID to attr object. Makes searching for attributes
        #easier 
        self.attr_id_map = {}

        self.connection = JsonConnection(url)
        write_output("Connecting...")
        if session_id is not None:
            write_output("Using existing session %s"% session_id)
            self.connection.session_id=session_id
        else:
            self.connection.login()

        #Dictionaries, keyed on their name, where all the nodes, links & groups will go
        self.hydra_nodes = {}
        self.hydra_links = {}
        self.hydra_groups = {}
        self.hydra_group_members = {}

        #Generators of NEGATIVE IDS for the nodes & links. 
        #Negative IDS are used as temporary client-side IDS until the network
        #is saved, at which point the resources will be given permanent, positive IDS.
        self.node_ids  = temp_ids()
        self.link_ids  = temp_ids()
        self.group_ids = temp_ids() # A group is an institution.
    
        #A mapping from the name of a type to the type itself.
        self.type_name_map = {}

        self.num_steps = 3

    def fetch_project(self, project_id):
        """
            If a project ID is not specified, a new one
            must be created to hold the incoming network. 
            If an ID is specified, we must retrieve the project to make sure
            it exists. If it does not exist, then throw an error.

            Returns the project object so that the network can access it's ID.
        """
        if project_id is not None:
            try:
                project = self.connection.call('get_project', {'project_id':project_id})
                log.info('Loading existing project (ID=%s)' % project_id)
                return project
            except RequestError, e:
                log.exception(e)
                raise HydraPluginError("An error occurred retrieving project ID %s"%project_id)

        #Using 'datetime.now()' in the name guarantees a unique project name.
        new_project = dict(
            name = "Hobbes Project created at %s" % (datetime.now()),
            description = \
            "Default project created by the %s plug-in." % \
                (self.__class__.__name__),
        )

        saved_project = self.connection.call('add_project', {'project':new_project})
        
        return saved_project 
Ejemplo n.º 17
0
class Importer (object):
    def __init__(self, vars, objs, actual_time_steps, url=None, session_id=None):
        self.vars=vars
        self.objs=objs
        self.actual_time_steps=actual_time_steps
        self.network_id = None
        self.scenario_id = None
        self.network = None
        self.res_scenario=None
        self.connection = JsonConnection(url)
        self.attrs = dict()
        self.time_axis = dict()
        if session_id is not None:
            log.info("Using existing session %s", session_id)
            self.connection.session_id = session_id
        else:
            self.connection.login()

    def load_network(self, network_id, scenario_id):
        """
         Load network and scenario from the server.
        """

        try:
            network_id = int(network_id)
        except (TypeError, ValueError):
            network_id = self.network_id
        if network_id is None:
            raise HydraPluginError("No network specified.")

        try:
            scenario_id = int(scenario_id)
        except (TypeError, ValueError):
            scenario_id = self.scenario_id
        if scenario_id is None:
            raise HydraPluginError("No scenario specified.")

        self.network = self.connection.call('get_network',
                                            {'network_id': int(network_id),
                                             'include_data': 'Y',
                                             'scenario_ids': [int(scenario_id)],
                                             'template_id': None})
        self.res_scenario = self.network.scenarios[0].resourcescenarios
        attrslist = self.connection.call('get_all_attributes', {})
        for attr in attrslist:
            self.attrs.update({attr.id: attr.name})

    #####################################################
    def set_network(self, network):
        """
           Load network and scenario from the server.
        """
        self.network =network
        #self.res_scenario = self.network.scenarios[0].resourcescenarios
        attrslist = self.connection.call('get_attributes', {})
        for attr in attrslist:
            self.attrs.update({attr.id: attr.name})
    #####################################################

    def create_timeseries(self, data):
        timeseries = {'0': {}}
        counter=0
        for time_s in self.actual_time_steps:
            timeseries['0'][time_s]=json.dumps(data[counter])
            counter+=1
        return json.dumps(timeseries)
    '''
    def create_timeseries(self, data):
        timeseries = {'0': {}}
        index=0
        for i, idx in enumerate(index):
            timeseries['0'][self.time_axis[int(idx)]] = float(data[i])
            index+=1
        return json.dumps(timeseries)
    '''
    def create_scalar(self, value):
        return json.dumps(value)

    def create_array(self, index, data):
        pass

    def create_descriptor(self, value):
        descriptor = json.dumps(value)
        return descriptor

    def save(self):
        self.network.scenarios[0].resourcescenarios = self.res_scenario
        self.connection.call('update_scenario', {'scen':self.network.scenarios[0]})

    def import_res(self):
        self.import_vars()
        self.import_objs()

    def import_vars(self):
        '''
        import varaible to Hydra
        '''
        nodes = dict()
        for node in self.network.nodes:
            nodes.update({node.id: node.name})
        for var in self.vars.keys():
            for varModel in self.vars[var]:
                for link in self.network.links:
                    #print "Owner: ", type(varModel.owner)
                    fromnode = nodes[link.node_1_id]
                    tonode = nodes[link.node_2_id]
                    if len(varModel.owner)==2 and fromnode == varModel.owner[0] and tonode == varModel.owner[1]:
                        # "It is here, link Var: ",var,": ",varModel.owner, ": ", varModel.data_set
                        for attr in link.attributes:
                            if self.attrs[attr.attr_id] == varModel.name:
                                if attr.attr_is_var == 'Y':
                                    dataset = dict(name = 'Pyomo import - ' + link.name + ' ' \
                                            + varModel.name)
                                    dataset['unit'] = varModel.unit
                                    if len(varModel.data_set)>1:
                                        dataset['type'] = 'timeseries'
                                        dataset['value'] = self.create_timeseries(varModel.data_set)
                                    elif len(varModel.data_set) == 1:
                                         try:
                                            data = float(varModel.data_set [0])
                                            dataset['type'] = 'scalar'
                                            dataset['value'] = \
                                                self.create_scalar(data)
                                         except ValueError:
                                            dataset['type'] = 'descriptor'
                                            dataset['value'] = self.create_descriptor(varModel.data_set [0])
                                    #print "link attr is added"
                                    metadata={}
                                    dataset['metadata']=json.dumps(metadata)
                                    dataset['dimension']=attr.resourcescenario.value.dimension
                                    res_scen = dict(resource_attr_id = attr.id,
                                                                    attr_id = attr.attr_id,
                                                                    value = dataset)
                                    self.res_scenario.append(res_scen) #dataset


                for node in self.network.nodes:
                    if node.name in varModel.owner and len(varModel.owner)==1:
                        # "node Var: ", var, ": ", varModel.owner,": ", varModel.data_set
                        for attr in node.attributes:
                            if self.attrs[attr.attr_id] == varModel.name:
                                if attr.attr_is_var == 'Y':
                                    dataset = dict(name = 'Pyomo import - ' + node.name + ' ' \
                                        + varModel.name)
                                    dataset['unit'] = varModel.unit
                                    if len(varModel.data_set)>1:
                                        dataset['type'] = 'timeseries'
                                        dataset['value'] = self.create_timeseries(varModel.data_set)
                                    elif len(varModel.data_set) == 1:
                                         try:
                                            data = float(varModel.data_set [0])
                                            dataset['type'] = 'scalar'
                                            dataset['value'] = \
                                            self.create_scalar(data)
                                         except ValueError:
                                            dataset['type'] = 'descriptor'
                                            dataset['value'] = self.create_descriptor(varModel.data_set [0])
                                    #print "node att is added"
                                    '''
                                    '''
                                    metadata={}
                                    dataset['metadata']=json.dumps(metadata)
                                    dataset['dimension']=attr.resourcescenario.value.dimension
                                    res_scen = dict(resource_attr_id = attr.id,
                                                    attr_id = attr.attr_id,
                                                    value = dataset)
                                    self.res_scenario.append(res_scen)
                                    # "Attr is added ...."

    def import_objs(self):
        '''
        import objects values to Hydra
        '''
        for var in self.objs.keys():
            for varModel in self.objs[var]:
                for attr in self.network.attributes:
                    if attr.attr_is_var == 'Y':
                        if self.attrs[attr.attr_id] == varModel.name:
                            dataset = dict(name = 'Pyomo import - ' + self.network.name + ' ' \
                                            + varModel.name)
                            dataset['unit'] = varModel.unit
                            if len(varModel.data_set)>1:
                                dataset['type'] = 'timeseries'

                                dataset['value'] = self.create_timeseries(varModel.data_set)
                            elif len(varModel.data_set) == 1:
                                 try:
                                    data = float(varModel.data_set [0])
                                    dataset['type'] = 'scalar'
                                    dataset['value'] = \
                                    self.create_scalar(data)
                                 except ValueError:
                                    dataset['type'] = 'descriptor'
                                    dataset['value'] = self.create_descriptor(varModel.data_set [0])
                            '''
                            '''
                            res_scen = dict(resource_attr_id = attr.id,
                                            attr_id = attr.attr_id,
                                            value = dataset)
                            metadata={}
                            dataset['metadata']=json.dumps(metadata)
                            dataset['dimension']=attr.resourcescenario.value.dimension
                            self.res_scenario.append(res_scen)