def setUp(self): self.create_user("UserA") self.create_user("UserB") self.create_user("UserC") self.project_id = self.create_project().id self.fmt = config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z") self.user_id = config.get('DEFAULT', 'root_user_id', 2)
def validate_plugin_xml(plugin_xml_file_path): log.info('Validating plugin xml file (%s).' % plugin_xml_file_path) try: with open(plugin_xml_file_path) as f: plugin_xml = f.read() except: raise HydraPluginError("Couldn't find plugin.xml.") try: plugin_xsd_path = os.path.expanduser( config.get('plugin', 'plugin_xsd_path')) log.info("Plugin Input xsd: %s", plugin_xsd_path) xmlschema_doc = etree.parse(plugin_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) xml_tree = etree.fromstring(plugin_xml) except XMLSyntaxError as e: raise HydraPluginError("There is an error in your XML syntax: %s" % e) except ParseError as e: raise HydraPluginError("There is an error in your XML: %s" % e) except Exception as e: log.exception(e) raise HydraPluginError( "An unknown error occurred with the plugin xsd: %s" % e.message) try: xmlschema.assertValid(xml_tree) except etree.DocumentInvalid as e: raise HydraPluginError('Plugin validation failed: ' + e.message) log.info("Plugin XML OK")
def validate(self): base_ts = pd.Timestamp("01-01-1970") #TODO: We need a more permanent solution to seasonal/repeating timeseries seasonal_year = config.get('DEFAULT','seasonal_year', '1678') seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999') jd = json.loads(self.value, object_pairs_hook=collections.OrderedDict) for k,v in jd.items(): for date in (six.text_type(d) for d in v.keys()): #A date with '9999' in it is a special case, but is an invalid year #for pandas, so replace it with the first year allowed by pandas -- 1678 if date.find(seasonal_key) >= 0: date = date.replace(seasonal_key, seasonal_year) ts = pd.Timestamp(date) assert isinstance(ts, base_ts.__class__) # Same type as known valid ts
def __init__(self): """Initialise job queue and similar... """ self.install_path = config.get('plugin', 'default_directory') if self.install_path is None: log.critical("Plugin folder not defined in config.ini! " "Cannot scan for installed plugins.") return None self.installed_apps = scan_installed_apps(self.install_path)
def create_sqlite_backup_db(audit_tables): """ return an inspector object """ #we always want to create a whole new DB, so delete the old one first #if it exists. try: Popen("rm %s"%(config.get('sqlite', 'backup_url')), shell=True) logging.warn("Old sqlite backup DB removed") except Exception as e: logging.warn(e) try: aux_dir = config.get('DEFAULT', 'hydra_aux_dir') os.mkdir(aux_dir) logging.warn("%s created", aux_dir) except Exception as e: logging.warn(e) try: backup_dir = config.get('db', 'export_target') os.mkdir(backup_dir) logging.warn("%s created", backup_dir) except Exception as e: logging.warn(e) db = create_engine(sqlite_engine, echo=True) db.connect() metadata = MetaData(db) for main_audit_table in audit_tables: cols = [] for c in main_audit_table.columns: col = c.copy() if col.type.python_type == Decimal: col.type = DECIMAL() cols.append(col) Table(main_audit_table.name, metadata, *cols, sqlite_autoincrement=True) metadata.create_all(db)
def setUp(self): util.connect() create_default_users_and_perms() self.create_user("UserA") self.create_user("UserB") self.create_user("UserC") self.project_id = self.create_project().id self.fmt = config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z") yield # perform the test # now tear down self.tearDown()
def test_create_template_from_network(self, client, network_with_data): network = network_with_data net_template = client.get_network_as_xml_template(network.id) assert net_template is not None template_xsd_path = config.get('templates', 'template_xsd_path') xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) xml_tree = etree.fromstring(net_template) xmlschema.assertValid(xml_tree)
def test_get_xml(self, client, template_json_object): xml_tmpl = template_json_object db_template = client.get_template_as_xml(xml_tmpl.id) assert db_template is not None template_xsd_path = config.get('templates', 'template_xsd_path') xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) xml_tree = etree.fromstring(db_template) xmlschema.assertValid(xml_tree)
def test_get_xml(self): xml_tmpl = self.test_add_xml() db_template = get_template_as_xml(xml_tmpl.id) assert db_template is not None template_xsd_path = config.get('templates', 'template_xsd_path') xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) xml_tree = etree.fromstring(db_template) xmlschema.assertValid(xml_tree)
def xsd_validate(template_file): """ Validate a template against the xsd. Return the xml tree if successful. """ with open(template_file) as f: xml_template = f.read() template_xsd_path = os.path.expanduser(config.get('templates', 'template_xsd_path')) log.info("Template xsd: %s", template_xsd_path) xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) xml_tree = etree.fromstring(xml_template) try: xmlschema.assertValid(xml_tree) except etree.DocumentInvalid as e: raise HydraPluginError('Template validation failed: ' + e.message) log.info("Template XSD validation successful.") return xml_tree
Column,\ Integer,\ String,\ TIMESTAMP,\ text,\ DDL from sqlalchemy.engine import reflection import logging from hydra_base import config from subprocess import Popen from sqlalchemy.types import DECIMAL, NUMERIC from sqlalchemy.dialects.mysql.base import DOUBLE from decimal import Decimal import os engine_name = config.get('mysqld', 'url') sqlite_engine = "sqlite:///%s"%(config.get('sqlite', 'backup_url')) def connect(): """ return an inspector object """ db = create_engine(engine_name, echo=True) db.connect() return db def create_sqlite_backup_db(audit_tables): """ return an inspector object """
from hydra_base.db import DeclarativeBase as _db from hydra_base.util.hdb import create_default_users_and_perms, make_root_user from hydra_base.lib.objects import JSONObject import hydra_base from hydra_base import config import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker import datetime import os from hydra_pywr.template import generate_pywr_attributes, generate_pywr_template, PYWR_DEFAULT_DATASETS global user_id user_id = config.get('DEFAULT', 'root_user_id', 1) @pytest.fixture def db_backend(request): return 'sqlite' @pytest.fixture() def testdb_uri(db_backend, tmpdir): if db_backend == 'sqlite': # Use a :memory: database for the tests. return 'sqlite:///{}'.format(os.path.join(tmpdir, 'test.db')) elif db_backend == 'postgres': # This is designed to work on Travis CI return 'postgresql://postgres@localhost:5432/hydra_base_test' elif db_backend == 'mysql':
from hydra_base.db import DeclarativeBase as _db from hydra_base.util.hdb import create_default_users_and_perms, make_root_user from hydra_base.lib.objects import JSONObject import hydra_base from hydra_base import config import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker import datetime import os from hydra_pywr.template import generate_pywr_attributes, generate_pywr_template, PYWR_DEFAULT_DATASETS global user_id user_id = config.get('DEFAULT', 'root_user_id', 1) @pytest.fixture def db_backend(request): return 'sqlite' @pytest.fixture() def testdb_uri(db_backend, tmpdir): if db_backend == 'sqlite': # Use a :memory: database for the tests. return 'sqlite:///{}'.format(os.path.join(tmpdir, 'test.db')) elif db_backend == 'postgres': # This is designed to work on Travis CI return 'postgresql://postgres@localhost:5432/hydra_base_test' elif db_backend == 'mysql': return 'mysql+mysqlconnector://root@localhost/hydra_base_test'
def __init__(self): self.app_registry = AppRegistry() self.job_queue = JobQueue(config.get('plugin', 'queue_directory', '/tmp')) self.job_queue.rebuild(self.app_registry) self.upload_dir = config.get('plugin', 'upload_dir', '/tmp/uploads')
import click import os from hydra_base import config from hydra_client.connection import JSONConnection from hydra_client.click import hydra_app, make_plugins, write_plugins import json from collections import defaultdict import pandas import re from .gis import import_nodes_from_shapefile, import_links_from_shapefile from . import data UPLOAD_DIR = config.get('plugin', 'upload_dir', '/tmp/uploads') UPLOAD_DIR = config.get('plugin', 'output_dir', '/tmp/uploads') def get_client(hostname, **kwargs): return JSONConnection(app_name='Pywr GIS App', db_url=hostname, **kwargs) def get_logged_in_client(context, user_id=None): session = context['session'] client = get_client(context['hostname'], session_id=session, user_id=user_id) if client.user_id is None: client.login(username=context['username'], password=context['password']) return client
def import_template_xml(template_xml, allow_update=True, **kwargs): """ Add the template, type and typeattrs described in an XML file. Delete type, typeattr entries in the DB that are not in the XML file The assumption is that they have been deleted and are no longer required. """ user_id = kwargs.get('user_id') template_xsd_path = config.get('templates', 'template_xsd_path') xmlschema_doc = etree.parse(template_xsd_path) xmlschema = etree.XMLSchema(xmlschema_doc) xml_tree = etree.fromstring(template_xml) xmlschema.assertValid(xml_tree) template_name = xml_tree.find('template_name').text template_description = xml_tree.find('template_description') if template_description is not None: template_description = template_description.text template_layout = None if xml_tree.find('layout') is not None and \ xml_tree.find('layout').text is not None: layout = xml_tree.find('layout') layout_string = get_etree_layout_as_dict(layout) template_layout = json.dumps(layout_string) try: tmpl_i = db.DBSession.query(Template).filter(Template.name == template_name)\ .options(joinedload('templatetypes') .joinedload('typeattrs') .joinedload('attr')).one() if allow_update == False: raise HydraError("Existing Template Found with name %s" % (template_name, )) else: log.debug("Existing template found. name=%s", template_name) tmpl_i.layout = template_layout tmpl_i.description = template_description except NoResultFound: log.debug("Template not found. Creating new one. name=%s", template_name) tmpl_i = Template(name=template_name, description=template_description, layout=template_layout) db.DBSession.add(tmpl_i) types = xml_tree.find('resources') #Delete any types which are in the DB but no longer in the XML file type_name_map = {r.name: r.id for r in tmpl_i.templatetypes} attr_name_map = {} for type_i in tmpl_i.templatetypes: for typeattr in type_i.typeattrs: attr_name_map[typeattr.attr.name] = (typeattr.attr.id, typeattr.type_id) existing_types = set([r.name for r in tmpl_i.templatetypes]) new_types = set([r.find('name').text for r in types.findall('resource')]) types_to_delete = existing_types - new_types for type_to_delete in types_to_delete: type_id = type_name_map[type_to_delete] try: type_i = db.DBSession.query(TemplateType).filter( TemplateType.id == type_id).one() log.debug("Deleting type %s", type_i.name) db.DBSession.delete(type_i) except NoResultFound: pass #Add or update types. for resource in types.findall('resource'): type_name = resource.find('name').text #check if the type is already in the DB. If not, create a new one. type_is_new = False if type_name in existing_types: type_id = type_name_map[type_name] type_i = db.DBSession.query(TemplateType).filter( TemplateType.id == type_id).options( joinedload('typeattrs').joinedload('attr')).one() else: log.debug("Type %s not found, creating new one.", type_name) type_i = TemplateType() type_i.name = type_name tmpl_i.templatetypes.append(type_i) type_is_new = True if resource.find('alias') is not None: type_i.alias = resource.find('alias').text if resource.find('description') is not None: type_i.description = resource.find('description').text if resource.find('type') is not None: type_i.resource_type = resource.find('type').text if resource.find('layout') is not None and \ resource.find('layout').text is not None: layout = resource.find('layout') layout_string = get_etree_layout_as_dict(layout) type_i.layout = json.dumps(layout_string) #delete any TypeAttrs which are in the DB but not in the XML file existing_attrs = [] if not type_is_new: for r in tmpl_i.templatetypes: if r.name == type_name: for typeattr in r.typeattrs: existing_attrs.append(typeattr.attr.name) existing_attrs = set(existing_attrs) template_attrs = set( [r.find('name').text for r in resource.findall('attribute')]) attrs_to_delete = existing_attrs - template_attrs for attr_to_delete in attrs_to_delete: attr_id, type_id = attr_name_map[attr_to_delete] try: attr_i = db.DBSession.query(TypeAttr).filter( TypeAttr.attr_id == attr_id, TypeAttr.type_id == type_id).options( joinedload('attr')).one() db.DBSession.delete(attr_i) log.debug("Attr %s in type %s deleted", attr_i.attr.name, attr_i.templatetype.name) except NoResultFound: log.debug("Attr %s not found in type %s", attr_id, type_id) continue #Add or update type typeattrs for attribute in resource.findall('attribute'): new_typeattr = _parse_xml_typeattr(type_i, attribute, user_id=user_id) db.DBSession.flush() return tmpl_i