def setUp(self):
self.create_user("UserA")
self.create_user("UserB")
self.create_user("UserC")
self.project_id = self.create_project().id
self.fmt = config.get('DEFAULT', 'datetime_format', "%Y-%m-%dT%H:%M:%S.%f000Z")
self.user_id = config.get('DEFAULT', 'root_user_id', 2)
def validate_plugin_xml(plugin_xml_file_path):
log.info('Validating plugin xml file (%s).' % plugin_xml_file_path)
try:
with open(plugin_xml_file_path) as f:
plugin_xml = f.read()
except:
raise HydraPluginError("Couldn't find plugin.xml.")
try:
plugin_xsd_path = os.path.expanduser(
config.get('plugin', 'plugin_xsd_path'))
log.info("Plugin Input xsd: %s", plugin_xsd_path)
xmlschema_doc = etree.parse(plugin_xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_tree = etree.fromstring(plugin_xml)
except XMLSyntaxError as e:
raise HydraPluginError("There is an error in your XML syntax: %s" % e)
except ParseError as e:
raise HydraPluginError("There is an error in your XML: %s" % e)
except Exception as e:
log.exception(e)
raise HydraPluginError(
"An unknown error occurred with the plugin xsd: %s" % e.message)
try:
xmlschema.assertValid(xml_tree)
except etree.DocumentInvalid as e:
raise HydraPluginError('Plugin validation failed: ' + e.message)
log.info("Plugin XML OK")
def validate(self):
base_ts = pd.Timestamp("01-01-1970")
#TODO: We need a more permanent solution to seasonal/repeating timeseries
seasonal_year = config.get('DEFAULT','seasonal_year', '1678')
seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999')
jd = json.loads(self.value, object_pairs_hook=collections.OrderedDict)
for k,v in jd.items():
for date in (six.text_type(d) for d in v.keys()):
#A date with '9999' in it is a special case, but is an invalid year
#for pandas, so replace it with the first year allowed by pandas -- 1678
if date.find(seasonal_key) >= 0:
date = date.replace(seasonal_key, seasonal_year)
ts = pd.Timestamp(date)
assert isinstance(ts, base_ts.__class__) # Same type as known valid ts
def __init__(self):
"""Initialise job queue and similar...
"""
self.install_path = config.get('plugin', 'default_directory')
if self.install_path is None:
log.critical("Plugin folder not defined in config.ini! "
"Cannot scan for installed plugins.")
return None
self.installed_apps = scan_installed_apps(self.install_path)
def create_sqlite_backup_db(audit_tables):
"""
return an inspector object
"""
#we always want to create a whole new DB, so delete the old one first
#if it exists.
try:
Popen("rm %s"%(config.get('sqlite', 'backup_url')), shell=True)
logging.warn("Old sqlite backup DB removed")
except Exception as e:
logging.warn(e)
try:
aux_dir = config.get('DEFAULT', 'hydra_aux_dir')
os.mkdir(aux_dir)
logging.warn("%s created", aux_dir)
except Exception as e:
logging.warn(e)
try:
backup_dir = config.get('db', 'export_target')
os.mkdir(backup_dir)
logging.warn("%s created", backup_dir)
except Exception as e:
logging.warn(e)
db = create_engine(sqlite_engine, echo=True)
db.connect()
metadata = MetaData(db)
for main_audit_table in audit_tables:
cols = []
for c in main_audit_table.columns:
col = c.copy()
if col.type.python_type == Decimal:
col.type = DECIMAL()
cols.append(col)
Table(main_audit_table.name, metadata, *cols, sqlite_autoincrement=True)
metadata.create_all(db)
def setUp(self):
util.connect()
create_default_users_and_perms()
self.create_user("UserA")
self.create_user("UserB")
self.create_user("UserC")
self.project_id = self.create_project().id
self.fmt = config.get('DEFAULT', 'datetime_format',
"%Y-%m-%dT%H:%M:%S.%f000Z")
yield # perform the test
# now tear down
self.tearDown()
def test_create_template_from_network(self, client, network_with_data):
network = network_with_data
net_template = client.get_network_as_xml_template(network.id)
assert net_template is not None
template_xsd_path = config.get('templates', 'template_xsd_path')
xmlschema_doc = etree.parse(template_xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_tree = etree.fromstring(net_template)
xmlschema.assertValid(xml_tree)
def test_get_xml(self, client, template_json_object):
xml_tmpl = template_json_object
db_template = client.get_template_as_xml(xml_tmpl.id)
assert db_template is not None
template_xsd_path = config.get('templates', 'template_xsd_path')
xmlschema_doc = etree.parse(template_xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_tree = etree.fromstring(db_template)
xmlschema.assertValid(xml_tree)
def test_get_xml(self):
xml_tmpl = self.test_add_xml()
db_template = get_template_as_xml(xml_tmpl.id)
assert db_template is not None
template_xsd_path = config.get('templates', 'template_xsd_path')
xmlschema_doc = etree.parse(template_xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_tree = etree.fromstring(db_template)
xmlschema.assertValid(xml_tree)
def xsd_validate(template_file):
"""
Validate a template against the xsd.
Return the xml tree if successful.
"""
with open(template_file) as f:
xml_template = f.read()
template_xsd_path = os.path.expanduser(config.get('templates',
'template_xsd_path'))
log.info("Template xsd: %s", template_xsd_path)
xmlschema_doc = etree.parse(template_xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_tree = etree.fromstring(xml_template)
try:
xmlschema.assertValid(xml_tree)
except etree.DocumentInvalid as e:
raise HydraPluginError('Template validation failed: ' + e.message)
log.info("Template XSD validation successful.")
return xml_tree
Column,\
Integer,\
String,\
TIMESTAMP,\
text,\
DDL
from sqlalchemy.engine import reflection
import logging
from hydra_base import config
from subprocess import Popen
from sqlalchemy.types import DECIMAL, NUMERIC
from sqlalchemy.dialects.mysql.base import DOUBLE
from decimal import Decimal
import os
engine_name = config.get('mysqld', 'url')
sqlite_engine = "sqlite:///%s"%(config.get('sqlite', 'backup_url'))
def connect():
"""
return an inspector object
"""
db = create_engine(engine_name, echo=True)
db.connect()
return db
def create_sqlite_backup_db(audit_tables):
"""
return an inspector object
"""
from hydra_base.db import DeclarativeBase as _db
from hydra_base.util.hdb import create_default_users_and_perms, make_root_user
from hydra_base.lib.objects import JSONObject
import hydra_base
from hydra_base import config
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import datetime
import os
from hydra_pywr.template import generate_pywr_attributes, generate_pywr_template, PYWR_DEFAULT_DATASETS
global user_id
user_id = config.get('DEFAULT', 'root_user_id', 1)
@pytest.fixture
def db_backend(request):
return 'sqlite'
@pytest.fixture()
def testdb_uri(db_backend, tmpdir):
if db_backend == 'sqlite':
# Use a :memory: database for the tests.
return 'sqlite:///{}'.format(os.path.join(tmpdir, 'test.db'))
elif db_backend == 'postgres':
# This is designed to work on Travis CI
return 'postgresql://postgres@localhost:5432/hydra_base_test'
elif db_backend == 'mysql':
from hydra_base.db import DeclarativeBase as _db
from hydra_base.util.hdb import create_default_users_and_perms, make_root_user
from hydra_base.lib.objects import JSONObject
import hydra_base
from hydra_base import config
import pytest
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
import datetime
import os
from hydra_pywr.template import generate_pywr_attributes, generate_pywr_template, PYWR_DEFAULT_DATASETS
global user_id
user_id = config.get('DEFAULT', 'root_user_id', 1)
@pytest.fixture
def db_backend(request):
return 'sqlite'
@pytest.fixture()
def testdb_uri(db_backend, tmpdir):
if db_backend == 'sqlite':
# Use a :memory: database for the tests.
return 'sqlite:///{}'.format(os.path.join(tmpdir, 'test.db'))
elif db_backend == 'postgres':
# This is designed to work on Travis CI
return 'postgresql://postgres@localhost:5432/hydra_base_test'
elif db_backend == 'mysql':
return 'mysql+mysqlconnector://root@localhost/hydra_base_test'
def __init__(self):
self.app_registry = AppRegistry()
self.job_queue = JobQueue(config.get('plugin', 'queue_directory', '/tmp'))
self.job_queue.rebuild(self.app_registry)
self.upload_dir = config.get('plugin', 'upload_dir', '/tmp/uploads')
import click
import os
from hydra_base import config
from hydra_client.connection import JSONConnection
from hydra_client.click import hydra_app, make_plugins, write_plugins
import json
from collections import defaultdict
import pandas
import re
from .gis import import_nodes_from_shapefile, import_links_from_shapefile
from . import data
UPLOAD_DIR = config.get('plugin', 'upload_dir', '/tmp/uploads')
UPLOAD_DIR = config.get('plugin', 'output_dir', '/tmp/uploads')
def get_client(hostname, **kwargs):
return JSONConnection(app_name='Pywr GIS App', db_url=hostname, **kwargs)
def get_logged_in_client(context, user_id=None):
session = context['session']
client = get_client(context['hostname'],
session_id=session,
user_id=user_id)
if client.user_id is None:
client.login(username=context['username'],
password=context['password'])
return client
def import_template_xml(template_xml, allow_update=True, **kwargs):
"""
Add the template, type and typeattrs described
in an XML file.
Delete type, typeattr entries in the DB that are not in the XML file
The assumption is that they have been deleted and are no longer required.
"""
user_id = kwargs.get('user_id')
template_xsd_path = config.get('templates', 'template_xsd_path')
xmlschema_doc = etree.parse(template_xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_tree = etree.fromstring(template_xml)
xmlschema.assertValid(xml_tree)
template_name = xml_tree.find('template_name').text
template_description = xml_tree.find('template_description')
if template_description is not None:
template_description = template_description.text
template_layout = None
if xml_tree.find('layout') is not None and \
xml_tree.find('layout').text is not None:
layout = xml_tree.find('layout')
layout_string = get_etree_layout_as_dict(layout)
template_layout = json.dumps(layout_string)
try:
tmpl_i = db.DBSession.query(Template).filter(Template.name == template_name)\
.options(joinedload('templatetypes')
.joinedload('typeattrs')
.joinedload('attr')).one()
if allow_update == False:
raise HydraError("Existing Template Found with name %s" %
(template_name, ))
else:
log.debug("Existing template found. name=%s", template_name)
tmpl_i.layout = template_layout
tmpl_i.description = template_description
except NoResultFound:
log.debug("Template not found. Creating new one. name=%s",
template_name)
tmpl_i = Template(name=template_name,
description=template_description,
layout=template_layout)
db.DBSession.add(tmpl_i)
types = xml_tree.find('resources')
#Delete any types which are in the DB but no longer in the XML file
type_name_map = {r.name: r.id for r in tmpl_i.templatetypes}
attr_name_map = {}
for type_i in tmpl_i.templatetypes:
for typeattr in type_i.typeattrs:
attr_name_map[typeattr.attr.name] = (typeattr.attr.id,
typeattr.type_id)
existing_types = set([r.name for r in tmpl_i.templatetypes])
new_types = set([r.find('name').text for r in types.findall('resource')])
types_to_delete = existing_types - new_types
for type_to_delete in types_to_delete:
type_id = type_name_map[type_to_delete]
try:
type_i = db.DBSession.query(TemplateType).filter(
TemplateType.id == type_id).one()
log.debug("Deleting type %s", type_i.name)
db.DBSession.delete(type_i)
except NoResultFound:
pass
#Add or update types.
for resource in types.findall('resource'):
type_name = resource.find('name').text
#check if the type is already in the DB. If not, create a new one.
type_is_new = False
if type_name in existing_types:
type_id = type_name_map[type_name]
type_i = db.DBSession.query(TemplateType).filter(
TemplateType.id == type_id).options(
joinedload('typeattrs').joinedload('attr')).one()
else:
log.debug("Type %s not found, creating new one.", type_name)
type_i = TemplateType()
type_i.name = type_name
tmpl_i.templatetypes.append(type_i)
type_is_new = True
if resource.find('alias') is not None:
type_i.alias = resource.find('alias').text
if resource.find('description') is not None:
type_i.description = resource.find('description').text
if resource.find('type') is not None:
type_i.resource_type = resource.find('type').text
if resource.find('layout') is not None and \
resource.find('layout').text is not None:
layout = resource.find('layout')
layout_string = get_etree_layout_as_dict(layout)
type_i.layout = json.dumps(layout_string)
#delete any TypeAttrs which are in the DB but not in the XML file
existing_attrs = []
if not type_is_new:
for r in tmpl_i.templatetypes:
if r.name == type_name:
for typeattr in r.typeattrs:
existing_attrs.append(typeattr.attr.name)
existing_attrs = set(existing_attrs)
template_attrs = set(
[r.find('name').text for r in resource.findall('attribute')])
attrs_to_delete = existing_attrs - template_attrs
for attr_to_delete in attrs_to_delete:
attr_id, type_id = attr_name_map[attr_to_delete]
try:
attr_i = db.DBSession.query(TypeAttr).filter(
TypeAttr.attr_id == attr_id,
TypeAttr.type_id == type_id).options(
joinedload('attr')).one()
db.DBSession.delete(attr_i)
log.debug("Attr %s in type %s deleted", attr_i.attr.name,
attr_i.templatetype.name)
except NoResultFound:
log.debug("Attr %s not found in type %s", attr_id, type_id)
continue
#Add or update type typeattrs
for attribute in resource.findall('attribute'):
new_typeattr = _parse_xml_typeattr(type_i,
attribute,
user_id=user_id)
db.DBSession.flush()
return tmpl_i
