def __init__(self, url=None, sessionid=None, app_name=None):
if url is None:
port = config.getint('hydra_client', 'port', 80)
domain = config.get('hydra_client', 'domain', '127.0.0.1')
path = config.get('hydra_client', 'soap_path', 'soap')
#The domain may or may not specify the protocol, so do a check.
if domain.find('http') == -1:
self.url = "http://%s:%s/%s?wsdl" % (domain, port, path)
else:
self.url = "%s:%s/%s?wsdl" % (domain, port, path)
else:
log.info("Using user-defined URL: %s", url)
port = _get_port(url)
hostname = _get_hostname(url)
path = _get_path(url)
protocol = _get_protocol(url)
self.url = "%s://%s:%s%s/soap?wsdl" % (protocol, hostname, port,
path)
log.info("Setting URL %s", self.url)
self.app_name = app_name
self.sessionid = sessionid
self.retxml = False
self.client = Client(self.url,
timeout=3600,
plugins=[FixNamespace()],
retxml=self.retxml)
self.client.add_prefix('hyd', 'soap_server.hydra_complexmodels')
cache = self.client.options.cache
cache.setduration(days=10)
def reindex_timeseries(ts_string, new_timestamps):
"""
get data for timesamp
:param a JSON string, in pandas-friendly format
:param a timestamp or list of timestamps (datetimes)
:returns a pandas data frame, reindexed with the supplied timestamos or None if no data is found
"""
#If a single timestamp is passed in, turn it into a list
#Reindexing can't work if it's not a list
if not isinstance(new_timestamps, list):
new_timestamps = [new_timestamps]
#Convert the incoming timestamps to datetimes
#if they are not datetimes.
new_timestamps_converted = []
for t in new_timestamps:
new_timestamps_converted.append(get_datetime(t))
new_timestamps = new_timestamps_converted
seasonal_year = config.get('DEFAULT','seasonal_year', '1678')
seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999')
ts = ts_string.replace(seasonal_key, seasonal_year)
timeseries = pd.read_json(ts)
idx = timeseries.index
ts_timestamps = new_timestamps
#'Fix' the incoming timestamp in case it's a seasonal value
if type(idx) == pd.DatetimeIndex:
if set(idx.year) == set([int(seasonal_year)]):
if isinstance(new_timestamps, list):
seasonal_timestamp = []
for t in ts_timestamps:
t_1900 = t.replace(year=int(seasonal_year))
seasonal_timestamp.append(t_1900)
ts_timestamps = seasonal_timestamp
#Reindex the timeseries to reflect the requested timestamps
reindexed_ts = timeseries.reindex(ts_timestamps, method='ffill')
i = reindexed_ts.index
reindexed_ts.index = pd.Index(new_timestamps, names=i.names)
#If there are no values at all, just return None
if len(reindexed_ts.dropna()) == 0:
return None
#Replace all numpy NAN values with None
pandas_ts = reindexed_ts.where(reindexed_ts.notnull(), None)
return pandas_ts
def login(self, username=None, password=None):
if username is None:
username = config.get('hydra_client', 'user')
if password is None:
password = config.get('hydra_client', 'password')
login_params = {'username': username, 'password': password}
resp = self.call('login', login_params)
#set variables for use in request headers
log.info(resp)
def login(self, username=None, password=None):
"""Establish a connection to the specified server. If the URL of the
server is not specified as an argument of this function, the URL
defined in the configuration file is used."""
# Connect
token = self.client.factory.create('RequestHeader')
if self.sessionid is None:
if username is None:
user = config.get('hydra_client', 'user')
if password is None:
passwd = config.get('hydra_client', 'password')
login_response = self.client.service.login(user, passwd)
token.user_id = login_response.user_id
sessionid = login_response.sessionid
token.username = user
token.sessionid = sessionid
self.client.set_options(soapheaders=token)
return session_id
def __init__(self, url=None, sessionid=None, app_name=None):
if url is None:
port = config.getint('hydra_client', 'port', 80)
domain = config.get('hydra_client', 'domain', '127.0.0.1')
path = config.get('hydra_client', 'json_path', 'json')
#The domain may or may not specify the protocol, so do a check.
if domain.find('http') == -1:
self.url = "http://%s:%s/%s" % (domain, port, path)
else:
self.url = "%s:%s/%s" % (domain, port, path)
else:
log.info("Using user-defined URL: %s", url)
port = _get_port(url)
hostname = _get_hostname(url)
path = _get_path(url)
protocol = _get_protocol(url)
self.url = "%s://%s:%s%s/json" % (protocol, hostname, port, path)
log.info("Setting URL %s", self.url)
self.app_name = app_name
self.session_id = sessionid
def date_to_string(date, seasonal=False):
"""Convert a date to a standard string used by Hydra. The resulting string
looks like this::
'2013-10-03 00:49:17.568-0400'
Hydra also accepts seasonal time series (yearly recurring). If the flag
``seasonal`` is set to ``True``, this function will generate a string
recognised by Hydra as seasonal time stamp.
"""
seasonal_key = config.get('DEFAULT', 'seasonal_key', '9999')
if seasonal:
FORMAT = seasonal_key+'-%m-%dT%H:%M:%S.%f'
else:
FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
return date.strftime(FORMAT)
def validate_plugin_xml(plugin_xml_file_path):
log.info('Validating plugin xml file (%s).' % plugin_xml_file_path)
try:
with open(plugin_xml_file_path) as f:
plugin_xml = f.read()
except:
raise HydraPluginError("Couldn't find plugin.xml.")
try:
plugin_xsd_path = os.path.expanduser(
config.get('plugin', 'plugin_xsd_path'))
log.info("Plugin Input xsd: %s", plugin_xsd_path)
xmlschema_doc = etree.parse(plugin_xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_tree = etree.fromstring(plugin_xml)
except XMLSyntaxError, e:
raise HydraPluginError("There is an error in your XML syntax: %s" % e)
def xsd_validate(template_file):
"""
Validate a template against the xsd.
Return the xml tree if successful.
"""
with open(template_file) as f:
xml_template = f.read()
template_xsd_path = os.path.expanduser(
config.get('templates', 'template_xsd_path'))
log.info("Template xsd: %s", template_xsd_path)
xmlschema_doc = etree.parse(template_xsd_path)
xmlschema = etree.XMLSchema(xmlschema_doc)
xml_tree = etree.fromstring(xml_template)
try:
xmlschema.assertValid(xml_tree)
except etree.DocumentInvalid as e:
raise HydraPluginError('Template validation failed: ' + e.message)
log.info("Template XSD validation successful.")
return xml_tree
def guess_timefmt(datestr):
"""
Try to guess the format a date is written in.
The following formats are supported:
================= ============== ===============
Format Example Python format
----------------- -------------- ---------------
``YYYY-MM-DD`` 2002-04-21 %Y-%m-%d
``YYYY.MM.DD`` 2002.04.21 %Y.%m.%d
``YYYY MM DD`` 2002 04 21 %Y %m %d
``DD-MM-YYYY`` 21-04-2002 %d-%m-%Y
``DD.MM.YYYY`` 21.04.2002 %d.%m.%Y
``DD MM YYYY`` 21 04 2002 %d %m %Y
``DD/MM/YYYY`` 21/04/2002 %d/%m/%Y
================= ============== ===============
These formats can also be used for seasonal (yearly recurring) time series.
The year needs to be replaced by ``9999`` or another configurable year
representing the seasonal year..
The following formats are recognised depending on your locale setting.
There is no guarantee that this will work.
================= ============== ===============
Format Example Python format
----------------- -------------- ---------------
``DD-mmm-YYYY`` 21-Apr-2002 %d-%b-%Y
``DD.mmm.YYYY`` 21.Apr.2002 %d.%b.%Y
``DD mmm YYYY`` 21 Apr 2002 %d %b %Y
``mmm DD YYYY`` Apr 21 2002 %b %d %Y
``Mmmmm DD YYYY`` April 21 2002 %B %d %Y
================= ============== ===============
.. note::
- The time needs to follow this definition without exception:
`%H:%M:%S.%f`. A complete date and time should therefore look like
this::
2002-04-21 15:29:37.522
- Be aware that in a file with comma separated values you should not
use a date format that contains commas.
"""
seasonal_key = str(config.get('DEFAULT', 'seasonal_key', '9999'))
#replace 'T' with space to handle ISO times.
if datestr.find('T') > 0:
dt_delim = 'T'
else:
dt_delim = ' '
delimiters = ['-', '.', ' ', '/']
formatstrings = [['%Y', '%m', '%d'],
['%d', '%m', '%Y'],
['%d', '%b', '%Y'],
['XXXX', '%m', '%d'],
['%d', '%m', 'XXXX'],
['%d', '%b', 'XXXX'],
[seasonal_key, '%m', '%d'],
['%d', '%m', seasonal_key],
['%d', '%b', seasonal_key]]
timeformats = ['%H:%M:%S.%f', '%H:%M:%S', '%H:%M', '%H:%M:%S.%f000Z', '%H:%M:%S.%fZ']
# Check if a time is indicated or not
for timefmt in timeformats:
try:
datetime.strptime(datestr.split(dt_delim)[-1].strip(), timefmt)
usetime = True
break
except ValueError:
usetime = False
# Check the simple ones:
for fmt in formatstrings:
for delim in delimiters:
datefmt = fmt[0] + delim + fmt[1] + delim + fmt[2]
if usetime:
for timefmt in timeformats:
complfmt = datefmt + dt_delim + timefmt
try:
datetime.strptime(datestr, complfmt)
return complfmt
except ValueError:
pass
else:
try:
datetime.strptime(datestr, datefmt)
return datefmt
except ValueError:
pass
# Check for other formats:
custom_formats = ['%d/%m/%Y', '%b %d %Y', '%B %d %Y','%d/%m/XXXX', '%d/%m/'+seasonal_key]
for fmt in custom_formats:
if usetime:
for timefmt in timeformats:
complfmt = fmt + dt_delim + timefmt
try:
datetime.strptime(datestr, complfmt)
return complfmt
except ValueError:
pass
else:
try:
datetime.strptime(datestr, fmt)
return fmt
except ValueError:
pass
return None