def _parse_response(self, res):
"""
Parse the response sent back from the Haystack server.
"""
#decoded = '' # Referenced before assignment protection
# content_type we get with nHaystack is Content_type : application/json; charset=UTF-8
content_type = res.headers['Content-Type']
if ';' in content_type:
# Separate encoding from content type
(content_type, encoding) = content_type.split(';', 1)
content_type = content_type.strip()
# TODO: do we need to convert to Unicode, of so, how?
if content_type in ('text/zinc', 'text/plain'):
decoded = hszinc.parse(res.text, mode=hszinc.MODE_ZINC)[0]
elif 'application/json' in content_type:
decoded = hszinc.parse(res.text, mode=hszinc.MODE_JSON)
else:
raise NotImplementedError("Don't know how to parse type %s" \
% content_type)
if 'err' in decoded.metadata:
raise HaystackError(decoded.metadata.get('dis', 'Unknown error'),
traceback=decoded.metadata.get(
'traceback', None))
return decoded
def _on_response(self, response):
"""
Process the response given back by the HTTP server.
"""
try:
# Does the session want to invoke any relevant hooks?
# This allows a session to detect problems in the session and
# abort the operation.
if hasattr(self._session, '_on_http_grid_response'):
self._session._on_http_grid_response(response)
# Process the HTTP error, if any.
if isinstance(response, AsynchronousException):
response.reraise()
# If we're expecting a raw response back, then just hand the
# request object back and finish here.
if self._raw_response:
self._state_machine.response_ok(result=response)
return
# What format grid did we get back?
content_type = response.content_type
body = response.text
if content_type in ('text/zinc', 'text/plain'):
# We have been given a grid in ZINC format.
decoded = hszinc.parse(body, mode=hszinc.MODE_ZINC)
elif content_type == 'application/json':
# We have been given a grid in JSON format.
decoded = [hszinc.parse(body, mode=hszinc.MODE_JSON)]
else:
# We don't recognise this!
raise ValueError('Unrecognised content type %s' % content_type)
# Check for exceptions
def _check_err(grid):
try:
if 'err' in grid.metadata:
raise HaystackError(
grid.metadata.get('dis', 'Unknown Error'),
grid.metadata.get('traceback', None))
return grid
except:
return AsynchronousException()
decoded = [_check_err(g) for g in decoded]
if not self._multi_grid:
decoded = decoded[0]
# If we get here, then the request itself succeeded.
if self._cache:
with self._session._grid_lk:
self._session._grid_cache[self._cache_key] = \
(None, time() + self._session._grid_expiry, decoded)
self._state_machine.response_ok(result=decoded)
except: # Catch all exceptions for the caller.
self._log.debug('Parse fails', exc_info=1)
self._state_machine.exception(result=AsynchronousException())
def _on_response(self, response):
"""
Process the response given back by the HTTP server.
"""
try:
# Process the HTTP error, if any.
if isinstance(response, AsynchronousException):
response.reraise()
# If we're expecting a raw response back, then just hand the
# request object back and finish here.
if self._raw_response:
self._state_machine.response_ok(result=response)
return
# What format grid did we get back?
content_type = response.content_type
body = response.text
if content_type in ('text/zinc', 'text/plain'):
# We have been given a grid in ZINC format.
decoded = hszinc.parse(body, mode=hszinc.MODE_ZINC)
elif content_type == 'application/json':
# We have been given a grid in JSON format.
decoded = [hszinc.parse(body, mode=hszinc.MODE_JSON)]
else:
# We don't recognise this!
raise ValueError('Unrecognised content type %s' % content_type)
# Check for exceptions
def _check_err(grid):
try:
if 'err' in grid.metadata:
raise HaystackError(
grid.metadata.get('dis', 'Unknown Error'),
grid.metadata.get('traceback', None))
return grid
except:
return AsynchronousException()
decoded = [_check_err(g) for g in decoded]
if not self._multi_grid:
decoded = decoded[0]
# If we get here, then the request itself succeeded.
if self._cache:
with self._session._grid_lk:
self._session._grid_cache[self._cache_key] = \
(None, time() + self._session._grid_expiry, decoded)
self._state_machine.response_ok(result=decoded)
except: # Catch all exceptions for the caller.
self._log.debug('Parse fails', exc_info=1)
self._state_machine.exception(result=AsynchronousException())
def test_col_meta():
grid_list = hszinc.parse('''ver:"2.0"
aColumn aString:"aValue" aNumber:3.14159 aNull:N aMarker:M anotherMarker aQuantity:123Hz aDate:2016-01-13 aTime:06:44:00 aTimestamp:2016-01-13T06:44:00+10:00 Brisbane aPlace:C(-27.4725,153.003)
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 0
assert len(grid_list[0].metadata) == 0
assert list(grid_list[0].column.keys()) == ['aColumn']
meta = grid_list[0].column['aColumn']
assert list(meta.keys()) == ['aString', 'aNumber', 'aNull',
'aMarker', 'anotherMarker', 'aQuantity', 'aDate', 'aTime',
'aTimestamp', 'aPlace']
assert meta['aString'] == 'aValue'
assert meta['aNumber'] == 3.14159
assert meta['aNull'] is None
assert meta['aMarker'] is hszinc.MARKER
assert meta['anotherMarker'] is hszinc.MARKER
assert isinstance(meta['aQuantity'], hszinc.Quantity)
assert meta['aQuantity'].value == 123
assert meta['aQuantity'].unit == 'Hz'
assert isinstance(meta['aDate'], datetime.date)
assert meta['aDate'] == datetime.date(2016,1,13)
assert isinstance(meta['aTime'], datetime.time)
assert meta['aTime'] == datetime.time(6,44)
assert isinstance(meta['aTimestamp'], datetime.datetime)
assert meta['aTimestamp'] == \
pytz.timezone('Australia/Brisbane').localize(\
datetime.datetime(2016,1,13,6,44))
assert isinstance(meta['aPlace'], hszinc.Coordinate)
assert meta['aPlace'].latitude == -27.4725
assert meta['aPlace'].longitude == 153.003
def test_nodehaystack_07():
grid_list = hszinc.parse(u'''ver:"2.0"
a, b
-3.1kg,4kg
5%,3.2%
5kWh/ft\u00b2,-15kWh/m\u00b2
123e+12kJ/kg_dry,74\u0394\u00b0F
''')
assert len(grid_list) == 1
grid = grid_list.pop(0)
assert len(grid) == 4
assert len(grid.metadata) == 0
assert list(grid.column.keys()) == ['a','b']
row = grid.pop(0)
assert row['a'] == hszinc.Quantity(-3.1,'kg')
assert row['b'] == hszinc.Quantity(4,'kg')
row = grid.pop(0)
assert row['a'] == hszinc.Quantity(5,'%')
assert row['b'] == hszinc.Quantity(3.2,'%')
row = grid.pop(0)
assert row['a'] == hszinc.Quantity(5,u'kWh/ft\u00b2')
assert row['b'] == hszinc.Quantity(-15,u'kWh/m\u00b2')
row = grid.pop(0)
assert row['a'] == hszinc.Quantity(123e12,'kJ/kg_dry')
assert row['b'] == hszinc.Quantity(74,u'\u0394\u00b0F')
def test_nodehaystack_10():
grid_list = hszinc.parse('''ver:"2.0" bg: Bin(image/jpeg) mark
file1 dis:"F1" icon: Bin(image/gif),file2 icon: Bin(image/jpg)
Bin(text/plain),N
4,Bin(image/png)
Bin(text/html; a=foo; bar="sep"),Bin(text/html; charset=utf8)
''')
assert len(grid_list) == 1
grid = grid_list.pop(0)
assert len(grid) == 3
assert list(grid.metadata.keys()) == ['bg','mark']
assert grid.metadata['bg'] == hszinc.Bin('image/jpeg')
assert grid.metadata['mark'] is hszinc.MARKER
assert list(grid.column.keys()) == ['file1','file2']
assert list(grid.column['file1'].keys()) == ['dis','icon']
assert grid.column['file1']['dis'] == 'F1'
assert grid.column['file1']['icon'] == hszinc.Bin('image/gif')
assert list(grid.column['file2'].keys()) == ['icon']
assert grid.column['file2']['icon'] == hszinc.Bin('image/jpg')
row = grid.pop(0)
assert row['file1'] == hszinc.Bin('text/plain')
assert row['file2'] is None
row = grid.pop(0)
assert row['file1'] == 4
assert row['file2'] == hszinc.Bin('image/png')
row = grid.pop(0)
assert row['file1'] == hszinc.Bin('text/html; a=foo; bar="sep"')
assert row['file2'] == hszinc.Bin('text/html; charset=utf8')
def test_nodehaystack_01():
grid_list = hszinc.parse('''ver:"2.0"
fooBar33
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 0
assert len(grid_list[0].metadata) == 0
assert list(grid_list[0].column.keys()) == ['fooBar33']
def test_multi_grid():
# Multiple grids are separated by newlines.
grid_list = hszinc.parse('\n'.join([
SIMPLE_EXAMPLE, METADATA_EXAMPLE, NULL_EXAMPLE]))
assert len(grid_list) == 3
check_simple(grid_list[0])
check_metadata(grid_list[1])
check_null(grid_list[2])
def test_multi_grid_json_str():
# Multiple grids are separated by newlines.
grid_list = hszinc.parse(list(map(json.dumps, [SIMPLE_EXAMPLE_JSON,
METADATA_EXAMPLE_JSON, NULL_EXAMPLE_JSON])), mode=hszinc.MODE_JSON)
assert len(grid_list) == 3
check_simple(grid_list[0])
check_metadata(grid_list[1], force_metadata_order=False)
check_null(grid_list[2])
def test_nodehaystack_05():
grid_list = hszinc.parse('''ver:"2.0"
a, b, c, d
T, F, N, -99
2.3, -5e-10, 2.4e20, 123e-10
"", "a", "\\" \\\\ \\t \\n \\r", "\\uabcd"
`path`, @12cbb082-0c02ae73, 4s, -2.5min
M,R,Bin(image/png),Bin(image/png)
2009-12-31, 23:59:01, 01:02:03.123, 2009-02-03T04:05:06Z
INF, -INF, "", NaN
C(12,-34),C(0.123,-.789),C(84.5,-77.45),C(-90,180)
''')
assert len(grid_list) == 1
grid = grid_list.pop(0)
assert len(grid) == 8
assert len(grid.metadata) == 0
assert list(grid.column.keys()) == ['a','b','c','d']
row = grid.pop(0)
assert row['a'] == True
assert row['b'] == False
assert row['c'] is None
assert row['d'] == -99.0
row = grid.pop(0)
assert row['a'] == 2.3
assert row['b'] == -5e-10
assert row['c'] == 2.4e20
assert row['d'] == 123e-10
row = grid.pop(0)
assert row['a'] == ''
assert row['b'] == 'a'
assert row['c'] == '\" \\ \t \n \r'
assert row['d'] == u'\uabcd'
row = grid.pop(0)
assert row['a'] == hszinc.Uri('path')
assert row['b'] == hszinc.Ref('12cbb082-0c02ae73')
assert row['c'] == hszinc.Quantity(4, 's')
assert row['d'] == hszinc.Quantity(-2.5, 'min')
row = grid.pop(0)
assert row['a'] is hszinc.MARKER
assert row['b'] is hszinc.REMOVE
assert row['c'] == hszinc.Bin('image/png')
assert row['d'] == hszinc.Bin('image/png')
row = grid.pop(0)
assert row['a'] == datetime.date(2009,12,31)
assert row['b'] == datetime.time(23,59,1)
assert row['c'] == datetime.time(1,2,3,123000)
assert row['d'] == \
datetime.datetime(2009,2,3,4,5,6,tzinfo=pytz.utc)
row = grid.pop(0)
assert math.isinf(row['a']) and (row['a'] > 0)
assert math.isinf(row['b']) and (row['b'] < 0)
assert row['c'] == ''
assert math.isnan(row['d'])
row = grid.pop(0)
assert row['a'] == hszinc.Coordinate(12,-34)
assert row['b'] == hszinc.Coordinate(.123,-.789)
assert row['c'] == hszinc.Coordinate(84.5,-77.45)
assert row['d'] == hszinc.Coordinate(-90,180)
def test_uri():
grid_list = hszinc.parse('''ver:"2.0"
uri
`http://www.vrt.com.au`
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 1
assert grid_list[0][0]['uri'] == hszinc.Uri('http://www.vrt.com.au')
def test_coord():
grid_list = hszinc.parse('''ver:"2.0"
coord
C(37.55,-77.45)
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 1
assert grid_list[0][0]['coord'] == hszinc.Coordinate(37.55,-77.45)
def test_bin():
grid_list = hszinc.parse('''ver:"2.0"
bin
Bin(text/plain)
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 1
assert grid_list[0][0]['bin'] == hszinc.Bin('text/plain')
def test_marker_in_row():
grid_list = hszinc.parse('''ver:"2.0"
str,marker
"No Marker",
"Marker",M
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 2
assert grid_list[0][0]['marker'] is None
assert grid_list[0][1]['marker'] is hszinc.MARKER
def test_bool():
grid_list = hszinc.parse('''ver:"2.0"
str,bool
"True",T
"False",F
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 2
assert grid_list[0][0]['bool'] == True
assert grid_list[0][1]['bool'] == False
def test_nodehaystack_03():
grid_list = hszinc.parse('''ver:"2.0"
val
N
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 1
assert len(grid_list[0].metadata) == 0
assert list(grid_list[0].column.keys()) == ['val']
assert grid_list[0][0]['val'] is None
def test_date():
grid_list = hszinc.parse('''ver:"2.0"
date
2010-03-13
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 1
assert isinstance(grid_list[0][0]['date'], datetime.date)
assert grid_list[0][0]['date'] == datetime.date(2010,3,13)
def test_nodehaystack_02():
grid_list = hszinc.parse('''ver:"2.0" tag foo:"bar"
xyz
"val"
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 1
assert list(grid_list[0].metadata.keys()) == ['tag', 'foo']
assert grid_list[0].metadata['tag'] is hszinc.MARKER
assert grid_list[0].metadata['foo'] == 'bar'
assert list(grid_list[0].column.keys()) == ['xyz']
assert grid_list[0][0]['xyz'] == 'val'
def test_ref():
grid_list = hszinc.parse('''ver:"2.0"
str,ref
"Basic",@a-basic-ref
"With value",@reference "With value"
''')
assert len(grid_list) == 1
grid = grid_list.pop(0)
assert len(grid) == 2
assert grid[0]['ref'] == hszinc.Ref('a-basic-ref')
assert grid[1]['ref'] == hszinc.Ref('reference', 'With value')
def test_uri_json():
grid = hszinc.parse({
'meta': {'ver':'2.0'},
'cols': [
{'name':'uri'},
],
'rows': [
{'uri': 'u:http://www.vrt.com.au'},
],
}, mode=hszinc.MODE_JSON)
assert len(grid) == 1
assert grid[0]['uri'] == hszinc.Uri('http://www.vrt.com.au')
def test_coord_json():
grid = hszinc.parse({
'meta': {'ver':'2.0'},
'cols': [
{'name':'coord'},
],
'rows': [
{'coord': 'c:37.55,-77.45'},
],
}, mode=hszinc.MODE_JSON)
assert len(grid) == 1
assert grid[0]['coord'] == hszinc.Coordinate(37.55,-77.45)
def test_datetime():
grid_list = hszinc.parse('''ver:"2.0"
datetime
2010-11-28T07:23:02.500-08:00 Los_Angeles
2010-11-28T23:19:29.500+08:00 Taipei
2010-11-28T18:21:58+03:00 GMT-3
2010-11-28T12:22:27-03:00 GMT+3
2010-01-08T05:00:00Z UTC
2010-01-08T05:00:00Z
''')
assert len(grid_list) == 1
check_datetime(grid_list.pop(0))
def test_time():
grid_list = hszinc.parse('''ver:"2.0"
time
08:12:05
08:12:05.5
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 2
assert isinstance(grid_list[0][0]['time'], datetime.time)
assert grid_list[0][0]['time'] == datetime.time(8,12,5)
assert isinstance(grid_list[0][1]['time'], datetime.time)
assert grid_list[0][1]['time'] == datetime.time(8,12,5,500000)
def test_string():
grid_list = hszinc.parse('''ver:"2.0"
str,strExample
"Empty",""
"Basic","Simple string"
"Escaped","This\\tIs\\nA\\r\\"Test\\"\\\\"
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 3
assert grid_list[0][0]['strExample'] == ''
assert grid_list[0][1]['strExample'] == 'Simple string'
assert grid_list[0][2]['strExample'] == 'This\tIs\nA\r"Test"\\'
def test_bin_json():
grid = hszinc.parse({
'meta': {'ver':'2.0'},
'cols': [
{'name':'bin'},
],
'rows': [
{'bin': 'b:text/plain'},
],
}, mode=hszinc.MODE_JSON)
assert len(grid) == 1
assert grid[0]['bin'] == hszinc.Bin('text/plain')
def test_marker_in_row_json():
grid = hszinc.parse({
'meta': {'ver':'2.0'},
'cols': [
{'name':'str'},
{'name':'marker'},
],
'rows': [
{'str': 'No Marker', 'marker':None},
{'str': 'Marker', 'marker':'m:'},
],
}, mode=hszinc.MODE_JSON)
assert grid[0]['marker'] is None
assert grid[1]['marker'] is hszinc.MARKER
def test_bool_json():
grid = hszinc.parse({
'meta': {'ver':'2.0'},
'cols': [
{'name':'str'},
{'name':'bool'},
],
'rows': [
{'str': 'True', 'bool':True},
{'str': 'False','bool':False},
],
}, mode=hszinc.MODE_JSON)
assert grid[0]['bool'] == True
assert grid[1]['bool'] == False
def test_nodehaystack_04():
grid_list = hszinc.parse('''ver:"2.0"
a,b
1,2
3,4
''')
assert len(grid_list) == 1
assert len(grid_list[0]) == 2
assert len(grid_list[0].metadata) == 0
assert list(grid_list[0].column.keys()) == ['a','b']
assert grid_list[0][0]['a'] == 1
assert grid_list[0][0]['b'] == 2
assert grid_list[0][1]['a'] == 3
assert grid_list[0][1]['b'] == 4
def test_date_json():
grid = hszinc.parse({
'meta': {'ver':'2.0'},
'cols': [
{'name':'date'},
],
'rows': [
{'date': 'd:2010-03-13'},
],
}, mode=hszinc.MODE_JSON)
assert len(grid) == 1
assert isinstance(grid[0]['date'], datetime.date)
assert grid[0]['date'] == datetime.date(2010,3,13)
def zincToJson(req):
# Parse the zinc text into grids
grids = hszinc.parse(req)
# Convert this back to a format that pyhaystack understands.
# JSON grids have the following structure:
# // JSON
# {
# "meta": {"ver":"2.0", "projName":"test"},
# "cols":[
# {"name":"dis", "dis":"Equip Name"},
# {"name":"equip"},
# {"name":"siteRef"},
# {"name":"installed"}
# ],
# "rows":[
# {"dis":"RTU-1", "equip":"m:",
# "siteRef":"r:153c-699a HQ", "installed":"d:2005-06-01"},
# {"dis":"RTU-2", "equip":"m:",
# "siteRef":"r:153c-699a HQ", "installed":"d:999-07-12"}
# ]
# }
#
# There is only support for one grid at a time. Out of pure laziness on my
# part, this does not convert scalar values to JSON format but rather, the
# equivalent Python type.
if len(grids) != 1:
raise ValueError('Unable to handle result with %d grids' % \
len(grids))
grid = grids[0]
grid_meta = {'ver':'2.0'}
grid_cols = []
grid_rows = []
json_grid = {
'meta': grid_meta,
'cols': grid_cols,
'rows': grid_rows,
}
grid_meta.update(grid.metadata)
def _col_to_json(c):
(name, meta) = c
json_col = dict(meta)
json_col['name'] = name
return json_col
grid_cols.extend(map(_col_to_json, grid.column.items()))
grid_rows.extend(grid)
return json_grid
def _parse_response(self, res):
"""
Parse the response sent back from the Haystack server.
"""
#decoded = '' # Referenced before assignment protection
# content_type we get with nHaystack is Content_type : application/json; charset=UTF-8
content_type = res.headers['Content-Type']
if ';' in content_type:
# Separate encoding from content type
(content_type, encoding) = content_type.split(';',1)
content_type = content_type.strip()
# TODO: do we need to convert to Unicode, of so, how?
if content_type in ('text/zinc', 'text/plain'):
decoded = hszinc.parse(res.text, mode=hszinc.MODE_ZINC)[0]
elif 'application/json' in content_type:
decoded = hszinc.parse(res.text, mode=hszinc.MODE_JSON)
else:
raise NotImplementedError("Don't know how to parse type %s" \
% content_type)
if 'err' in decoded.metadata:
raise HaystackError(decoded.metadata.get('dis', 'Unknown error'),
traceback=decoded.metadata.get('traceback',None))
return decoded
def test_nodehaystack_08():
grid_list = hszinc.parse('''ver:"2.0"
a,b
2010-03-01T23:55:00.013-05:00 GMT+5,2010-03-01T23:55:00.013+10:00 GMT-10
''')
assert len(grid_list) == 1
grid = grid_list.pop(0)
assert len(grid) == 1
assert len(grid.metadata) == 0
assert list(grid.column.keys()) == ['a','b']
row = grid.pop(0)
assert row['a'] == hszinc.zoneinfo.timezone('GMT+5').localize(\
datetime.datetime(2010,3,1,23,55,0,13000))
assert row['b'] == hszinc.zoneinfo.timezone('GMT-10').localize(\
datetime.datetime(2010,3,1,23,55,0,13000))
def _try_dump_parse(ref_grid, mode):
try:
# Dump the randomised grid to a string
grid_str = hszinc.dump(ref_grid, mode=mode)
except:
# Dump some detail about the grid
print('Failed to dump grid.')
dump_grid(ref_grid)
raise
# Parse the grid string
try:
parsed_grid = hszinc.parse(grid_str, mode=mode, single=True)
except:
print('Failed to parse dumped grid')
dump_grid(ref_grid)
print('--- Parsed string ---')
print(grid_str)
raise
approx_check_grid(parsed_grid, ref_grid)
def test_read_many_id(self, server_session):
(server, session) = server_session
op = session.read([
hszinc.Ref("my.entity.id1"),
hszinc.Ref("my.entity.id2"),
hszinc.Ref("my.entity.id3"),
])
# The operation should still be in progress
assert not op.is_done
# There shall be one request
assert server.requests() == 1
rq = server.next_request()
# Request shall be a GET
assert rq.method == "POST", "Expecting POST, got %s" % rq
# Request shall be for a specific URI
assert rq.uri == BASE_URI + "api/read"
# Body shall be in ZINC
assert rq.headers[b"Content-Type"] == "text/zinc"
# Body shall be a single valid grid of this form:
expected = hszinc.Grid()
expected.column["id"] = {}
expected.extend([
{
"id": hszinc.Ref("my.entity.id1")
},
{
"id": hszinc.Ref("my.entity.id2")
},
{
"id": hszinc.Ref("my.entity.id3")
},
])
actual = hszinc.parse(rq.body.decode("utf-8"), mode=hszinc.MODE_ZINC)
assert len(actual) == 1
grid_cmp(expected, actual[0])
# Accept header shall be given
assert rq.headers[b"Accept"] == "text/zinc"
# Make a grid to respond with
expected = hszinc.Grid()
expected.column["id"] = {}
expected.column["dis"] = {}
expected.extend([
{
"id": hszinc.Ref("my.entity.id1"),
"dis": "my entity 1"
},
{
"id": hszinc.Ref("my.entity.id2"),
"dis": "my entity 2"
},
{
"id": hszinc.Ref("my.entity.id3"),
"dis": "my entity 3"
},
])
rq.respond(
status=200,
headers={b"Content-Type": "text/zinc"},
content=hszinc.dump(expected, mode=hszinc.MODE_ZINC),
)
# State machine should now be done
assert op.is_done
actual = op.result
grid_cmp(expected, actual)
def test_get_multi_entity(self, server_session):
(server, session) = server_session
# Try retrieving existing multiple entities
op = session.get_entity(["my.entity.id1", "my.entity.id2"],
single=False)
# The operation should still be in progress
assert not op.is_done
# There shall be one request
assert server.requests() == 1
rq = server.next_request()
# Request shall be a POST
assert rq.method == "POST", "Expecting POST, got %s" % rq
# Request shall be for base + 'api/[email protected]'
assert rq.uri == BASE_URI + "api/read"
# Accept header shall be given
assert rq.headers[b"Accept"] == "text/zinc"
assert rq.headers[b"Content-Type"] == "text/zinc"
# Body shall be a single grid:
rq_grid = hszinc.parse(rq.body.decode("utf-8"),
mode=hszinc.MODE_ZINC,
single=True)
# It shall have one column; id
assert set(rq_grid.column.keys()) == set(["id"])
# It shall have 2 rows
assert len(rq_grid) == 2
# Each row should only have 'id' values
assert all([(set(r.keys()) == set(["id"])) for r in rq_grid])
# The rows' 'id' column should *only* contain Refs.
assert all([isinstance(r["id"], hszinc.Ref) for r in rq_grid])
# Both IDs shall be listed, we don't about order
assert set([r["id"].name for r in rq_grid
]) == set(["my.entity.id1", "my.entity.id2"])
# Make a grid to respond with
response = hszinc.Grid()
response.column["id"] = {}
response.column["dis"] = {}
response.column["randomTag"] = {}
response.extend([
{
"id": hszinc.Ref("my.entity.id1", value="id1"),
"dis": "A test entity #1",
"randomTag": hszinc.MARKER,
},
{
"id": hszinc.Ref("my.entity.id2", value="id2"),
"dis": "A test entity #2",
"randomTag": hszinc.MARKER,
},
])
rq.respond(
status=200,
headers={b"Content-Type": "text/zinc"},
content=hszinc.dump(response, mode=hszinc.MODE_ZINC),
)
# State machine should now be done
assert op.is_done
entities = op.result
# Response should be a dict
assert isinstance(entities, dict), "%r not a dict" % entities
# Response should have these keys
assert set(entities.keys()) == set(["my.entity.id1", "my.entity.id2"])
entity = entities.pop("my.entity.id1")
assert isinstance(entity, Entity), "%r not an entity" % entity
# The tags should be passed through from the response
assert entity.id.name == "my.entity.id1"
assert entity.tags["dis"] == response[0]["dis"]
assert entity.tags["randomTag"] == response[0]["randomTag"]
entity = entities.pop("my.entity.id2")
assert isinstance(entity, Entity), "%r not an entity" % entity
# The tags should be passed through from the response
assert entity.id.name == "my.entity.id2"
assert entity.tags["dis"] == response[1]["dis"]
assert entity.tags["randomTag"] == response[1]["randomTag"]
def read_file(path):
with open(path, encoding="utf-8") as f:
return f.read()
# 16 cols, 287 rows
SMALL_FILENAME = get_abspath("small_example.zinc")
# 32 cols, 287 rows
MEDIUM_FILENAME = get_abspath("medium_example.zinc")
small_example = read_file(SMALL_FILENAME)
medium_example = read_file(MEDIUM_FILENAME)
print(f"parsing {SMALL_FILENAME} with zincio...")
zincio_total = timeit.timeit(lambda: zincio.parse(small_example), number=20)
print(f"parsing with zincio took {zincio_total / 20} seconds, avg of 20")
print(f"parsing {SMALL_FILENAME} with hszinc...")
hszinc_total = timeit.timeit(lambda: hszinc.parse(small_example), number=5)
print(f"parsing with hszinc took {hszinc_total / 5} seconds, avg of 5")
print(f"parsing {MEDIUM_FILENAME} with zincio...")
zincio_total = timeit.timeit(lambda: zincio.parse(medium_example), number=20)
print(f"parsing with zincio took {zincio_total / 20} seconds, avg of 20")
print(f"parsing {MEDIUM_FILENAME} with hszinc...")
hszinc_total = timeit.timeit(lambda: hszinc.parse(medium_example), number=3)
print(f"parsing with hszinc took {hszinc_total / 3} seconds, avg of 3")
def try_dump_parse():
# Generate a randomised grid of values and try parsing it back.
ref_grid = hszinc.Grid()
ref_grid.metadata.extend(gen_random_meta())
# Randomised columns
for n in range(0, random.randint(1, 5)):
col_name = gen_random_name(existing=ref_grid.column)
if random.choice([True, False]):
ref_grid.column[col_name] = gen_random_meta()
else:
ref_grid.column[col_name] = {}
# Randomised rows
for n in range(0, random.randint(0, 20)):
row = {}
for c in ref_grid.column.keys():
if random.choice([True, False]):
row[c] = gen_random_scalar()
ref_grid.append(row)
try:
# Dump the randomised grid to a string
grid_str = hszinc.dump(ref_grid)
except:
# Dump some detail about the grid
print('Failed to dump grid.')
dump_grid(ref_grid)
raise
# Parse the grid string
try:
grid_list = hszinc.parse(grid_str)
except:
print('Failed to parse dumped grid')
dump_grid(ref_grid)
print('--- Parsed string ---')
print(grid_str)
raise
assert len(grid_list) == 1
parsed_grid = grid_list.pop(0)
# Check metadata matches
try:
assert list(ref_grid.metadata.keys()) \
== list(parsed_grid.metadata.keys())
for key in ref_grid.metadata.keys():
approx_check(ref_grid.metadata[key], parsed_grid.metadata[key])
except:
print('Mismatch in metadata')
print('Reference grid')
dump_grid(ref_grid)
print('Parsed grid')
dump_grid(parsed_grid)
raise
try:
# Check column matches
assert list(ref_grid.column.keys()) \
== list(parsed_grid.column.keys())
except:
print('Mismatch in column')
print('Reference grid')
dump_grid(ref_grid)
print('Parsed grid')
dump_grid(parsed_grid)
raise
for col in ref_grid.column.keys():
try:
for key in ref_grid.column[col].keys():
approx_check(ref_grid.column[col][key], \
parsed_grid.column[col][key])
except:
print('Mismatch in metadata for column %s' % col)
print('Reference: %r' % ref_grid.column[col])
print('Parsed: %r' % parsed_grid.column[col])
raise
try:
# Check row matches
assert len(ref_grid) == len(parsed_grid)
except:
print('Mismatch in row count')
print('Reference grid')
dump_grid(ref_grid)
print('Parsed grid')
dump_grid(parsed_grid)
for (ref_row, parsed_row) in zip(ref_grid, parsed_grid):
try:
for col in ref_grid.column.keys():
approx_check(ref_row.get(col), parsed_row.get(col))
except:
print('Mismatch in row')
print('Reference:')
print(ref_row)
print('Parsed:')
print(parsed_row)
raise
def _on_response(self, response):
"""
Process the response given back by the HTTP server.
"""
# When problems occur :
# print("RESPONSE", response.__dict__)
try:
# Does the session want to invoke any relevant hooks?
# This allows a session to detect problems in the session and
# abort the operation.
if hasattr(self._session, "_on_http_grid_response"):
self._session._on_http_grid_response(response)
# Process the HTTP error, if any.
if isinstance(response, AsynchronousException):
response.reraise()
# If we're expecting a raw response back, then just hand the
# request object back and finish here.
if self._raw_response:
self._state_machine.response_ok(result=response)
return
# What format grid did we get back?
content_type = response.content_type
body = response.text
if content_type in ("text/zinc", "text/plain"):
# We have been given a grid in ZINC format.
decoded = hszinc.parse(body,
mode=hszinc.MODE_ZINC,
single=False)
elif content_type == "application/json":
# We have been given a grid in JSON format.
decoded = hszinc.parse(body,
mode=hszinc.MODE_JSON,
single=False)
elif content_type in ("text/html"):
# We probably fell back to a login screen after auto logoff.
self._state_machine.exception(AsynchronousException())
else:
# We don't recognise this!
raise ValueError("Unrecognised content type %s" % content_type)
# Check for exceptions
def _check_err(grid):
try:
if "err" in grid.metadata:
raise HaystackError(
grid.metadata.get("dis", "Unknown Error"),
grid.metadata.get("traceback", None),
)
return grid
except:
return AsynchronousException()
decoded = [_check_err(g) for g in decoded]
if not self._multi_grid:
decoded = decoded[0]
# If we get here, then the request itself succeeded.
if self._cache:
with self._session._grid_lk:
self._session._grid_cache[self._cache_key] = (
None,
time() + self._session._grid_expiry,
decoded,
)
self._state_machine.response_ok(result=decoded)
except: # Catch all exceptions for the caller.
self._log.debug("Parse fails", exc_info=1)
self._state_machine.exception(result=AsynchronousException())
def test_read_many_id(self, server_session):
(server, session) = server_session
op = session.read([
hszinc.Ref('my.entity.id1'),
hszinc.Ref('my.entity.id2'),
hszinc.Ref('my.entity.id3'),
])
# The operation should still be in progress
assert not op.is_done
# There shall be one request
assert server.requests() == 1
rq = server.next_request()
# Request shall be a GET
assert rq.method == 'POST', 'Expecting POST, got %s' % rq
# Request shall be for a specific URI
assert rq.uri == BASE_URI + 'api/read'
# Body shall be in ZINC
assert rq.headers['Content-Type'] == 'text/zinc'
# Body shall be a single valid grid of this form:
expected = hszinc.Grid()
expected.column['id'] = {}
expected.extend([{
"id": hszinc.Ref('my.entity.id1'),
}, {
"id": hszinc.Ref('my.entity.id2'),
}, {
"id": hszinc.Ref('my.entity.id3'),
}])
actual = hszinc.parse(rq.body, mode=hszinc.MODE_ZINC)
assert len(actual) == 1
grid_cmp(expected, actual[0])
# Accept header shall be given
assert rq.headers['Accept'] == 'text/zinc'
# Make a grid to respond with
expected = hszinc.Grid()
expected.column['id'] = {}
expected.column['dis'] = {}
expected.extend([{
"id": hszinc.Ref('my.entity.id1'),
"dis": 'my entity 1'
}, {
"id": hszinc.Ref('my.entity.id2'),
"dis": 'my entity 2'
}, {
"id": hszinc.Ref('my.entity.id3'),
"dis": 'my entity 3'
}])
rq.respond(status=200, headers={
'Content-Type': 'text/zinc',
}, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC))
# State machine should now be done
assert op.is_done
actual = op.result
grid_cmp(expected, actual)
def test_get_multi_entity(self, server_session):
(server, session) = server_session
# Try retrieving existing multiple entities
op = session.get_entity(['my.entity.id1', 'my.entity.id2'],
single=False)
# The operation should still be in progress
assert not op.is_done
# There shall be one request
assert server.requests() == 1
rq = server.next_request()
# Request shall be a POST
assert rq.method == 'POST', 'Expecting POST, got %s' % rq
# Request shall be for base + 'api/[email protected]'
assert rq.uri == BASE_URI + 'api/read'
# Accept header shall be given
assert rq.headers[b'Accept'] == 'text/zinc'
assert rq.headers[b'Content-Type'] == 'text/zinc'
# Body shall be a single grid:
rq_grid = hszinc.parse(rq.body.decode('utf-8'), mode=hszinc.MODE_ZINC)
assert len(rq_grid) == 1
rq_grid = rq_grid[0]
# It shall have one column; id
assert set(rq_grid.column.keys()) == set(['id'])
# It shall have 2 rows
assert len(rq_grid) == 2
# Each row should only have 'id' values
assert all([(set(r.keys()) == set(['id'])) for r in rq_grid])
# The rows' 'id' column should *only* contain Refs.
assert all([isinstance(r['id'], hszinc.Ref) for r in rq_grid])
# Both IDs shall be listed, we don't about order
assert set([r['id'].name for r in rq_grid]) \
== set(['my.entity.id1', 'my.entity.id2'])
# Make a grid to respond with
response = hszinc.Grid()
response.column['id'] = {}
response.column['dis'] = {}
response.column['randomTag'] = {}
response.extend([{
'id': hszinc.Ref('my.entity.id1', value='id1'),
'dis': 'A test entity #1',
'randomTag': hszinc.MARKER
}, {
'id': hszinc.Ref('my.entity.id2', value='id2'),
'dis': 'A test entity #2',
'randomTag': hszinc.MARKER
}])
rq.respond(status=200,
headers={
b'Content-Type': 'text/zinc',
},
content=hszinc.dump(response, mode=hszinc.MODE_ZINC))
# State machine should now be done
assert op.is_done
entities = op.result
# Response should be a dict
assert isinstance(entities, dict), '%r not a dict' % entity
# Response should have these keys
assert set(entities.keys()) == set(['my.entity.id1', 'my.entity.id2'])
entity = entities.pop('my.entity.id1')
assert isinstance(entity, Entity), '%r not an entity' % entity
# The tags should be passed through from the response
assert entity.id.name == 'my.entity.id1'
assert entity.tags['dis'] == response[0]['dis']
assert entity.tags['randomTag'] == response[0]['randomTag']
entity = entities.pop('my.entity.id2')
assert isinstance(entity, Entity), '%r not an entity' % entity
# The tags should be passed through from the response
assert entity.id.name == 'my.entity.id2'
assert entity.tags['dis'] == response[1]['dis']
assert entity.tags['randomTag'] == response[1]['randomTag']
