def _parse_response(self, res): """ Parse the response sent back from the Haystack server. """ #decoded = '' # Referenced before assignment protection # content_type we get with nHaystack is Content_type : application/json; charset=UTF-8 content_type = res.headers['Content-Type'] if ';' in content_type: # Separate encoding from content type (content_type, encoding) = content_type.split(';', 1) content_type = content_type.strip() # TODO: do we need to convert to Unicode, of so, how? if content_type in ('text/zinc', 'text/plain'): decoded = hszinc.parse(res.text, mode=hszinc.MODE_ZINC)[0] elif 'application/json' in content_type: decoded = hszinc.parse(res.text, mode=hszinc.MODE_JSON) else: raise NotImplementedError("Don't know how to parse type %s" \ % content_type) if 'err' in decoded.metadata: raise HaystackError(decoded.metadata.get('dis', 'Unknown error'), traceback=decoded.metadata.get( 'traceback', None)) return decoded
def _on_response(self, response): """ Process the response given back by the HTTP server. """ try: # Does the session want to invoke any relevant hooks? # This allows a session to detect problems in the session and # abort the operation. if hasattr(self._session, '_on_http_grid_response'): self._session._on_http_grid_response(response) # Process the HTTP error, if any. if isinstance(response, AsynchronousException): response.reraise() # If we're expecting a raw response back, then just hand the # request object back and finish here. if self._raw_response: self._state_machine.response_ok(result=response) return # What format grid did we get back? content_type = response.content_type body = response.text if content_type in ('text/zinc', 'text/plain'): # We have been given a grid in ZINC format. decoded = hszinc.parse(body, mode=hszinc.MODE_ZINC) elif content_type == 'application/json': # We have been given a grid in JSON format. decoded = [hszinc.parse(body, mode=hszinc.MODE_JSON)] else: # We don't recognise this! raise ValueError('Unrecognised content type %s' % content_type) # Check for exceptions def _check_err(grid): try: if 'err' in grid.metadata: raise HaystackError( grid.metadata.get('dis', 'Unknown Error'), grid.metadata.get('traceback', None)) return grid except: return AsynchronousException() decoded = [_check_err(g) for g in decoded] if not self._multi_grid: decoded = decoded[0] # If we get here, then the request itself succeeded. if self._cache: with self._session._grid_lk: self._session._grid_cache[self._cache_key] = \ (None, time() + self._session._grid_expiry, decoded) self._state_machine.response_ok(result=decoded) except: # Catch all exceptions for the caller. self._log.debug('Parse fails', exc_info=1) self._state_machine.exception(result=AsynchronousException())
def _on_response(self, response): """ Process the response given back by the HTTP server. """ try: # Process the HTTP error, if any. if isinstance(response, AsynchronousException): response.reraise() # If we're expecting a raw response back, then just hand the # request object back and finish here. if self._raw_response: self._state_machine.response_ok(result=response) return # What format grid did we get back? content_type = response.content_type body = response.text if content_type in ('text/zinc', 'text/plain'): # We have been given a grid in ZINC format. decoded = hszinc.parse(body, mode=hszinc.MODE_ZINC) elif content_type == 'application/json': # We have been given a grid in JSON format. decoded = [hszinc.parse(body, mode=hszinc.MODE_JSON)] else: # We don't recognise this! raise ValueError('Unrecognised content type %s' % content_type) # Check for exceptions def _check_err(grid): try: if 'err' in grid.metadata: raise HaystackError( grid.metadata.get('dis', 'Unknown Error'), grid.metadata.get('traceback', None)) return grid except: return AsynchronousException() decoded = [_check_err(g) for g in decoded] if not self._multi_grid: decoded = decoded[0] # If we get here, then the request itself succeeded. if self._cache: with self._session._grid_lk: self._session._grid_cache[self._cache_key] = \ (None, time() + self._session._grid_expiry, decoded) self._state_machine.response_ok(result=decoded) except: # Catch all exceptions for the caller. self._log.debug('Parse fails', exc_info=1) self._state_machine.exception(result=AsynchronousException())
def test_col_meta(): grid_list = hszinc.parse('''ver:"2.0" aColumn aString:"aValue" aNumber:3.14159 aNull:N aMarker:M anotherMarker aQuantity:123Hz aDate:2016-01-13 aTime:06:44:00 aTimestamp:2016-01-13T06:44:00+10:00 Brisbane aPlace:C(-27.4725,153.003) ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 0 assert len(grid_list[0].metadata) == 0 assert list(grid_list[0].column.keys()) == ['aColumn'] meta = grid_list[0].column['aColumn'] assert list(meta.keys()) == ['aString', 'aNumber', 'aNull', 'aMarker', 'anotherMarker', 'aQuantity', 'aDate', 'aTime', 'aTimestamp', 'aPlace'] assert meta['aString'] == 'aValue' assert meta['aNumber'] == 3.14159 assert meta['aNull'] is None assert meta['aMarker'] is hszinc.MARKER assert meta['anotherMarker'] is hszinc.MARKER assert isinstance(meta['aQuantity'], hszinc.Quantity) assert meta['aQuantity'].value == 123 assert meta['aQuantity'].unit == 'Hz' assert isinstance(meta['aDate'], datetime.date) assert meta['aDate'] == datetime.date(2016,1,13) assert isinstance(meta['aTime'], datetime.time) assert meta['aTime'] == datetime.time(6,44) assert isinstance(meta['aTimestamp'], datetime.datetime) assert meta['aTimestamp'] == \ pytz.timezone('Australia/Brisbane').localize(\ datetime.datetime(2016,1,13,6,44)) assert isinstance(meta['aPlace'], hszinc.Coordinate) assert meta['aPlace'].latitude == -27.4725 assert meta['aPlace'].longitude == 153.003
def test_nodehaystack_07(): grid_list = hszinc.parse(u'''ver:"2.0" a, b -3.1kg,4kg 5%,3.2% 5kWh/ft\u00b2,-15kWh/m\u00b2 123e+12kJ/kg_dry,74\u0394\u00b0F ''') assert len(grid_list) == 1 grid = grid_list.pop(0) assert len(grid) == 4 assert len(grid.metadata) == 0 assert list(grid.column.keys()) == ['a','b'] row = grid.pop(0) assert row['a'] == hszinc.Quantity(-3.1,'kg') assert row['b'] == hszinc.Quantity(4,'kg') row = grid.pop(0) assert row['a'] == hszinc.Quantity(5,'%') assert row['b'] == hszinc.Quantity(3.2,'%') row = grid.pop(0) assert row['a'] == hszinc.Quantity(5,u'kWh/ft\u00b2') assert row['b'] == hszinc.Quantity(-15,u'kWh/m\u00b2') row = grid.pop(0) assert row['a'] == hszinc.Quantity(123e12,'kJ/kg_dry') assert row['b'] == hszinc.Quantity(74,u'\u0394\u00b0F')
def test_nodehaystack_10(): grid_list = hszinc.parse('''ver:"2.0" bg: Bin(image/jpeg) mark file1 dis:"F1" icon: Bin(image/gif),file2 icon: Bin(image/jpg) Bin(text/plain),N 4,Bin(image/png) Bin(text/html; a=foo; bar="sep"),Bin(text/html; charset=utf8) ''') assert len(grid_list) == 1 grid = grid_list.pop(0) assert len(grid) == 3 assert list(grid.metadata.keys()) == ['bg','mark'] assert grid.metadata['bg'] == hszinc.Bin('image/jpeg') assert grid.metadata['mark'] is hszinc.MARKER assert list(grid.column.keys()) == ['file1','file2'] assert list(grid.column['file1'].keys()) == ['dis','icon'] assert grid.column['file1']['dis'] == 'F1' assert grid.column['file1']['icon'] == hszinc.Bin('image/gif') assert list(grid.column['file2'].keys()) == ['icon'] assert grid.column['file2']['icon'] == hszinc.Bin('image/jpg') row = grid.pop(0) assert row['file1'] == hszinc.Bin('text/plain') assert row['file2'] is None row = grid.pop(0) assert row['file1'] == 4 assert row['file2'] == hszinc.Bin('image/png') row = grid.pop(0) assert row['file1'] == hszinc.Bin('text/html; a=foo; bar="sep"') assert row['file2'] == hszinc.Bin('text/html; charset=utf8')
def test_nodehaystack_01(): grid_list = hszinc.parse('''ver:"2.0" fooBar33 ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 0 assert len(grid_list[0].metadata) == 0 assert list(grid_list[0].column.keys()) == ['fooBar33']
def test_multi_grid(): # Multiple grids are separated by newlines. grid_list = hszinc.parse('\n'.join([ SIMPLE_EXAMPLE, METADATA_EXAMPLE, NULL_EXAMPLE])) assert len(grid_list) == 3 check_simple(grid_list[0]) check_metadata(grid_list[1]) check_null(grid_list[2])
def test_multi_grid_json_str(): # Multiple grids are separated by newlines. grid_list = hszinc.parse(list(map(json.dumps, [SIMPLE_EXAMPLE_JSON, METADATA_EXAMPLE_JSON, NULL_EXAMPLE_JSON])), mode=hszinc.MODE_JSON) assert len(grid_list) == 3 check_simple(grid_list[0]) check_metadata(grid_list[1], force_metadata_order=False) check_null(grid_list[2])
def test_nodehaystack_05(): grid_list = hszinc.parse('''ver:"2.0" a, b, c, d T, F, N, -99 2.3, -5e-10, 2.4e20, 123e-10 "", "a", "\\" \\\\ \\t \\n \\r", "\\uabcd" `path`, @12cbb082-0c02ae73, 4s, -2.5min M,R,Bin(image/png),Bin(image/png) 2009-12-31, 23:59:01, 01:02:03.123, 2009-02-03T04:05:06Z INF, -INF, "", NaN C(12,-34),C(0.123,-.789),C(84.5,-77.45),C(-90,180) ''') assert len(grid_list) == 1 grid = grid_list.pop(0) assert len(grid) == 8 assert len(grid.metadata) == 0 assert list(grid.column.keys()) == ['a','b','c','d'] row = grid.pop(0) assert row['a'] == True assert row['b'] == False assert row['c'] is None assert row['d'] == -99.0 row = grid.pop(0) assert row['a'] == 2.3 assert row['b'] == -5e-10 assert row['c'] == 2.4e20 assert row['d'] == 123e-10 row = grid.pop(0) assert row['a'] == '' assert row['b'] == 'a' assert row['c'] == '\" \\ \t \n \r' assert row['d'] == u'\uabcd' row = grid.pop(0) assert row['a'] == hszinc.Uri('path') assert row['b'] == hszinc.Ref('12cbb082-0c02ae73') assert row['c'] == hszinc.Quantity(4, 's') assert row['d'] == hszinc.Quantity(-2.5, 'min') row = grid.pop(0) assert row['a'] is hszinc.MARKER assert row['b'] is hszinc.REMOVE assert row['c'] == hszinc.Bin('image/png') assert row['d'] == hszinc.Bin('image/png') row = grid.pop(0) assert row['a'] == datetime.date(2009,12,31) assert row['b'] == datetime.time(23,59,1) assert row['c'] == datetime.time(1,2,3,123000) assert row['d'] == \ datetime.datetime(2009,2,3,4,5,6,tzinfo=pytz.utc) row = grid.pop(0) assert math.isinf(row['a']) and (row['a'] > 0) assert math.isinf(row['b']) and (row['b'] < 0) assert row['c'] == '' assert math.isnan(row['d']) row = grid.pop(0) assert row['a'] == hszinc.Coordinate(12,-34) assert row['b'] == hszinc.Coordinate(.123,-.789) assert row['c'] == hszinc.Coordinate(84.5,-77.45) assert row['d'] == hszinc.Coordinate(-90,180)
def test_uri(): grid_list = hszinc.parse('''ver:"2.0" uri `http://www.vrt.com.au` ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 1 assert grid_list[0][0]['uri'] == hszinc.Uri('http://www.vrt.com.au')
def test_coord(): grid_list = hszinc.parse('''ver:"2.0" coord C(37.55,-77.45) ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 1 assert grid_list[0][0]['coord'] == hszinc.Coordinate(37.55,-77.45)
def test_bin(): grid_list = hszinc.parse('''ver:"2.0" bin Bin(text/plain) ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 1 assert grid_list[0][0]['bin'] == hszinc.Bin('text/plain')
def test_marker_in_row(): grid_list = hszinc.parse('''ver:"2.0" str,marker "No Marker", "Marker",M ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 2 assert grid_list[0][0]['marker'] is None assert grid_list[0][1]['marker'] is hszinc.MARKER
def test_bool(): grid_list = hszinc.parse('''ver:"2.0" str,bool "True",T "False",F ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 2 assert grid_list[0][0]['bool'] == True assert grid_list[0][1]['bool'] == False
def test_nodehaystack_03(): grid_list = hszinc.parse('''ver:"2.0" val N ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 1 assert len(grid_list[0].metadata) == 0 assert list(grid_list[0].column.keys()) == ['val'] assert grid_list[0][0]['val'] is None
def test_date(): grid_list = hszinc.parse('''ver:"2.0" date 2010-03-13 ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 1 assert isinstance(grid_list[0][0]['date'], datetime.date) assert grid_list[0][0]['date'] == datetime.date(2010,3,13)
def test_nodehaystack_02(): grid_list = hszinc.parse('''ver:"2.0" tag foo:"bar" xyz "val" ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 1 assert list(grid_list[0].metadata.keys()) == ['tag', 'foo'] assert grid_list[0].metadata['tag'] is hszinc.MARKER assert grid_list[0].metadata['foo'] == 'bar' assert list(grid_list[0].column.keys()) == ['xyz'] assert grid_list[0][0]['xyz'] == 'val'
def test_ref(): grid_list = hszinc.parse('''ver:"2.0" str,ref "Basic",@a-basic-ref "With value",@reference "With value" ''') assert len(grid_list) == 1 grid = grid_list.pop(0) assert len(grid) == 2 assert grid[0]['ref'] == hszinc.Ref('a-basic-ref') assert grid[1]['ref'] == hszinc.Ref('reference', 'With value')
def test_uri_json(): grid = hszinc.parse({ 'meta': {'ver':'2.0'}, 'cols': [ {'name':'uri'}, ], 'rows': [ {'uri': 'u:http://www.vrt.com.au'}, ], }, mode=hszinc.MODE_JSON) assert len(grid) == 1 assert grid[0]['uri'] == hszinc.Uri('http://www.vrt.com.au')
def test_coord_json(): grid = hszinc.parse({ 'meta': {'ver':'2.0'}, 'cols': [ {'name':'coord'}, ], 'rows': [ {'coord': 'c:37.55,-77.45'}, ], }, mode=hszinc.MODE_JSON) assert len(grid) == 1 assert grid[0]['coord'] == hszinc.Coordinate(37.55,-77.45)
def test_datetime(): grid_list = hszinc.parse('''ver:"2.0" datetime 2010-11-28T07:23:02.500-08:00 Los_Angeles 2010-11-28T23:19:29.500+08:00 Taipei 2010-11-28T18:21:58+03:00 GMT-3 2010-11-28T12:22:27-03:00 GMT+3 2010-01-08T05:00:00Z UTC 2010-01-08T05:00:00Z ''') assert len(grid_list) == 1 check_datetime(grid_list.pop(0))
def test_time(): grid_list = hszinc.parse('''ver:"2.0" time 08:12:05 08:12:05.5 ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 2 assert isinstance(grid_list[0][0]['time'], datetime.time) assert grid_list[0][0]['time'] == datetime.time(8,12,5) assert isinstance(grid_list[0][1]['time'], datetime.time) assert grid_list[0][1]['time'] == datetime.time(8,12,5,500000)
def test_string(): grid_list = hszinc.parse('''ver:"2.0" str,strExample "Empty","" "Basic","Simple string" "Escaped","This\\tIs\\nA\\r\\"Test\\"\\\\" ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 3 assert grid_list[0][0]['strExample'] == '' assert grid_list[0][1]['strExample'] == 'Simple string' assert grid_list[0][2]['strExample'] == 'This\tIs\nA\r"Test"\\'
def test_bin_json(): grid = hszinc.parse({ 'meta': {'ver':'2.0'}, 'cols': [ {'name':'bin'}, ], 'rows': [ {'bin': 'b:text/plain'}, ], }, mode=hszinc.MODE_JSON) assert len(grid) == 1 assert grid[0]['bin'] == hszinc.Bin('text/plain')
def test_marker_in_row_json(): grid = hszinc.parse({ 'meta': {'ver':'2.0'}, 'cols': [ {'name':'str'}, {'name':'marker'}, ], 'rows': [ {'str': 'No Marker', 'marker':None}, {'str': 'Marker', 'marker':'m:'}, ], }, mode=hszinc.MODE_JSON) assert grid[0]['marker'] is None assert grid[1]['marker'] is hszinc.MARKER
def test_bool_json(): grid = hszinc.parse({ 'meta': {'ver':'2.0'}, 'cols': [ {'name':'str'}, {'name':'bool'}, ], 'rows': [ {'str': 'True', 'bool':True}, {'str': 'False','bool':False}, ], }, mode=hszinc.MODE_JSON) assert grid[0]['bool'] == True assert grid[1]['bool'] == False
def test_nodehaystack_04(): grid_list = hszinc.parse('''ver:"2.0" a,b 1,2 3,4 ''') assert len(grid_list) == 1 assert len(grid_list[0]) == 2 assert len(grid_list[0].metadata) == 0 assert list(grid_list[0].column.keys()) == ['a','b'] assert grid_list[0][0]['a'] == 1 assert grid_list[0][0]['b'] == 2 assert grid_list[0][1]['a'] == 3 assert grid_list[0][1]['b'] == 4
def test_date_json(): grid = hszinc.parse({ 'meta': {'ver':'2.0'}, 'cols': [ {'name':'date'}, ], 'rows': [ {'date': 'd:2010-03-13'}, ], }, mode=hszinc.MODE_JSON) assert len(grid) == 1 assert isinstance(grid[0]['date'], datetime.date) assert grid[0]['date'] == datetime.date(2010,3,13)
def zincToJson(req): # Parse the zinc text into grids grids = hszinc.parse(req) # Convert this back to a format that pyhaystack understands. # JSON grids have the following structure: # // JSON # { # "meta": {"ver":"2.0", "projName":"test"}, # "cols":[ # {"name":"dis", "dis":"Equip Name"}, # {"name":"equip"}, # {"name":"siteRef"}, # {"name":"installed"} # ], # "rows":[ # {"dis":"RTU-1", "equip":"m:", # "siteRef":"r:153c-699a HQ", "installed":"d:2005-06-01"}, # {"dis":"RTU-2", "equip":"m:", # "siteRef":"r:153c-699a HQ", "installed":"d:999-07-12"} # ] # } # # There is only support for one grid at a time. Out of pure laziness on my # part, this does not convert scalar values to JSON format but rather, the # equivalent Python type. if len(grids) != 1: raise ValueError('Unable to handle result with %d grids' % \ len(grids)) grid = grids[0] grid_meta = {'ver':'2.0'} grid_cols = [] grid_rows = [] json_grid = { 'meta': grid_meta, 'cols': grid_cols, 'rows': grid_rows, } grid_meta.update(grid.metadata) def _col_to_json(c): (name, meta) = c json_col = dict(meta) json_col['name'] = name return json_col grid_cols.extend(map(_col_to_json, grid.column.items())) grid_rows.extend(grid) return json_grid
def _parse_response(self, res): """ Parse the response sent back from the Haystack server. """ #decoded = '' # Referenced before assignment protection # content_type we get with nHaystack is Content_type : application/json; charset=UTF-8 content_type = res.headers['Content-Type'] if ';' in content_type: # Separate encoding from content type (content_type, encoding) = content_type.split(';',1) content_type = content_type.strip() # TODO: do we need to convert to Unicode, of so, how? if content_type in ('text/zinc', 'text/plain'): decoded = hszinc.parse(res.text, mode=hszinc.MODE_ZINC)[0] elif 'application/json' in content_type: decoded = hszinc.parse(res.text, mode=hszinc.MODE_JSON) else: raise NotImplementedError("Don't know how to parse type %s" \ % content_type) if 'err' in decoded.metadata: raise HaystackError(decoded.metadata.get('dis', 'Unknown error'), traceback=decoded.metadata.get('traceback',None)) return decoded
def test_nodehaystack_08(): grid_list = hszinc.parse('''ver:"2.0" a,b 2010-03-01T23:55:00.013-05:00 GMT+5,2010-03-01T23:55:00.013+10:00 GMT-10 ''') assert len(grid_list) == 1 grid = grid_list.pop(0) assert len(grid) == 1 assert len(grid.metadata) == 0 assert list(grid.column.keys()) == ['a','b'] row = grid.pop(0) assert row['a'] == hszinc.zoneinfo.timezone('GMT+5').localize(\ datetime.datetime(2010,3,1,23,55,0,13000)) assert row['b'] == hszinc.zoneinfo.timezone('GMT-10').localize(\ datetime.datetime(2010,3,1,23,55,0,13000))
def _try_dump_parse(ref_grid, mode): try: # Dump the randomised grid to a string grid_str = hszinc.dump(ref_grid, mode=mode) except: # Dump some detail about the grid print('Failed to dump grid.') dump_grid(ref_grid) raise # Parse the grid string try: parsed_grid = hszinc.parse(grid_str, mode=mode, single=True) except: print('Failed to parse dumped grid') dump_grid(ref_grid) print('--- Parsed string ---') print(grid_str) raise approx_check_grid(parsed_grid, ref_grid)
def test_read_many_id(self, server_session): (server, session) = server_session op = session.read([ hszinc.Ref("my.entity.id1"), hszinc.Ref("my.entity.id2"), hszinc.Ref("my.entity.id3"), ]) # The operation should still be in progress assert not op.is_done # There shall be one request assert server.requests() == 1 rq = server.next_request() # Request shall be a GET assert rq.method == "POST", "Expecting POST, got %s" % rq # Request shall be for a specific URI assert rq.uri == BASE_URI + "api/read" # Body shall be in ZINC assert rq.headers[b"Content-Type"] == "text/zinc" # Body shall be a single valid grid of this form: expected = hszinc.Grid() expected.column["id"] = {} expected.extend([ { "id": hszinc.Ref("my.entity.id1") }, { "id": hszinc.Ref("my.entity.id2") }, { "id": hszinc.Ref("my.entity.id3") }, ]) actual = hszinc.parse(rq.body.decode("utf-8"), mode=hszinc.MODE_ZINC) assert len(actual) == 1 grid_cmp(expected, actual[0]) # Accept header shall be given assert rq.headers[b"Accept"] == "text/zinc" # Make a grid to respond with expected = hszinc.Grid() expected.column["id"] = {} expected.column["dis"] = {} expected.extend([ { "id": hszinc.Ref("my.entity.id1"), "dis": "my entity 1" }, { "id": hszinc.Ref("my.entity.id2"), "dis": "my entity 2" }, { "id": hszinc.Ref("my.entity.id3"), "dis": "my entity 3" }, ]) rq.respond( status=200, headers={b"Content-Type": "text/zinc"}, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC), ) # State machine should now be done assert op.is_done actual = op.result grid_cmp(expected, actual)
def test_get_multi_entity(self, server_session): (server, session) = server_session # Try retrieving existing multiple entities op = session.get_entity(["my.entity.id1", "my.entity.id2"], single=False) # The operation should still be in progress assert not op.is_done # There shall be one request assert server.requests() == 1 rq = server.next_request() # Request shall be a POST assert rq.method == "POST", "Expecting POST, got %s" % rq # Request shall be for base + 'api/[email protected]' assert rq.uri == BASE_URI + "api/read" # Accept header shall be given assert rq.headers[b"Accept"] == "text/zinc" assert rq.headers[b"Content-Type"] == "text/zinc" # Body shall be a single grid: rq_grid = hszinc.parse(rq.body.decode("utf-8"), mode=hszinc.MODE_ZINC, single=True) # It shall have one column; id assert set(rq_grid.column.keys()) == set(["id"]) # It shall have 2 rows assert len(rq_grid) == 2 # Each row should only have 'id' values assert all([(set(r.keys()) == set(["id"])) for r in rq_grid]) # The rows' 'id' column should *only* contain Refs. assert all([isinstance(r["id"], hszinc.Ref) for r in rq_grid]) # Both IDs shall be listed, we don't about order assert set([r["id"].name for r in rq_grid ]) == set(["my.entity.id1", "my.entity.id2"]) # Make a grid to respond with response = hszinc.Grid() response.column["id"] = {} response.column["dis"] = {} response.column["randomTag"] = {} response.extend([ { "id": hszinc.Ref("my.entity.id1", value="id1"), "dis": "A test entity #1", "randomTag": hszinc.MARKER, }, { "id": hszinc.Ref("my.entity.id2", value="id2"), "dis": "A test entity #2", "randomTag": hszinc.MARKER, }, ]) rq.respond( status=200, headers={b"Content-Type": "text/zinc"}, content=hszinc.dump(response, mode=hszinc.MODE_ZINC), ) # State machine should now be done assert op.is_done entities = op.result # Response should be a dict assert isinstance(entities, dict), "%r not a dict" % entities # Response should have these keys assert set(entities.keys()) == set(["my.entity.id1", "my.entity.id2"]) entity = entities.pop("my.entity.id1") assert isinstance(entity, Entity), "%r not an entity" % entity # The tags should be passed through from the response assert entity.id.name == "my.entity.id1" assert entity.tags["dis"] == response[0]["dis"] assert entity.tags["randomTag"] == response[0]["randomTag"] entity = entities.pop("my.entity.id2") assert isinstance(entity, Entity), "%r not an entity" % entity # The tags should be passed through from the response assert entity.id.name == "my.entity.id2" assert entity.tags["dis"] == response[1]["dis"] assert entity.tags["randomTag"] == response[1]["randomTag"]
def read_file(path): with open(path, encoding="utf-8") as f: return f.read() # 16 cols, 287 rows SMALL_FILENAME = get_abspath("small_example.zinc") # 32 cols, 287 rows MEDIUM_FILENAME = get_abspath("medium_example.zinc") small_example = read_file(SMALL_FILENAME) medium_example = read_file(MEDIUM_FILENAME) print(f"parsing {SMALL_FILENAME} with zincio...") zincio_total = timeit.timeit(lambda: zincio.parse(small_example), number=20) print(f"parsing with zincio took {zincio_total / 20} seconds, avg of 20") print(f"parsing {SMALL_FILENAME} with hszinc...") hszinc_total = timeit.timeit(lambda: hszinc.parse(small_example), number=5) print(f"parsing with hszinc took {hszinc_total / 5} seconds, avg of 5") print(f"parsing {MEDIUM_FILENAME} with zincio...") zincio_total = timeit.timeit(lambda: zincio.parse(medium_example), number=20) print(f"parsing with zincio took {zincio_total / 20} seconds, avg of 20") print(f"parsing {MEDIUM_FILENAME} with hszinc...") hszinc_total = timeit.timeit(lambda: hszinc.parse(medium_example), number=3) print(f"parsing with hszinc took {hszinc_total / 3} seconds, avg of 3")
def try_dump_parse(): # Generate a randomised grid of values and try parsing it back. ref_grid = hszinc.Grid() ref_grid.metadata.extend(gen_random_meta()) # Randomised columns for n in range(0, random.randint(1, 5)): col_name = gen_random_name(existing=ref_grid.column) if random.choice([True, False]): ref_grid.column[col_name] = gen_random_meta() else: ref_grid.column[col_name] = {} # Randomised rows for n in range(0, random.randint(0, 20)): row = {} for c in ref_grid.column.keys(): if random.choice([True, False]): row[c] = gen_random_scalar() ref_grid.append(row) try: # Dump the randomised grid to a string grid_str = hszinc.dump(ref_grid) except: # Dump some detail about the grid print('Failed to dump grid.') dump_grid(ref_grid) raise # Parse the grid string try: grid_list = hszinc.parse(grid_str) except: print('Failed to parse dumped grid') dump_grid(ref_grid) print('--- Parsed string ---') print(grid_str) raise assert len(grid_list) == 1 parsed_grid = grid_list.pop(0) # Check metadata matches try: assert list(ref_grid.metadata.keys()) \ == list(parsed_grid.metadata.keys()) for key in ref_grid.metadata.keys(): approx_check(ref_grid.metadata[key], parsed_grid.metadata[key]) except: print('Mismatch in metadata') print('Reference grid') dump_grid(ref_grid) print('Parsed grid') dump_grid(parsed_grid) raise try: # Check column matches assert list(ref_grid.column.keys()) \ == list(parsed_grid.column.keys()) except: print('Mismatch in column') print('Reference grid') dump_grid(ref_grid) print('Parsed grid') dump_grid(parsed_grid) raise for col in ref_grid.column.keys(): try: for key in ref_grid.column[col].keys(): approx_check(ref_grid.column[col][key], \ parsed_grid.column[col][key]) except: print('Mismatch in metadata for column %s' % col) print('Reference: %r' % ref_grid.column[col]) print('Parsed: %r' % parsed_grid.column[col]) raise try: # Check row matches assert len(ref_grid) == len(parsed_grid) except: print('Mismatch in row count') print('Reference grid') dump_grid(ref_grid) print('Parsed grid') dump_grid(parsed_grid) for (ref_row, parsed_row) in zip(ref_grid, parsed_grid): try: for col in ref_grid.column.keys(): approx_check(ref_row.get(col), parsed_row.get(col)) except: print('Mismatch in row') print('Reference:') print(ref_row) print('Parsed:') print(parsed_row) raise
def _on_response(self, response): """ Process the response given back by the HTTP server. """ # When problems occur : # print("RESPONSE", response.__dict__) try: # Does the session want to invoke any relevant hooks? # This allows a session to detect problems in the session and # abort the operation. if hasattr(self._session, "_on_http_grid_response"): self._session._on_http_grid_response(response) # Process the HTTP error, if any. if isinstance(response, AsynchronousException): response.reraise() # If we're expecting a raw response back, then just hand the # request object back and finish here. if self._raw_response: self._state_machine.response_ok(result=response) return # What format grid did we get back? content_type = response.content_type body = response.text if content_type in ("text/zinc", "text/plain"): # We have been given a grid in ZINC format. decoded = hszinc.parse(body, mode=hszinc.MODE_ZINC, single=False) elif content_type == "application/json": # We have been given a grid in JSON format. decoded = hszinc.parse(body, mode=hszinc.MODE_JSON, single=False) elif content_type in ("text/html"): # We probably fell back to a login screen after auto logoff. self._state_machine.exception(AsynchronousException()) else: # We don't recognise this! raise ValueError("Unrecognised content type %s" % content_type) # Check for exceptions def _check_err(grid): try: if "err" in grid.metadata: raise HaystackError( grid.metadata.get("dis", "Unknown Error"), grid.metadata.get("traceback", None), ) return grid except: return AsynchronousException() decoded = [_check_err(g) for g in decoded] if not self._multi_grid: decoded = decoded[0] # If we get here, then the request itself succeeded. if self._cache: with self._session._grid_lk: self._session._grid_cache[self._cache_key] = ( None, time() + self._session._grid_expiry, decoded, ) self._state_machine.response_ok(result=decoded) except: # Catch all exceptions for the caller. self._log.debug("Parse fails", exc_info=1) self._state_machine.exception(result=AsynchronousException())
def test_read_many_id(self, server_session): (server, session) = server_session op = session.read([ hszinc.Ref('my.entity.id1'), hszinc.Ref('my.entity.id2'), hszinc.Ref('my.entity.id3'), ]) # The operation should still be in progress assert not op.is_done # There shall be one request assert server.requests() == 1 rq = server.next_request() # Request shall be a GET assert rq.method == 'POST', 'Expecting POST, got %s' % rq # Request shall be for a specific URI assert rq.uri == BASE_URI + 'api/read' # Body shall be in ZINC assert rq.headers['Content-Type'] == 'text/zinc' # Body shall be a single valid grid of this form: expected = hszinc.Grid() expected.column['id'] = {} expected.extend([{ "id": hszinc.Ref('my.entity.id1'), }, { "id": hszinc.Ref('my.entity.id2'), }, { "id": hszinc.Ref('my.entity.id3'), }]) actual = hszinc.parse(rq.body, mode=hszinc.MODE_ZINC) assert len(actual) == 1 grid_cmp(expected, actual[0]) # Accept header shall be given assert rq.headers['Accept'] == 'text/zinc' # Make a grid to respond with expected = hszinc.Grid() expected.column['id'] = {} expected.column['dis'] = {} expected.extend([{ "id": hszinc.Ref('my.entity.id1'), "dis": 'my entity 1' }, { "id": hszinc.Ref('my.entity.id2'), "dis": 'my entity 2' }, { "id": hszinc.Ref('my.entity.id3'), "dis": 'my entity 3' }]) rq.respond(status=200, headers={ 'Content-Type': 'text/zinc', }, content=hszinc.dump(expected, mode=hszinc.MODE_ZINC)) # State machine should now be done assert op.is_done actual = op.result grid_cmp(expected, actual)
def test_get_multi_entity(self, server_session): (server, session) = server_session # Try retrieving existing multiple entities op = session.get_entity(['my.entity.id1', 'my.entity.id2'], single=False) # The operation should still be in progress assert not op.is_done # There shall be one request assert server.requests() == 1 rq = server.next_request() # Request shall be a POST assert rq.method == 'POST', 'Expecting POST, got %s' % rq # Request shall be for base + 'api/[email protected]' assert rq.uri == BASE_URI + 'api/read' # Accept header shall be given assert rq.headers[b'Accept'] == 'text/zinc' assert rq.headers[b'Content-Type'] == 'text/zinc' # Body shall be a single grid: rq_grid = hszinc.parse(rq.body.decode('utf-8'), mode=hszinc.MODE_ZINC) assert len(rq_grid) == 1 rq_grid = rq_grid[0] # It shall have one column; id assert set(rq_grid.column.keys()) == set(['id']) # It shall have 2 rows assert len(rq_grid) == 2 # Each row should only have 'id' values assert all([(set(r.keys()) == set(['id'])) for r in rq_grid]) # The rows' 'id' column should *only* contain Refs. assert all([isinstance(r['id'], hszinc.Ref) for r in rq_grid]) # Both IDs shall be listed, we don't about order assert set([r['id'].name for r in rq_grid]) \ == set(['my.entity.id1', 'my.entity.id2']) # Make a grid to respond with response = hszinc.Grid() response.column['id'] = {} response.column['dis'] = {} response.column['randomTag'] = {} response.extend([{ 'id': hszinc.Ref('my.entity.id1', value='id1'), 'dis': 'A test entity #1', 'randomTag': hszinc.MARKER }, { 'id': hszinc.Ref('my.entity.id2', value='id2'), 'dis': 'A test entity #2', 'randomTag': hszinc.MARKER }]) rq.respond(status=200, headers={ b'Content-Type': 'text/zinc', }, content=hszinc.dump(response, mode=hszinc.MODE_ZINC)) # State machine should now be done assert op.is_done entities = op.result # Response should be a dict assert isinstance(entities, dict), '%r not a dict' % entity # Response should have these keys assert set(entities.keys()) == set(['my.entity.id1', 'my.entity.id2']) entity = entities.pop('my.entity.id1') assert isinstance(entity, Entity), '%r not an entity' % entity # The tags should be passed through from the response assert entity.id.name == 'my.entity.id1' assert entity.tags['dis'] == response[0]['dis'] assert entity.tags['randomTag'] == response[0]['randomTag'] entity = entities.pop('my.entity.id2') assert isinstance(entity, Entity), '%r not an entity' % entity # The tags should be passed through from the response assert entity.id.name == 'my.entity.id2' assert entity.tags['dis'] == response[1]['dis'] assert entity.tags['randomTag'] == response[1]['randomTag']