def test_scalar_unknown(): try: hszinc.dump_scalar(hszinc.VER_2_0, mode=hszinc.MODE_ZINC, version=hszinc.VER_2_0) assert False, 'Serialised a list in Haystack v2.0' except NotImplementedError: pass
def test_scalar_json_ver(): # Test that versions are respected. try: hszinc.dump_scalar(["a list is not allowed in v2.0"], mode=hszinc.MODE_JSON, version=hszinc.VER_2_0) assert False, 'Serialised a list in Haystack v2.0' except ValueError: pass
def test_scalar_na_json_ver(): # Test that versions are respected. try: hszinc.dump_scalar(hszinc.NA, mode=hszinc.MODE_JSON, version=hszinc.VER_2_0) assert False, 'Serialised a NA in Haystack v2.0' except ValueError: pass
def test_scalar_dict_zinc_ver(): # Test that versions are respected. try: hszinc.dump_scalar({"a": "b"}, mode=hszinc.MODE_ZINC, version=hszinc.VER_2_0) assert False, 'Serialised a list in Haystack v2.0' except ValueError: pass
def __init__(self, session, point, rng, tz, series_format): """ Read the series data and return it. :param session: Haystack HTTP session object. :param point: ID of historical 'point' object to read. :param rng: Range to read from 'point' :param tz: Timezone to translate timezones to. May be None. :param series_format: What format to present the series in. """ super(HisReadSeriesOperation, self).__init__() if series_format not in ( self.FORMAT_LIST, self.FORMAT_DICT, self.FORMAT_SERIES, ): raise ValueError("Unrecognised series_format %s" % series_format) if (series_format == self.FORMAT_SERIES) and (not HAVE_PANDAS): raise NotImplementedError("pandas not available.") if isinstance(rng, slice): rng = ",".join([ hszinc.dump_scalar(p, mode=hszinc.MODE_ZINC) for p in (rng.start, rng.stop) ]) self._session = session self._point = point self._range = hszinc.dump_scalar(rng, mode=hszinc.MODE_ZINC) self._tz = _resolve_tz(tz) self._series_format = series_format self._state_machine = fysom.Fysom( initial="init", final="done", events=[ # Event Current State New State ("go", "init", "read"), ("read_done", "read", "done"), ("exception", "*", "done"), ], callbacks={ "onenterread": self._do_read, "onenterdone": self._do_done }, )
def _on_his_read(self, point, rng, callback, **kwargs): if isinstance(rng, slice): str_rng = ",".join([hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) elif not isinstance(rng, string_types): str_rng = hszinc.dump_scalar(rng) else: # Better be valid! # str_rng = rng str_rng = hszinc.dump_scalar(rng, mode=hszinc.MODE_ZINC) return self._get_grid( "hisRead", callback, args={"id": self._obj_to_ref(point), "range": str_rng}, **kwargs )
def __init__(self, session, point, rng, tz, series_format): """ Read the series data and return it. :param session: Haystack HTTP session object. :param point: ID of historical 'point' object to read. :param rng: Range to read from 'point' :param tz: Timezone to translate timezones to. May be None. :param series_format: What format to present the series in. """ super(HisReadSeriesOperation, self).__init__() if series_format not in (self.FORMAT_LIST, self.FORMAT_DICT, self.FORMAT_SERIES): raise ValueError("Unrecognised series_format %s" % series_format) if (series_format == self.FORMAT_SERIES) and (not HAVE_PANDAS): raise NotImplementedError("pandas not available.") self._session = session self._point = point self._range = hszinc.dump_scalar(rng, mode=hszinc.MODE_ZINC) self._tz = _resolve_tz(tz) self._series_format = series_format self._state_machine = fysom.Fysom( initial="init", final="done", events=[ # Event Current State New State ("go", "init", "read"), ("read_done", "read", "done"), ("exception", "*", "done"), ], callbacks={"onenterread": self._do_read, "onenterdone": self._do_done}, )
def __init__(self, session, uri, args=None, expect_format=hszinc.MODE_ZINC, multi_grid=False, raw_response=False, retries=2, cache=False, cache_key=None, accept_status=None): """ Initialise a request for the grid with the given URI and arguments. :param session: Haystack HTTP session object. :param uri: Possibly partial URI relative to the server base address to perform a query. No arguments shall be given here. :param expect_format: Request that the grid be sent in the given format. :param args: Dictionary of key-value pairs to be given as arguments. :param multi_grid: Boolean indicating if we are to expect multiple grids or not. If True, then the operation will _always_ return a list, otherwise, it will _always_ return a single grid. :param raw_response: Boolean indicating if we should try to parse the result. If True, then we should just pass back the raw HTTPResponse object. :param retries: Number of retries permitted in case of failure. :param cache: Whether or not to cache this result. If True, the result is cached by the session object. :param cache_key: Name of the key to use when the object is cached. :param accept_status: What status codes to accept, in addition to the usual ones? """ super(BaseGridOperation, self).__init__(session, uri) if args is not None: # Convert scalars to strings args = dict([(param, hszinc.dump_scalar(value) \ if not isinstance(value, string_types) \ else value) for param, value in args.items()]) self._retries = retries self._session = session self._multi_grid = multi_grid self._uri = uri self._args = args self._expect_format = expect_format self._raw_response = raw_response self._headers = {} self._accept_status = accept_status self._cache = cache if cache and (cache_key is None): cache_key = uri self._cache_key = cache_key if not raw_response: if expect_format == hszinc.MODE_ZINC: self._headers[b'Accept'] = 'text/zinc' elif expect_format == hszinc.MODE_JSON: self._headers[b'Accept'] = 'application/json' elif expect_format is not None: raise ValueError( 'expect_format must be one onf hszinc.MODE_ZINC '\ 'or hszinc.MODE_JSON')
def _on_his_read(self, point, rng, callback, **kwargs): if isinstance(rng, slice): str_rng = ','.join( [hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) elif not isinstance(rng, string_types): str_rng = hszinc.dump_scalar(rng) else: # Better be valid! str_rng = rng return self._get_grid('hisRead', callback, args={ 'id': self._obj_to_ref(point), 'range': str_rng, }, **kwargs)
def __init__(self, session, columns, rng, tz, frame_format): """ Read the series data and return it. :param session: Haystack HTTP session object. :param columns: IDs of historical point objects to read. :param rng: Range to read from 'point' :param tz: Timezone to translate timezones to. May be None. :param frame_format: What format to present the frame in. """ super(HisReadFrameOperation, self).__init__() self._log = session._log.getChild('his_read_frame') if frame_format not in (self.FORMAT_LIST, self.FORMAT_DICT, self.FORMAT_FRAME): raise ValueError('Unrecognised frame_format %s' % frame_format) if (frame_format == self.FORMAT_FRAME) and (not HAVE_PANDAS): raise NotImplementedError('pandas not available.') # Convert the columns to a list of tuples. strip_ref = lambda r: r.name if isinstance(r, hszinc.Ref) else r if isinstance(columns, dict): # Ensure all are strings to references columns = [(str(c), strip_ref(r)) for c, r in columns.items()] else: # Translate to a dict: columns = [(strip_ref(c), c) for c in columns] self._session = session self._columns = columns self._range = hszinc.dump_scalar(rng, mode=hszinc.MODE_ZINC) self._tz = _resolve_tz(tz) self._frame_format = frame_format self._data_by_ts = {} self._todo = set([c[0] for c in columns]) self._state_machine = fysom.Fysom( initial='init', final='done', events=[ # Event Current State New State ('probe_multi', 'init', 'probing'), ('do_multi_read', 'probing', 'multi_read'), ('all_read_done', 'multi_read', 'postprocess'), ('do_single_read', 'probing', 'single_read'), ('all_read_done', 'single_read', 'postprocess'), ('process_done', 'postprocess', 'done'), ('exception', '*', 'done'), ], callbacks={ 'onenterprobing': self._do_probe_multi, 'onentermulti_read': self._do_multi_read, 'onentersingle_read': self._do_single_read, 'onenterpostprocess': self._do_postprocess, 'onenterdone': self._do_done, })
def multi_his_read(self, points, rng, callback=None): """ Read the historical data for multiple points. This processes each point given by the list points and returns the data from that point in a numbered column named idN where N starts counting from zero. """ if isinstance(rng, slice): str_rng = ",".join([hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) elif not isinstance(rng, string_types): str_rng = hszinc.dump_scalar(rng) else: # Better be valid! str_rng = rng args = {"range": str_rng} for (col, point) in enumerate(points): args["id%d" % col] = self._obj_to_ref(point) return self._get_grid("hisRead", callback, args=args)
def __init__(self, session, columns, rng, tz, frame_format): """ Read the series data and return it. :param session: Haystack HTTP session object. :param columns: IDs of historical point objects to read. :param rng: Range to read from 'point' :param tz: Timezone to translate timezones to. May be None. :param frame_format: What format to present the frame in. """ super(HisReadFrameOperation, self).__init__() self._log = session._log.getChild('his_read_frame') if frame_format not in (self.FORMAT_LIST, self.FORMAT_DICT, self.FORMAT_FRAME): raise ValueError('Unrecognised frame_format %s' % frame_format) if (frame_format == self.FORMAT_FRAME) and (not HAVE_PANDAS): raise NotImplementedError('pandas not available.') # Convert the columns to a list of tuples. strip_ref = lambda r : r.name if isinstance(r, hszinc.Ref) else r if isinstance(columns, dict): # Ensure all are strings to references columns = [(str(c),strip_ref(r)) for c, r in columns.items()] else: # Translate to a dict: columns = [(strip_ref(c), c) for c in columns] self._session = session self._columns = columns self._range = hszinc.dump_scalar(rng, mode=hszinc.MODE_ZINC) self._tz = _resolve_tz(tz) self._frame_format = frame_format self._data_by_ts = {} self._todo = set([c[0] for c in columns]) self._state_machine = fysom.Fysom( initial='init', final='done', events=[ # Event Current State New State ('probe_multi', 'init', 'probing'), ('do_multi_read', 'probing', 'multi_read'), ('all_read_done', 'multi_read', 'postprocess'), ('do_single_read', 'probing', 'single_read'), ('all_read_done', 'single_read', 'postprocess'), ('process_done', 'postprocess', 'done'), ('exception', '*', 'done'), ], callbacks={ 'onenterprobing': self._do_probe_multi, 'onentermulti_read': self._do_multi_read, 'onentersingle_read': self._do_single_read, 'onenterpostprocess': self._do_postprocess, 'onenterdone': self._do_done, })
def find_entity(self, filter_expr=None, limit=None, single=False, callback=None): """ Retrieve the entities that are linked to this equipment. This is a convenience around the session find_entity method. """ equip_ref = hszinc.dump_scalar(self.id) if filter_expr is None: filter_expr = "equipRef==%s" % equip_ref else: filter_expr = "(equipRef==%s) and (%s)" % (equip_ref, filter_expr) return self._session.find_entity(filter_expr, limit, single, callback)
def _refresh_meta(self, force=False): """ Retrieve the metadata from the server if out of date. """ if not (force or self._refresh_due): # We're not being forced and the refresh isn't due yet. return meta = self._session._get_grid('read', \ id=hszinc.dump_scalar(hszinc.Ref(self._point_id))) self._load_meta(meta[0])
def multi_his_read(self, points, rng, callback=None): """ Read the historical data for multiple points. This processes each point given by the list points and returns the data from that point in a numbered column named idN where N starts counting from zero. """ if isinstance(rng, slice): str_rng = ",".join( [hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) elif not isinstance(rng, string_types): str_rng = hszinc.dump_scalar(rng) else: # Better be valid! str_rng = rng args = {"range": str_rng} for (col, point) in enumerate(points): args["id%d" % col] = self._obj_to_ref(point) return self._get_grid("hisRead", callback, args=args)
def find_entity(self, filter_expr=None, single=False, limit=None, callback=None): """ Retrieve the entities that are linked to this site. This is a convenience around the session find_entity method. """ site_ref = hszinc.dump_scalar(self.id) if filter_expr is None: filter_expr = 'siteRef==%s' % site_ref else: filter_expr = '(siteRef==%s) and (%s)' % (site_ref, filter_expr) return self._session.find_entity(filter_expr, limit, single, callback)
def his_read(self, point, rng, callback=None): """ point is either the ID of the historical point entity, or an instance of the historical point entity to read historical from. rng is either a string describing a time range (e.g. "today", "yesterday"), a datetime.date object (providing all samples on the nominated day), a datetime.datetime (providing all samples since the nominated time) or a slice of datetime.dates or datetime.datetimes. """ if isinstance(rng, slice): str_rng = ','.join([hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) elif not isinstance(rng, string_types): str_rng = hszinc.dump_scalar(rng) else: # Better be valid! str_rng = rng return self._get_grid('hisRead', callback, args={ 'id': self._obj_to_ref(point), 'range': str_rng, })
def _on_his_read(self, point, rng, callback, **kwargs): """ Skyspark will not accept GET request for his_read by default [ref : https://project-haystack.org/forum/topic/787#c6] The default behavior of SkySpark is now to disallow GET requests non-idempotent operations. So its still allowed on certain operations such as about, formats, read. However as Chris said it can be toggled back on using Settings|API for backward compatibility. However as a recommendation I think we should always be using POST as a safer alternative. Using GET for ops with side-effects is against the HTTP spec. Plus it is an attack vector if cookies are involved. And it provides a more precise way to pass the request payload. Its not really from a theoretical perspective. But in SkySpark we allow customers to generate histories using their own custom functions. So from a security perspective we took the safest route and consider it to potentially have side effects. If your code is all using GET, then just have the customer set Settings|API allowGetWithSideEffects flag to false and it should all work. """ if isinstance(rng, slice): str_rng = ",".join( [hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) elif not isinstance(rng, string_types): str_rng = hszinc.dump_scalar(rng) else: # No conversion here as this will be added to the grid as-is str_rng = rng his_grid = hszinc.Grid() his_grid.metadata["id"] = self._obj_to_ref(point) his_grid.column["id"] = {} his_grid.column["range"] = {} his_grid.append({"id": self._obj_to_ref(point), "range": str_rng}) return self._post_grid("hisRead", his_grid, callback, **kwargs)
def his_read(self, point, rng, callback=None): """ point is either the ID of the historical point entity, or an instance of the historical point entity to read historical from. rng is either a string describing a time range (e.g. "today", "yesterday"), a datetime.date object (providing all samples on the nominated day), a datetime.datetime (providing all samples since the nominated time) or a slice of datetime.dates or datetime.datetimes. """ if isinstance(rng, slice): str_rng = ','.join( [hszinc.dump_scalar(p) for p in (rng.start, rng.stop)]) elif not isinstance(rng, string_types): str_rng = hszinc.dump_scalar(rng) else: # Better be valid! str_rng = rng return self._get_grid('hisRead', callback, args={ 'id': self._obj_to_ref(point), 'range': str_rng, })
def __init__(self, session, point, rng, tz, series_format): """ Read the series data and return it. :param session: Haystack HTTP session object. :param point: ID of historical 'point' object to read. :param rng: Range to read from 'point' :param tz: Timezone to translate timezones to. May be None. :param series_format: What format to present the series in. """ super(HisReadSeriesOperation, self).__init__() if series_format not in (self.FORMAT_LIST, self.FORMAT_DICT, self.FORMAT_SERIES): raise ValueError('Unrecognised series_format %s' % series_format) if (series_format == self.FORMAT_SERIES) and (not HAVE_PANDAS): raise NotImplementedError('pandas not available.') self._session = session self._point = point self._range = hszinc.dump_scalar(rng, mode=hszinc.MODE_ZINC) self._tz = _resolve_tz(tz) self._series_format = series_format self._state_machine = fysom.Fysom( initial='init', final='done', events=[ # Event Current State New State ('go', 'init', 'read'), ('read_done', 'read', 'done'), ('exception', '*', 'done'), ], callbacks={ 'onenterread': self._do_read, 'onenterdone': self._do_done, })
def __str__(self): return hszinc.dump_scalar(self.value, mode=hszinc.MODE_ZINC)
def __init__(self, session, uri, args=None, expect_format=hszinc.MODE_ZINC, multi_grid=False, raw_response=False, retries=2, cache=False, cache_key=None): """ Initialise a request for the grid with the given URI and arguments. :param session: Haystack HTTP session object. :param uri: Possibly partial URI relative to the server base address to perform a query. No arguments shall be given here. :param expect_format: Request that the grid be sent in the given format. :param args: Dictionary of key-value pairs to be given as arguments. :param multi_grid: Boolean indicating if we are to expect multiple grids or not. If True, then the operation will _always_ return a list, otherwise, it will _always_ return a single grid. :param raw_response: Boolean indicating if we should try to parse the result. If True, then we should just pass back the raw HTTPResponse object. :param retries: Number of retries permitted in case of failure. :param cache: Whether or not to cache this result. If True, the result is cached by the session object. :param cache_key: Name of the key to use when the object is cached. """ super(BaseGridOperation, self).__init__() if args is not None: # Convert scalars to strings args = dict([(param, hszinc.dump_scalar(value) \ if not isinstance(value, string_types) \ else value) for param, value in args.items()]) self._retries = retries self._session = session self._multi_grid = multi_grid self._uri = uri self._args = args self._expect_format = expect_format self._raw_response = raw_response self._headers = {} self._cache = cache if cache and (cache_key is None): cache_key = uri self._cache_key = cache_key if not raw_response: if expect_format == hszinc.MODE_ZINC: self._headers[b'Accept'] = 'text/zinc' elif expect_format == hszinc.MODE_JSON: self._headers[b'Accept'] = 'application/json' elif expect_format is not None: raise ValueError( 'expect_format must be one onf hszinc.MODE_ZINC '\ 'or hszinc.MODE_JSON') self._state_machine = fysom.Fysom( initial='init', final='done', events=[ # Event Current State New State ('auth_ok', 'init', 'check_cache'), ('auth_not_ok', 'init', 'auth_attempt'), ('auth_ok', 'auth_attempt', 'check_cache'), ('auth_not_ok', 'auth_attempt', 'auth_failed'), ('auth_failed', 'auth_attempt', 'done'), ('cache_hit', 'check_cache', 'done'), ('cache_miss', 'check_cache', 'submit'), ('response_ok', 'submit', 'done'), ('exception', '*', 'failed'), ('retry', 'failed', 'init'), ('abort', 'failed', 'done'), ], callbacks={ 'onretry': self._check_auth, 'onenterauth_attempt': self._do_auth_attempt, 'onenterauth_failed': self._do_auth_failed, 'onentercheck_cache': self._do_check_cache, 'onentersubmit': self._do_submit, 'onenterfailed': self._do_fail_retry, 'onenterdone': self._do_done, })
def __getitem__(self, point_ids): """ Get a specific named point or list of points. If a single point is requested, the return type is that point, or None if not found. If a list is given, a dict is returned with the located IDs as keys. """ log = self._log.getChild('getitem') multi = isinstance(point_ids, list) if not multi: log.debug('Retrieving single point %s', point_ids) point_ids = [point_ids] elif not bool(point_ids): log.debug('No points to retrieve') return {} # Locate items that already exist. found = {} for point_id in point_ids: try: point = self._point[point_id] except KeyError: # It doesn't exist. log.debug('Not yet retrieved point %s', point_id) continue # Is the point due for refresh? if point._refresh_due: # Pretend it doesn't exist. log.debug('Stale point %s', point_id) continue log.debug('Existing point %s', point_id) found[point_id] = point # Get a list of points that need fetching to_fetch = filter(lambda pid: pid not in found, point_ids) log.debug('Need to retrieve points %s', to_fetch) if bool(to_fetch): if len(to_fetch) > 1: # Make a request grid and POST it grid = hszinc.Grid() grid.column['id'] = {} grid.extend([{ 'id': hszinc.Ref(point_id) } for point_id in to_fetch]) res = self._post_grid_rq('read', grid) else: # Make a GET request res = self._get_grid('read', id=hszinc.dump_scalar( hszinc.Ref(to_fetch[0]))) found.update(self._get_points_from_grid(res)) log.debug('Retrieved %s', list(found.keys())) if not multi: return found.get(point_ids[0]) else: return found
def test_scalar_json(): # No need to be exhaustive, the underlying function is tested heavily by # the grid dump tests. assert hszinc.dump_scalar(hszinc.Ref('areference', 'a display name'), mode=hszinc.MODE_JSON) == 'r:areference a display name'
def __getitem__(self, point_ids): """ Get a specific named point or list of points. If a single point is requested, the return type is that point, or None if not found. If a list is given, a dict is returned with the located IDs as keys. """ log = self._log.getChild('getitem') multi = isinstance(point_ids, list) if not multi: log.debug('Retrieving single point %s', point_ids) point_ids = [point_ids] elif not bool(point_ids): log.debug('No points to retrieve') return {} # Locate items that already exist. found = {} for point_id in point_ids: try: point = self._point[point_id] except KeyError: # It doesn't exist. log.debug('Not yet retrieved point %s', point_id) continue # Is the point due for refresh? if point._refresh_due: # Pretend it doesn't exist. log.debug('Stale point %s', point_id) continue log.debug('Existing point %s', point_id) found[point_id] = point # Get a list of points that need fetching to_fetch = filter(lambda pid : pid not in found, point_ids) log.debug('Need to retrieve points %s', to_fetch) if bool(to_fetch): if len(to_fetch) > 1: # Make a request grid and POST it grid = hszinc.Grid() grid.column['id'] = {} grid.extend([{'id': hszinc.Ref(point_id)} for point_id in to_fetch]) res = self._post_grid_rq('read', grid) else: # Make a GET request res = self._get_grid('read', id=hszinc.dump_scalar(hszinc.Ref(to_fetch[0]))) found.update(self._get_points_from_grid(res)) log.debug('Retrieved %s', list(found.keys())) if not multi: return found.get(point_ids[0]) else: return found