def test_filter_and_limit(): grid = Grid(columns={'id': {}, 'site': {}}) grid.append({ 'id': Ref('id1'), 'site': MARKER, 'equip': 'Chicago', 'geoPostalCode': "78280", 'ahu': MARKER, 'geoCity': 'Chicago', 'curVal': 76 }) grid.append({ 'id': Ref('id2'), 'hvac': MARKER, 'geoPostalCode': "23220", 'curVal': 75 }) grid.append({ 'equip': 'Chicago', 'hvac': MARKER, 'siteRef': Ref('id1'), 'curVal': 74 }) assert len(grid.filter('not acme', limit=1)) == 1
def test_invoke_action_with_zinc(mock) -> None: # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = ping._PingGrid mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = shaystack.Grid(metadata={ 'id': Ref('123'), 'action': 'doIt' }, columns={ 'key': {}, 'value': {} }) grid.append({'param': 'value'}) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=shaystack.MODE_ZINC) # WHEN response = shaystack.invoke_action(envs, request, "dev") # THEN mock.assert_called_once_with(Ref("123"), "doIt", {}) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, shaystack.MODE_ZINC) is not None
def test_his_read_with_range_today(mock) -> None: # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = ping._PingGrid mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = shaystack.Grid(columns=['id', 'range']) grid.append({"id": Ref("1234"), "range": "today"}) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.his_read(envs, request, "dev") # THEN today = datetime.combine(date.today(), datetime.min.time()).replace(tzinfo=get_localzone()) mock.assert_called_once_with(Ref("1234"), (today, today + timedelta(days=1) ), None) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, mime_type) is not None
def test_his_write_with_args(mock): # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} time_serie = [ (datetime(2020, 1, 1, tzinfo=pytz.utc).isoformat() + " UTC", 100), (datetime(2020, 1, 2, tzinfo=pytz.utc).isoformat() + " UTC", 200)] mock.return_value = ping._PingGrid mime_type = DEFAULT_MIME_TYPE request = HaystackHttpRequest() request.args['id'] = str(Ref("1234")) request.args['ts'] = str(time_serie) # WHEN response = shaystack.his_write(envs, request, "dev") # THEN result_ts = Grid(version=VER_3_0, columns=["date", "val"]) result_ts.extend([{"date": parse_hs_datetime_format(d, pytz.UTC), "val": v} for d, v in time_serie]) mock.assert_called_once_with(Ref("1234"), result_ts, None) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, mime_type) is not None
def test_watch_unsub_with_zinc(mock) -> None: # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = ping._PingGrid mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = shaystack.Grid(metadata={"close": True, "watchId": "0123456789ABCDEF"}, columns=['id']) grid.append({"id": Ref("id1")}) grid.append({"id": Ref("id2")}) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=shaystack.MODE_ZINC) # WHEN response = shaystack.watch_unsub(envs, request, "dev") # THEN mock.assert_called_once_with("0123456789ABCDEF", [Ref("id1"), Ref("id2")], True) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, shaystack.MODE_ZINC) is not None
def test_update_grid(): try: envs = {'HAYSTACK_DB': HAYSTACK_DB} with cast(MongoProvider, get_provider("shaystack.providers.mongodb", envs)) as provider: provider.purge_db() provider.create_db() grid = Grid(metadata={"dis": "hello"}, columns=[("id", {}), ("a", { "dis": "a" }), ("b", { "dis": "b" })]) grid.append({"id": Ref("1"), "a": "a", "b": "b"}) grid.append({"id": Ref("2"), "a": "c", "b": "d"}) provider.update_grid(grid, None, "customer", FAKE_NOW) in_table = [ _conv_row_to_entity(row['entity']) for row in provider.get_collection().find() ] assert len(in_table) == len(grid) assert in_table[0] == grid[0] assert in_table[1] == grid[1] except ServerSelectionTimeoutError as ex: raise SkipTest("Mongo db not started") from ex
def test_pop_key(): grid = Grid(columns=["id", "a"]) row = {"id": Ref("myid"), "a": 1, "b": 2} grid.append(row) old = grid.pop(Ref("myid")) assert not grid assert id(old) == id(row)
def test_point_write_write_with_zinc(mock) -> None: # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = Grid(version=VER_3_0, columns=["level", "levelDis", "val", "who"]) mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = shaystack.Grid(columns=['id', "level", "val", "who", "duration"]) grid.append({ "id": Ref("1234"), "level": 1, "val": 100.0, "who": "PPR", "duration": Quantity(1, "min") }) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.point_write(envs, request, "dev") # THEN mock.assert_called_once_with(Ref("1234"), 1, 100, "PPR", Quantity(1, "min"), None) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, mime_type) is not None
def test_read_with_zinc_and_id(mock) -> None: # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = ping._PingGrid mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = shaystack.Grid(columns=['id']) grid.append({"id": Ref("me")}) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.read(envs, request, "dev") # THEN ids = [Ref("me")] mock.assert_called_once_with(0, None, ids, '', None) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, mime_type) is not None
def test_his_read_with_range_one_datetime(mock) -> None: # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = ping._PingGrid mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = shaystack.Grid(columns=['id', 'range']) datetime_1 = datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC) grid.append({"id": Ref("1234"), "range": datetime_1.isoformat()}) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.his_read(envs, request, "dev") # THEN cur_datetime = datetime.combine(datetime_1, datetime.min.time()).replace(tzinfo=pytz.UTC) mock.assert_called_once_with(Ref("1234"), (cur_datetime, datetime.max.replace(tzinfo=pytz.UTC) ), None) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, mime_type) is not None
def test_his_read_with_zinc(mock, no_cache) -> None: # GIVEN """ Args: mock: no_cache: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = ping._PingGrid no_cache.return_value = True mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = shaystack.Grid(columns={'id': {}}) grid.append({"id": Ref("1234")}) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.his_read(envs, request, "dev") # THEN mock.assert_called_once_with(Ref("1234"), (datetime.min.replace(tzinfo=pytz.UTC), datetime.max.replace(tzinfo=pytz.UTC)), None) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, mime_type) is not None
def test_pop_invalid_key(): grid = Grid(columns=["id", "a"]) row = {"id": Ref("myid"), "a": 1, "b": 2} grid.append(row) old = grid.pop(Ref("other_id")) assert grid assert not old
def test_grid_getitem_with_id(): grid = Grid() grid.column['id'] = {} grid.column['test'] = {} row = {'id': Ref('myid'), 'test': 'This is a test'} grid.append(row) assert grid[Ref('myid')] is row
def test_grid_extend(): grid = Grid(columns=['id']) grid.reindex() rows = [{'id': Ref('id1')}, {'id': Ref('id2')}, {'id': Ref('id3')}] grid.extend(rows) assert len(grid) == 3 assert Ref('id1') in grid._index assert Ref('id2') in grid._index
def test_select_grid(): grid = Grid(columns=["id", "a", "b"]) grid.append({"id": Ref("myid1"), "a": 1, "b": 2}) grid.append({"id": Ref("myid2"), "a": 1, "b": 2}) assert grid.select("id,a").column == {"id": {}, 'a': {}} assert grid.select("id,b").column == {"id": {}, 'b': {}} assert grid.select("!a").column == {"id": {}, 'b': {}} assert grid.select("!id,!b").column == {'a': {}}
def test_grid_add(): left = Grid(columns=["id", "a", "b"]) left.append({"id": Ref("my_id"), "a": 1, "b": 2}) right = Grid(columns={"id": {}, "a": {}, "b": {}}) right.append({"id": Ref("my_id"), "a": 3, "c": 4}) add_grid = left + right assert isinstance(add_grid, Grid) assert len(add_grid) == 1
def test_pop_multiple_keys(): grid = Grid(columns=["id", "a"]) row1 = {"id": Ref("id1"), "a": 1, "b": 2} row2 = {"id": Ref("id2"), "a": 1, "b": 2} grid.append(row1) grid.append(row2) old = grid.pop(Ref("id1"), Ref("id2")) assert not grid assert id(old) == id(row1)
def test_grid_del(): grid = Grid(columns=['id', 'test']) rows = [{'test': 1}, {'test': 2}, {'test': 3}, {'id': Ref('myid')}] grid.extend(rows) assert len(grid) == 4 del grid[1] del grid[Ref('myid')] assert len(grid) == 2 assert grid[0] is rows[0] assert grid[1] is rows[2]
def test_grid_contain(): grid = Grid(columns=['id', 'site']) grid.append({ 'id': Ref('id1'), }) grid.append({'id': Ref('id2')}) grid.append({'id': Ref('id3')}) assert Ref('id1') in grid assert Ref('id2') in grid
def test_slice(): grid = Grid(columns=['id', 'site']) grid.append({ 'id': Ref('id1'), }) grid.append({'id': Ref('id2')}) grid.append({'id': Ref('id3')}) result = grid[0:2] assert isinstance(result, Grid) assert len(result) == 2 assert result[Ref('id1')] # pylint: disable=invalid-sequence-index assert result[Ref('id2')] # pylint: disable=invalid-sequence-index
def test_values_for_tag(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" with get_provider("shaystack.providers.url", {}) as provider: result = provider.values_for_tag("col") assert result == [1.0, 2.0] result = provider.values_for_tag("id") assert result == [Ref("id1"), Ref("id2")]
def test_grid_sub(): left = Grid(columns=["id", "a", "b"]) left.append({"id": Ref("my_id"), "a": 1}) right = Grid(columns={"id": {}, "a": {}, "b": {}}) right.append({"id": Ref("my_id"), "a": 3, "b": 4}) diff = right - left assert isinstance(diff, Grid) assert len(diff) == 1 diff = left - right assert isinstance(diff, Grid) assert len(diff) == 1
def test_import_ts_grid_in_db_and_his_read(mock1, mock2): mock1.return_value = "customer" mock2.return_value = "customer" envs = {'HAYSTACK_DB': HAYSTACK_DB, 'HAYSTACK_TS': HAYSTACK_TS, 'AWS_PROFILE': os.environ['AWS_PROFILE'], 'AWS_REGION': os.environ['AWS_REGION'] } with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider: values = [ (XStr("hex", "deadbeef"), "Str"), ("100", "Str"), (100.0, "Number"), (Quantity(1, "m"), "Number"), (100, "Number"), (True, "Bool"), (False, "Bool"), (MARKER, "Marker"), (None, "Marker"), (REMOVE, "Remove"), (None, "Remove"), (NA, "NA"), (None, "NA"), (Ref("abc"), "Ref"), (datetime.datetime.utcnow().replace(microsecond=0), "DateTime"), (datetime.date.today(), "Date"), (datetime.datetime.utcnow().time(), "Time"), (datetime.time(16, 58, 57, 994), "Time"), (Coordinate(100.0, 200.0), "Coord"), ] # Check TS with all types entity_id = Ref("abc") for val, kind in values: # Clean DB for the specific kind provider.purge_db() provider.create_db() # Insert an entity for the TS, with an attribut "kind" grid = Grid(columns=["id", "kind"]) grid.append({"id": entity_id, "kind": kind}) # Without "kind", the default is "Number" or "float" version = datetime.datetime.now(tz=pytz.UTC) provider.update_grid(diff_grid=grid, version=version, customer_id="customer") # WARNING: timestream accept only datetime in the memory retention period. # Not before and not after. log.debug("Test %s", type(val)) grid = Grid(columns=["ts", "val"]) # You must use utcnow() and a retention grid.append({"ts": datetime.datetime.utcnow(), "val": val}) provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW) grid_ts = provider.his_read(entity_id, parse_date_range("today", provider.get_tz()), None) assert grid_ts[0]['val'] == val, f"with kind={kind} and val={val}"
def test_update_grid(): try: envs = {'HAYSTACK_DB': HAYSTACK_DB} with cast(SQLProvider, get_provider("shaystack.providers.sql", envs, use_cache=False)) as provider: provider.purge_db() provider.create_db() grid = Grid(metadata={"dis": "hello"}, columns=[("id", {}), ("a", {"dis": "a"}), ("b", {"dis": "b"})]) grid.append({"id": Ref("1"), "a": "a", "b": "b"}) grid.append({"id": Ref("2"), "a": "c", "b": "d"}) provider.update_grid(grid, None, "customer", FAKE_NOW) except psycopg2.OperationalError as ex: raise SkipTest("Postgres db not started") from ex
def test_read_version_with_ids(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" with get_provider("shaystack.providers.url", {}) as provider: version_2 = datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC) result = provider.read(0, None, [Ref("id1")], None, version_2) assert result.metadata["v"] == "2" assert len(result) == 1 assert result[0]['id'] == Ref("id1")
def test_pop_multiple_pos(): grid = Grid(columns=["id", "a"]) row = {"id": Ref("id2"), "a": 1, "b": 2} grid.append(row) old = grid.pop(0, 1) assert not grid assert id(old) == id(row)
def test_import_ts_grid_in_db_with_a_lot_of_records(mock1, mock2): mock1.return_value = "customer" mock2.return_value = "customer" envs = {'HAYSTACK_DB': HAYSTACK_DB, 'HAYSTACK_TS': HAYSTACK_TS, 'AWS_PROFILE': os.environ['AWS_PROFILE'], 'AWS_REGION': os.environ['AWS_REGION'] } with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider: # Check TS with all types entity_id = Ref("abc") # Insert an antity for the TS, with an attribut "kind" provider.purge_db() grid = Grid(columns=["id", "kind"]) grid.append({"id": entity_id, "kind": "Number"}) # Without "kind", the default is "Number" or "float" version = datetime.datetime.now(tz=pytz.UTC) provider.update_grid(diff_grid=grid, version=version, customer_id="customer") # WARNING: timestream accept only datetime in the memory retention period. # Not before and not after. # It's not possible to extend the memory retention temporary to inject an old value provider.purge_ts() provider.create_db() grid = Grid(columns=["ts", "val"]) # You must use utcnow() and a retention for i in range(0, 200): grid.append({"ts": datetime.datetime.utcnow().replace(microsecond=i * 1000), "val": i}) provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
def _test_read_with_str_greater(provider_name: str, db: str): with cast( DBHaystackInterface, get_provider(provider_name, {'HAYSTACK_DB': db}, use_cache=False)) as provider: _populate_db(provider) result = provider.read(0, None, None, 'dis > "A"', None) assert len(result) == 2, f"with {db}" assert result[Ref("id1")] == { "id": Ref("id1"), "col": 5, "dis": "Dis 1", "his": MARKER, "hour": datetime.time(20, 0), "date": datetime.date(2021, 1, 1), "datetime": datetime.datetime(2021, 1, 1, 10, 0, tzinfo=pytz.UTC) }, f"with {db}"
def test_grid_filter(): grid = Grid( columns={ 'id': {}, 'site': {}, 'equip': {}, 'geoPostalCode': {}, 'ahu': {}, 'geoCity': {}, 'curVal': {}, 'hvac': {}, 'siteRef': {} }) grid.append({ 'id': Ref('id1'), 'site': MARKER, 'equip': 'Chicago', 'geoPostalCode': "78280", 'ahu': MARKER, 'geoCity': 'Chicago', 'curVal': 76 }) grid.append({ 'id': Ref('id2'), 'hvac': MARKER, 'geoPostalCode': "23220", 'curVal': 75 }) grid.append({ 'equip': 'Chicago', 'hvac': MARKER, 'siteRef': Ref('id1'), 'curVal': 74 }) result = grid.filter('site') assert len(result) == 1 assert result[Ref('id1')] result = grid.filter('equip == "Chicago"') assert len(result) == 2 assert result[0]['id'] == Ref('id1') assert result[1]['equip'] == 'Chicago' result = grid.filter('not id') assert len(result) == 1 assert result[0]['equip'] == 'Chicago'
def test_diff_remove_value_with_id(): left = Grid(columns={"id": {}, "a": {}, "b": {}}) left.append({"id": Ref("my_id"), "a": 1, "b": 2}) right = Grid(columns={"id": {}, "a": {}, "b": {}}) right.append({"id": Ref("my_id"), "a": 3}) diff = grid_diff(left, right) assert len(diff) == 1 assert len(diff[0].keys()) == 3 assert diff[0]['id'] == Ref("my_id") assert diff[0]['a'] == 3 assert diff[0]['b'] == REMOVE assert 'id' in diff.column assert 'a' in diff.column assert 'b' in diff.column assert grid_merge(left, diff) == right