def test_update_grid(): try: envs = {'HAYSTACK_DB': HAYSTACK_DB} with cast(MongoProvider, get_provider("shaystack.providers.mongodb", envs)) as provider: provider.purge_db() provider.create_db() grid = Grid(metadata={"dis": "hello"}, columns=[("id", {}), ("a", { "dis": "a" }), ("b", { "dis": "b" })]) grid.append({"id": Ref("1"), "a": "a", "b": "b"}) grid.append({"id": Ref("2"), "a": "c", "b": "d"}) provider.update_grid(grid, None, "customer", FAKE_NOW) in_table = [ _conv_row_to_entity(row['entity']) for row in provider.get_collection().find() ] assert len(in_table) == len(grid) assert in_table[0] == grid[0] assert in_table[1] == grid[1] except ServerSelectionTimeoutError as ex: raise SkipTest("Mongo db not started") from ex
def test_his_write_with_args(mock): # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} time_serie = [ (datetime(2020, 1, 1, tzinfo=pytz.utc).isoformat() + " UTC", 100), (datetime(2020, 1, 2, tzinfo=pytz.utc).isoformat() + " UTC", 200)] mock.return_value = ping._PingGrid mime_type = DEFAULT_MIME_TYPE request = HaystackHttpRequest() request.args['id'] = str(Ref("1234")) request.args['ts'] = str(time_serie) # WHEN response = shaystack.his_write(envs, request, "dev") # THEN result_ts = Grid(version=VER_3_0, columns=["date", "val"]) result_ts.extend([{"date": parse_hs_datetime_format(d, pytz.UTC), "val": v} for d, v in time_serie]) mock.assert_called_once_with(Ref("1234"), result_ts, None) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, mime_type) is not None
def test_his_read_with_boolean(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() his = Grid(version=VER_3_0, columns=["ts", "val"]) his.extend([ { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": MARKER }, { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": False }, { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": 1 }, { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": 1.0 }, { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": "" }, ]) mock_s3.return_value.history = his mock_get_url.return_value = "s3://bucket/grid.zinc" envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"} with get_provider("shaystack.providers.url", envs) as _: client = Client(schema) executed = client.execute(''' { haystack { histories(ids:"@id1") { ts val bool } } } ''') assert executed == \ {'data': {'haystack': {'histories': [[{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'm:', 'bool': True}, {'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': False, 'bool': False}, {'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'n:1.000000', 'bool': True}, {'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'n:1.000000', 'bool': True}, {'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 's:', 'bool': False}]]}}}
def test_merge_timeseries_with_same_ts(): ts = Grid(columns=["ts", "value"]) ts.append({ "ts": datetime(2020, 1, 1, 0, 0, 2, 0, tzinfo=pytz.UTC), "value": 100 }) result_grid = merge_timeseries(ts, ts) assert ts == result_grid
def test_diff_version(): left = Grid(version=VER_2_0) right = Grid(version=VER_3_0) diff = grid_diff(left, right) assert diff.version == VER_3_0 assert grid_merge(left, diff) == right # A + (B - A) == B
def test_diff_with_same_grid(): left = Grid(columns={"a": {}, "b": {}}) left.append({"a": 1, "b": 2}) left.append({"a": 3, "b": 4}) diff = grid_diff(left, left) assert len(diff) == 0 assert grid_merge(left.copy(), diff) == left
def test_diff_cols_move_col(): left = Grid(columns={"a": {'m': 1}, "b": {'m': 2}, "c": {'m': 3}}) right = Grid(columns={"b": {'m': 2}, "a": {'m': 1}, "c": {'m': 3}}) diff = grid_diff(left, right) assert len(diff) == 0 assert len(diff.column) == 3 assert grid_merge(left, diff) == right
def test_merge_timeseries_with_diff_ts_same_values(): destination = Grid(columns=["ts", "value"]) destination.append({ "ts": datetime(2020, 1, 1, 0, 0, 2, 0, tzinfo=pytz.UTC), "value": 100 }) source = destination.copy() result_grid = merge_timeseries(source, destination) assert source == result_grid
def test_diff_metadata_change_value(): left = Grid(metadata={"a": 1, "b": 2}) right = Grid(metadata={"a": 3, "b": 4}) diff = grid_diff(left, right) assert len(diff) == 0 assert diff.metadata["a"] == 3 assert diff.metadata["b"] == 4 assert grid_merge(left, diff) == right
def test_diff_cols_add_value_in_map(): left = Grid(columns={"a": {'m': 1}, "b": {'m': 2}}) right = Grid(columns={"a": {'m': 1, 'n': 2}, "b": {'m': 4}}) diff = grid_diff(left, right) assert len(diff) == 0 assert diff.column["a"] == {"n": 2} assert diff.column["b"] == {"m": 4} assert grid_merge(left, diff) == right
def test_diff_metadata_remove_value(): left = Grid(metadata={"a": 1, "b": 2}) right = Grid(metadata={"b": 2}) diff = grid_diff(left, right) assert len(diff) == 0 assert diff.metadata["a"] == REMOVE assert "b" not in diff.metadata assert grid_merge(left, diff) == right
def test_diff_cols_remove_value_in_map(): left = Grid(columns={"a": {'m': 1, 'n': 2}, "b": {'m': 2}}) right = Grid(columns={"a": {'n': 2}, "b": {'m': 2}}) diff = grid_diff(left, right) assert len(diff) == 0 assert diff.column["a"] == {"m": REMOVE} assert diff.column["b"] == {} assert grid_merge(left, diff) == right
def test_diff_cols_remove_first_col(): left = Grid(columns={"a": {'m': 1}, "b": {'m': 2}, "c": {'m': 3}}) right = Grid(columns={"b": {'m': 4}, "c": {'m': 3}}) diff = grid_diff(left, right) assert len(diff) == 0 assert diff.column["a"] == {'remove_': REMOVE} assert len(diff.column) == 3 assert grid_merge(left, diff) == right
def test_filter_and_limit(): grid = Grid(columns={'id': {}, 'site': {}}) grid.append({ 'id': Ref('id1'), 'site': MARKER, 'equip': 'Chicago', 'geoPostalCode': "78280", 'ahu': MARKER, 'geoCity': 'Chicago', 'curVal': 76 }) grid.append({ 'id': Ref('id2'), 'hvac': MARKER, 'geoPostalCode': "23220", 'curVal': 75 }) grid.append({ 'equip': 'Chicago', 'hvac': MARKER, 'siteRef': Ref('id1'), 'curVal': 74 }) assert len(grid.filter('not acme', limit=1)) == 1
def test_diff_metadata_add_value(): left = Grid(metadata={"a": 1, "b": 2}) right = Grid(metadata={"a": 1, "b": 2, "c": 3}) diff = grid_diff(left, right) assert len(diff) == 0 assert diff.metadata["c"] == 3 assert "a" not in diff.metadata assert "b" not in diff.metadata assert grid_merge(left, diff) == right
def test_diff_same_record_without_id_in_target(): left = Grid(columns={"a": {}, "b": {}}) left.append({"a": 1, "b": 2}) left.append({"a": 3, "b": 4}) right = Grid(columns={"a": {}, "b": {}}) right.append({"a": 1, "b": 2}) right.append({"a": 3, "b": 4}) right.append({"a": 3, "b": 4}) diff = grid_diff(left, right) assert len(diff) == 1
def test_diff_remove_record_without_id(): left = Grid(columns={"a": {}, "b": {}}) left.append({"a": 1, "b": 2}) left.append({"a": 3, "b": 4}) right = Grid(columns={"a": {}, "b": {}}) right.append({"a": 5, "b": 6}) diff = grid_diff(left, right) assert len(diff) == 3 assert len(diff[0].keys()) == 3 assert diff[0]['a'] == 1 assert diff[0]['b'] == 2 assert diff[0]['remove_'] == REMOVE assert len(diff[1].keys()) == 3 assert diff[1]['a'] == 3 assert diff[1]['b'] == 4 assert diff[1]['remove_'] == REMOVE assert len(diff[2].keys()) == 2 assert diff[2]['a'] == 5 assert diff[2]['b'] == 6 assert 'id' not in diff.column assert 'a' in diff.column assert 'b' in diff.column assert grid_merge(left, diff) == right
def test_import_ts_grid_in_db_with_a_lot_of_records(mock1, mock2): mock1.return_value = "customer" mock2.return_value = "customer" envs = {'HAYSTACK_DB': HAYSTACK_DB, 'HAYSTACK_TS': HAYSTACK_TS, 'AWS_PROFILE': os.environ['AWS_PROFILE'], 'AWS_REGION': os.environ['AWS_REGION'] } with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider: # Check TS with all types entity_id = Ref("abc") # Insert an antity for the TS, with an attribut "kind" provider.purge_db() grid = Grid(columns=["id", "kind"]) grid.append({"id": entity_id, "kind": "Number"}) # Without "kind", the default is "Number" or "float" version = datetime.datetime.now(tz=pytz.UTC) provider.update_grid(diff_grid=grid, version=version, customer_id="customer") # WARNING: timestream accept only datetime in the memory retention period. # Not before and not after. # It's not possible to extend the memory retention temporary to inject an old value provider.purge_ts() provider.create_db() grid = Grid(columns=["ts", "val"]) # You must use utcnow() and a retention for i in range(0, 200): grid.append({"ts": datetime.datetime.utcnow().replace(microsecond=i * 1000), "val": i}) provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
def test_update_grid(): try: envs = {'HAYSTACK_DB': HAYSTACK_DB} with cast(SQLProvider, get_provider("shaystack.providers.sql", envs, use_cache=False)) as provider: provider.purge_db() provider.create_db() grid = Grid(metadata={"dis": "hello"}, columns=[("id", {}), ("a", {"dis": "a"}), ("b", {"dis": "b"})]) grid.append({"id": Ref("1"), "a": "a", "b": "b"}) grid.append({"id": Ref("2"), "a": "c", "b": "d"}) provider.update_grid(grid, None, "customer", FAKE_NOW) except psycopg2.OperationalError as ex: raise SkipTest("Postgres db not started") from ex
def test_point_write_write_with_zinc(mock) -> None: # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = Grid(version=VER_3_0, columns=["level", "levelDis", "val", "who"]) mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = shaystack.Grid(columns=['id', "level", "val", "who", "duration"]) grid.append({ "id": Ref("1234"), "level": 1, "val": 100.0, "who": "PPR", "duration": Quantity(1, "min") }) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.point_write(envs, request, "dev") # THEN mock.assert_called_once_with(Ref("1234"), 1, 100, "PPR", Quantity(1, "min"), None) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, mime_type) is not None
def test_watch_sub_with_args(mock): # GIVEN """ Args: mock: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mock.return_value = Grid(version=VER_3_0, metadata={ "watchId": "0123456789ABCDEF", "lease": 1 }, columns=["empty"]) mock.return_value.append({}) mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() request.headers["Accept"] = mime_type request.args["watchDis"] = "myWatch" request.args["watchId"] = "myid" request.args["lease"] = "1" ids = ["@id1", "@id2"] request.args["ids"] = str(ids) # WHEN response = shaystack.watch_sub(envs, request, "dev") # THEN mock.assert_called_once_with("myWatch", "myid", [Ref("id1"), Ref("id2")], 1) assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) assert shaystack.parse(response.body, shaystack.MODE_ZINC) is not None
def test_negociation_with_complex_accept() -> None: # GIVEN envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = Grid(columns={'filter': {}, 'limit': {}}) grid.append({'filter': '', 'limit': -1}) request.headers["Accept"] = "text/json;q=0.9,text/zinc;q=1" request.headers["Content-Type"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.read(envs, request, "dev") # THEN assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type)
def test_negociation_without_accept() -> None: # GIVEN envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mime_type = shaystack.MODE_CSV request = HaystackHttpRequest() grid = Grid(columns={'filter': {}, 'limit': {}}) grid.append({'filter': '', 'limit': -1}) del request.headers["Accept"] request.headers["Content-Type"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN try: shaystack.read(envs, request, "dev") assert False, "Must generate exception" except HttpError as ex: assert ex.error == 406
def test_negociation_zinc_without_content_type() -> None: # GIVEN envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mime_type = shaystack.MODE_CSV request = HaystackHttpRequest() grid = Grid(columns={'id': {}}) grid.append({"id": Ref("1234")}) del request.headers["Content-Type"] request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.read(envs, request, "dev") # THEN assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) shaystack.parse(response.body, mime_type)
def test_his_read_with_datetime(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() his = Grid(version=VER_3_0, columns=["ts", "val"]) his.append({ "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": datetime(2020, 1, 1, tzinfo=pytz.utc) }) mock_s3.return_value.history = his mock_s3.return_value.his_count = 300 mock_get_url.return_value = "s3://bucket/grid.zinc" envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"} with cast(URLProvider, get_provider("shaystack.providers.url", envs)) as Provider: Provider.cache_clear() client = Client(schema) executed = client.execute(''' { haystack { histories(ids:"@id1") { ts val date time #datetime } } } ''') assert executed == \ {'data': {'haystack': {'histories': [[{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 't:2020-01-01T00:00:00+00:00 UTC', 'date': '2020-01-01T00:00:00+00:00', 'time': '00:00:00'}]]}}}
def test_diff_with_no_column(): left = Grid(columns={"a": {}, "b": {}}) left.append({"a": 1, "b": 2}) left.append({"a": 3, "b": 4}) left.append({"a": 1, "b": 2}) left.append({"a": 1, "b": 2}) right = Grid() diff = grid_diff(left, right) assert len(diff) == 4 assert grid_merge(left.copy(), diff) == right
def test_import_ts_grid_in_db_and_his_read(mock1, mock2): mock1.return_value = "customer" mock2.return_value = "customer" envs = {'HAYSTACK_DB': HAYSTACK_DB, 'HAYSTACK_TS': HAYSTACK_TS, 'AWS_PROFILE': os.environ['AWS_PROFILE'], 'AWS_REGION': os.environ['AWS_REGION'] } with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider: values = [ (XStr("hex", "deadbeef"), "Str"), ("100", "Str"), (100.0, "Number"), (Quantity(1, "m"), "Number"), (100, "Number"), (True, "Bool"), (False, "Bool"), (MARKER, "Marker"), (None, "Marker"), (REMOVE, "Remove"), (None, "Remove"), (NA, "NA"), (None, "NA"), (Ref("abc"), "Ref"), (datetime.datetime.utcnow().replace(microsecond=0), "DateTime"), (datetime.date.today(), "Date"), (datetime.datetime.utcnow().time(), "Time"), (datetime.time(16, 58, 57, 994), "Time"), (Coordinate(100.0, 200.0), "Coord"), ] # Check TS with all types entity_id = Ref("abc") for val, kind in values: # Clean DB for the specific kind provider.purge_db() provider.create_db() # Insert an entity for the TS, with an attribut "kind" grid = Grid(columns=["id", "kind"]) grid.append({"id": entity_id, "kind": kind}) # Without "kind", the default is "Number" or "float" version = datetime.datetime.now(tz=pytz.UTC) provider.update_grid(diff_grid=grid, version=version, customer_id="customer") # WARNING: timestream accept only datetime in the memory retention period. # Not before and not after. log.debug("Test %s", type(val)) grid = Grid(columns=["ts", "val"]) # You must use utcnow() and a retention grid.append({"ts": datetime.datetime.utcnow(), "val": val}) provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW) grid_ts = provider.his_read(entity_id, parse_date_range("today", provider.get_tz()), None) assert grid_ts[0]['val'] == val, f"with kind={kind} and val={val}"
def get_object(self, Bucket, Key, IfNoneMatch): # pylint: disable=unused-argument, no-self-use if Key == 'sample/carytown.zinc': grid = Grid(columns=["hisURI"]) grid.append({"hisURL": "ts.zinc"}) else: grid = Grid(columns=["tz", "value"]) grid.append({ "ts": datetime(2020, 1, 1, 0, 0, 2, 0, tzinfo=pytz.UTC), "value": 100 }) data = dump(grid, MODE_ZINC).encode("UTF8") return {"Body": BytesIO(data)}
def test_negociation_with_navigator_accept() -> None: # GIVEN envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} mime_type = shaystack.MODE_CSV request = HaystackHttpRequest() grid = Grid(columns={'filter': {}, 'limit': {}}) grid.append({'filter': '', 'limit': -1}) request.headers[ "Accept"] = "Accept:text/html,application/xhtml+xml," \ "application/xml;q=0.9," \ "image/webp,image/apng," \ "*/*;q=0.8," \ "application/signed-exchange;v=b3;q=0.9" request.headers["Content-Type"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.read(envs, request, "dev") # THEN assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) shaystack.parse(response.body, mime_type)
def test_negociation_with_zinc(no_cache) -> None: # GIVEN """ Args: no_cache: """ envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'} no_cache.return_value = True mime_type = shaystack.MODE_ZINC request = HaystackHttpRequest() grid = Grid(columns={'filter': {}, "limit": {}}) grid.append({"filter": "id==@me", "limit": 1}) request.headers["Content-Type"] = mime_type request.headers["Accept"] = mime_type request.body = shaystack.dump(grid, mode=mime_type) # WHEN response = shaystack.read(envs, request, "dev") # THEN assert response.status_code == 200 assert response.headers["Content-Type"].startswith(mime_type) shaystack.parse(response.body, shaystack.MODE_ZINC)