def test_filter_and_limit():
grid = Grid(columns={'id': {}, 'site': {}})
grid.append({
'id': Ref('id1'),
'site': MARKER,
'equip': 'Chicago',
'geoPostalCode': "78280",
'ahu': MARKER,
'geoCity': 'Chicago',
'curVal': 76
})
grid.append({
'id': Ref('id2'),
'hvac': MARKER,
'geoPostalCode': "23220",
'curVal': 75
})
grid.append({
'equip': 'Chicago',
'hvac': MARKER,
'siteRef': Ref('id1'),
'curVal': 74
})
assert len(grid.filter('not acme', limit=1)) == 1
def test_invoke_action_with_zinc(mock) -> None:
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = ping._PingGrid
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = shaystack.Grid(metadata={
'id': Ref('123'),
'action': 'doIt'
},
columns={
'key': {},
'value': {}
})
grid.append({'param': 'value'})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=shaystack.MODE_ZINC)
# WHEN
response = shaystack.invoke_action(envs, request, "dev")
# THEN
mock.assert_called_once_with(Ref("123"), "doIt", {})
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, shaystack.MODE_ZINC) is not None
def test_his_read_with_range_today(mock) -> None:
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = ping._PingGrid
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = shaystack.Grid(columns=['id', 'range'])
grid.append({"id": Ref("1234"), "range": "today"})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.his_read(envs, request, "dev")
# THEN
today = datetime.combine(date.today(), datetime.min.time()).replace(tzinfo=get_localzone())
mock.assert_called_once_with(Ref("1234"),
(today,
today + timedelta(days=1)
), None)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, mime_type) is not None
def test_his_write_with_args(mock):
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
time_serie = [
(datetime(2020, 1, 1, tzinfo=pytz.utc).isoformat() + " UTC", 100),
(datetime(2020, 1, 2, tzinfo=pytz.utc).isoformat() + " UTC", 200)]
mock.return_value = ping._PingGrid
mime_type = DEFAULT_MIME_TYPE
request = HaystackHttpRequest()
request.args['id'] = str(Ref("1234"))
request.args['ts'] = str(time_serie)
# WHEN
response = shaystack.his_write(envs, request, "dev")
# THEN
result_ts = Grid(version=VER_3_0, columns=["date", "val"])
result_ts.extend([{"date": parse_hs_datetime_format(d, pytz.UTC), "val": v} for d, v in time_serie])
mock.assert_called_once_with(Ref("1234"), result_ts, None)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, mime_type) is not None
def test_watch_unsub_with_zinc(mock) -> None:
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = ping._PingGrid
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = shaystack.Grid(metadata={"close": True,
"watchId": "0123456789ABCDEF"},
columns=['id'])
grid.append({"id": Ref("id1")})
grid.append({"id": Ref("id2")})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=shaystack.MODE_ZINC)
# WHEN
response = shaystack.watch_unsub(envs, request, "dev")
# THEN
mock.assert_called_once_with("0123456789ABCDEF", [Ref("id1"), Ref("id2")], True)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, shaystack.MODE_ZINC) is not None
def test_update_grid():
try:
envs = {'HAYSTACK_DB': HAYSTACK_DB}
with cast(MongoProvider,
get_provider("shaystack.providers.mongodb",
envs)) as provider:
provider.purge_db()
provider.create_db()
grid = Grid(metadata={"dis": "hello"},
columns=[("id", {}), ("a", {
"dis": "a"
}), ("b", {
"dis": "b"
})])
grid.append({"id": Ref("1"), "a": "a", "b": "b"})
grid.append({"id": Ref("2"), "a": "c", "b": "d"})
provider.update_grid(grid, None, "customer", FAKE_NOW)
in_table = [
_conv_row_to_entity(row['entity'])
for row in provider.get_collection().find()
]
assert len(in_table) == len(grid)
assert in_table[0] == grid[0]
assert in_table[1] == grid[1]
except ServerSelectionTimeoutError as ex:
raise SkipTest("Mongo db not started") from ex
def test_pop_key():
grid = Grid(columns=["id", "a"])
row = {"id": Ref("myid"), "a": 1, "b": 2}
grid.append(row)
old = grid.pop(Ref("myid"))
assert not grid
assert id(old) == id(row)
def test_point_write_write_with_zinc(mock) -> None:
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = Grid(version=VER_3_0,
columns=["level", "levelDis", "val", "who"])
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = shaystack.Grid(columns=['id', "level", "val", "who", "duration"])
grid.append({
"id": Ref("1234"),
"level": 1,
"val": 100.0,
"who": "PPR",
"duration": Quantity(1, "min")
})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.point_write(envs, request, "dev")
# THEN
mock.assert_called_once_with(Ref("1234"), 1, 100, "PPR",
Quantity(1, "min"), None)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, mime_type) is not None
def test_read_with_zinc_and_id(mock) -> None:
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = ping._PingGrid
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = shaystack.Grid(columns=['id'])
grid.append({"id": Ref("me")})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.read(envs, request, "dev")
# THEN
ids = [Ref("me")]
mock.assert_called_once_with(0, None, ids, '', None)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, mime_type) is not None
def test_his_read_with_range_one_datetime(mock) -> None:
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = ping._PingGrid
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = shaystack.Grid(columns=['id', 'range'])
datetime_1 = datetime(2020, 1, 1, 0, 0, 0, tzinfo=pytz.UTC)
grid.append({"id": Ref("1234"), "range": datetime_1.isoformat()})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.his_read(envs, request, "dev")
# THEN
cur_datetime = datetime.combine(datetime_1, datetime.min.time()).replace(tzinfo=pytz.UTC)
mock.assert_called_once_with(Ref("1234"),
(cur_datetime,
datetime.max.replace(tzinfo=pytz.UTC)
), None)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, mime_type) is not None
def test_his_read_with_zinc(mock, no_cache) -> None:
# GIVEN
"""
Args:
mock:
no_cache:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = ping._PingGrid
no_cache.return_value = True
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = shaystack.Grid(columns={'id': {}})
grid.append({"id": Ref("1234")})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.his_read(envs, request, "dev")
# THEN
mock.assert_called_once_with(Ref("1234"),
(datetime.min.replace(tzinfo=pytz.UTC),
datetime.max.replace(tzinfo=pytz.UTC)), None)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, mime_type) is not None
def test_pop_invalid_key():
grid = Grid(columns=["id", "a"])
row = {"id": Ref("myid"), "a": 1, "b": 2}
grid.append(row)
old = grid.pop(Ref("other_id"))
assert grid
assert not old
def test_grid_getitem_with_id():
grid = Grid()
grid.column['id'] = {}
grid.column['test'] = {}
row = {'id': Ref('myid'), 'test': 'This is a test'}
grid.append(row)
assert grid[Ref('myid')] is row
def test_grid_extend():
grid = Grid(columns=['id'])
grid.reindex()
rows = [{'id': Ref('id1')}, {'id': Ref('id2')}, {'id': Ref('id3')}]
grid.extend(rows)
assert len(grid) == 3
assert Ref('id1') in grid._index
assert Ref('id2') in grid._index
def test_select_grid():
grid = Grid(columns=["id", "a", "b"])
grid.append({"id": Ref("myid1"), "a": 1, "b": 2})
grid.append({"id": Ref("myid2"), "a": 1, "b": 2})
assert grid.select("id,a").column == {"id": {}, 'a': {}}
assert grid.select("id,b").column == {"id": {}, 'b': {}}
assert grid.select("!a").column == {"id": {}, 'b': {}}
assert grid.select("!id,!b").column == {'a': {}}
def test_grid_add():
left = Grid(columns=["id", "a", "b"])
left.append({"id": Ref("my_id"), "a": 1, "b": 2})
right = Grid(columns={"id": {}, "a": {}, "b": {}})
right.append({"id": Ref("my_id"), "a": 3, "c": 4})
add_grid = left + right
assert isinstance(add_grid, Grid)
assert len(add_grid) == 1
def test_pop_multiple_keys():
grid = Grid(columns=["id", "a"])
row1 = {"id": Ref("id1"), "a": 1, "b": 2}
row2 = {"id": Ref("id2"), "a": 1, "b": 2}
grid.append(row1)
grid.append(row2)
old = grid.pop(Ref("id1"), Ref("id2"))
assert not grid
assert id(old) == id(row1)
def test_grid_del():
grid = Grid(columns=['id', 'test'])
rows = [{'test': 1}, {'test': 2}, {'test': 3}, {'id': Ref('myid')}]
grid.extend(rows)
assert len(grid) == 4
del grid[1]
del grid[Ref('myid')]
assert len(grid) == 2
assert grid[0] is rows[0]
assert grid[1] is rows[2]
def test_grid_contain():
grid = Grid(columns=['id', 'site'])
grid.append({
'id': Ref('id1'),
})
grid.append({'id': Ref('id2')})
grid.append({'id': Ref('id3')})
assert Ref('id1') in grid
assert Ref('id2') in grid
def test_slice():
grid = Grid(columns=['id', 'site'])
grid.append({
'id': Ref('id1'),
})
grid.append({'id': Ref('id2')})
grid.append({'id': Ref('id3')})
result = grid[0:2]
assert isinstance(result, Grid)
assert len(result) == 2
assert result[Ref('id1')] # pylint: disable=invalid-sequence-index
assert result[Ref('id2')] # pylint: disable=invalid-sequence-index
def test_values_for_tag(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
with get_provider("shaystack.providers.url", {}) as provider:
result = provider.values_for_tag("col")
assert result == [1.0, 2.0]
result = provider.values_for_tag("id")
assert result == [Ref("id1"), Ref("id2")]
def test_grid_sub():
left = Grid(columns=["id", "a", "b"])
left.append({"id": Ref("my_id"), "a": 1})
right = Grid(columns={"id": {}, "a": {}, "b": {}})
right.append({"id": Ref("my_id"), "a": 3, "b": 4})
diff = right - left
assert isinstance(diff, Grid)
assert len(diff) == 1
diff = left - right
assert isinstance(diff, Grid)
assert len(diff) == 1
def test_import_ts_grid_in_db_and_his_read(mock1, mock2):
mock1.return_value = "customer"
mock2.return_value = "customer"
envs = {'HAYSTACK_DB': HAYSTACK_DB,
'HAYSTACK_TS': HAYSTACK_TS,
'AWS_PROFILE': os.environ['AWS_PROFILE'],
'AWS_REGION': os.environ['AWS_REGION']
}
with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider:
values = [
(XStr("hex", "deadbeef"), "Str"),
("100", "Str"),
(100.0, "Number"), (Quantity(1, "m"), "Number"), (100, "Number"),
(True, "Bool"), (False, "Bool"),
(MARKER, "Marker"), (None, "Marker"),
(REMOVE, "Remove"), (None, "Remove"),
(NA, "NA"), (None, "NA"),
(Ref("abc"), "Ref"),
(datetime.datetime.utcnow().replace(microsecond=0), "DateTime"),
(datetime.date.today(), "Date"),
(datetime.datetime.utcnow().time(), "Time"),
(datetime.time(16, 58, 57, 994), "Time"),
(Coordinate(100.0, 200.0), "Coord"),
]
# Check TS with all types
entity_id = Ref("abc")
for val, kind in values:
# Clean DB for the specific kind
provider.purge_db()
provider.create_db()
# Insert an entity for the TS, with an attribut "kind"
grid = Grid(columns=["id", "kind"])
grid.append({"id": entity_id, "kind": kind}) # Without "kind", the default is "Number" or "float"
version = datetime.datetime.now(tz=pytz.UTC)
provider.update_grid(diff_grid=grid, version=version, customer_id="customer")
# WARNING: timestream accept only datetime in the memory retention period.
# Not before and not after.
log.debug("Test %s", type(val))
grid = Grid(columns=["ts", "val"])
# You must use utcnow() and a retention
grid.append({"ts": datetime.datetime.utcnow(), "val": val})
provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
grid_ts = provider.his_read(entity_id, parse_date_range("today", provider.get_tz()), None)
assert grid_ts[0]['val'] == val, f"with kind={kind} and val={val}"
def test_update_grid():
try:
envs = {'HAYSTACK_DB': HAYSTACK_DB}
with cast(SQLProvider, get_provider("shaystack.providers.sql", envs,
use_cache=False)) as provider:
provider.purge_db()
provider.create_db()
grid = Grid(metadata={"dis": "hello"},
columns=[("id", {}), ("a", {"dis": "a"}), ("b", {"dis": "b"})])
grid.append({"id": Ref("1"), "a": "a", "b": "b"})
grid.append({"id": Ref("2"), "a": "c", "b": "d"})
provider.update_grid(grid, None, "customer", FAKE_NOW)
except psycopg2.OperationalError as ex:
raise SkipTest("Postgres db not started") from ex
def test_read_version_with_ids(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
with get_provider("shaystack.providers.url", {}) as provider:
version_2 = datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC)
result = provider.read(0, None, [Ref("id1")], None, version_2)
assert result.metadata["v"] == "2"
assert len(result) == 1
assert result[0]['id'] == Ref("id1")
def test_pop_multiple_pos():
grid = Grid(columns=["id", "a"])
row = {"id": Ref("id2"), "a": 1, "b": 2}
grid.append(row)
old = grid.pop(0, 1)
assert not grid
assert id(old) == id(row)
def test_import_ts_grid_in_db_with_a_lot_of_records(mock1, mock2):
mock1.return_value = "customer"
mock2.return_value = "customer"
envs = {'HAYSTACK_DB': HAYSTACK_DB,
'HAYSTACK_TS': HAYSTACK_TS,
'AWS_PROFILE': os.environ['AWS_PROFILE'],
'AWS_REGION': os.environ['AWS_REGION']
}
with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider:
# Check TS with all types
entity_id = Ref("abc")
# Insert an antity for the TS, with an attribut "kind"
provider.purge_db()
grid = Grid(columns=["id", "kind"])
grid.append({"id": entity_id, "kind": "Number"}) # Without "kind", the default is "Number" or "float"
version = datetime.datetime.now(tz=pytz.UTC)
provider.update_grid(diff_grid=grid, version=version, customer_id="customer")
# WARNING: timestream accept only datetime in the memory retention period.
# Not before and not after.
# It's not possible to extend the memory retention temporary to inject an old value
provider.purge_ts()
provider.create_db()
grid = Grid(columns=["ts", "val"])
# You must use utcnow() and a retention
for i in range(0, 200):
grid.append({"ts": datetime.datetime.utcnow().replace(microsecond=i * 1000), "val": i})
provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
def _test_read_with_str_greater(provider_name: str, db: str):
with cast(
DBHaystackInterface,
get_provider(provider_name, {'HAYSTACK_DB': db},
use_cache=False)) as provider:
_populate_db(provider)
result = provider.read(0, None, None, 'dis > "A"', None)
assert len(result) == 2, f"with {db}"
assert result[Ref("id1")] == {
"id": Ref("id1"),
"col": 5,
"dis": "Dis 1",
"his": MARKER,
"hour": datetime.time(20, 0),
"date": datetime.date(2021, 1, 1),
"datetime": datetime.datetime(2021, 1, 1, 10, 0, tzinfo=pytz.UTC)
}, f"with {db}"
def test_grid_filter():
grid = Grid(
columns={
'id': {},
'site': {},
'equip': {},
'geoPostalCode': {},
'ahu': {},
'geoCity': {},
'curVal': {},
'hvac': {},
'siteRef': {}
})
grid.append({
'id': Ref('id1'),
'site': MARKER,
'equip': 'Chicago',
'geoPostalCode': "78280",
'ahu': MARKER,
'geoCity': 'Chicago',
'curVal': 76
})
grid.append({
'id': Ref('id2'),
'hvac': MARKER,
'geoPostalCode': "23220",
'curVal': 75
})
grid.append({
'equip': 'Chicago',
'hvac': MARKER,
'siteRef': Ref('id1'),
'curVal': 74
})
result = grid.filter('site')
assert len(result) == 1
assert result[Ref('id1')]
result = grid.filter('equip == "Chicago"')
assert len(result) == 2
assert result[0]['id'] == Ref('id1')
assert result[1]['equip'] == 'Chicago'
result = grid.filter('not id')
assert len(result) == 1
assert result[0]['equip'] == 'Chicago'
def test_diff_remove_value_with_id():
left = Grid(columns={"id": {}, "a": {}, "b": {}})
left.append({"id": Ref("my_id"), "a": 1, "b": 2})
right = Grid(columns={"id": {}, "a": {}, "b": {}})
right.append({"id": Ref("my_id"), "a": 3})
diff = grid_diff(left, right)
assert len(diff) == 1
assert len(diff[0].keys()) == 3
assert diff[0]['id'] == Ref("my_id")
assert diff[0]['a'] == 3
assert diff[0]['b'] == REMOVE
assert 'id' in diff.column
assert 'a' in diff.column
assert 'b' in diff.column
assert grid_merge(left, diff) == right