def test_update_grid():
try:
envs = {'HAYSTACK_DB': HAYSTACK_DB}
with cast(MongoProvider,
get_provider("shaystack.providers.mongodb",
envs)) as provider:
provider.purge_db()
provider.create_db()
grid = Grid(metadata={"dis": "hello"},
columns=[("id", {}), ("a", {
"dis": "a"
}), ("b", {
"dis": "b"
})])
grid.append({"id": Ref("1"), "a": "a", "b": "b"})
grid.append({"id": Ref("2"), "a": "c", "b": "d"})
provider.update_grid(grid, None, "customer", FAKE_NOW)
in_table = [
_conv_row_to_entity(row['entity'])
for row in provider.get_collection().find()
]
assert len(in_table) == len(grid)
assert in_table[0] == grid[0]
assert in_table[1] == grid[1]
except ServerSelectionTimeoutError as ex:
raise SkipTest("Mongo db not started") from ex
def test_his_write_with_args(mock):
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
time_serie = [
(datetime(2020, 1, 1, tzinfo=pytz.utc).isoformat() + " UTC", 100),
(datetime(2020, 1, 2, tzinfo=pytz.utc).isoformat() + " UTC", 200)]
mock.return_value = ping._PingGrid
mime_type = DEFAULT_MIME_TYPE
request = HaystackHttpRequest()
request.args['id'] = str(Ref("1234"))
request.args['ts'] = str(time_serie)
# WHEN
response = shaystack.his_write(envs, request, "dev")
# THEN
result_ts = Grid(version=VER_3_0, columns=["date", "val"])
result_ts.extend([{"date": parse_hs_datetime_format(d, pytz.UTC), "val": v} for d, v in time_serie])
mock.assert_called_once_with(Ref("1234"), result_ts, None)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, mime_type) is not None
def test_his_read_with_boolean(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
his = Grid(version=VER_3_0, columns=["ts", "val"])
his.extend([
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": MARKER
},
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": False
},
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": 1
},
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": 1.0
},
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": ""
},
])
mock_s3.return_value.history = his
mock_get_url.return_value = "s3://bucket/grid.zinc"
envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"}
with get_provider("shaystack.providers.url", envs) as _:
client = Client(schema)
executed = client.execute('''
{
haystack
{
histories(ids:"@id1")
{
ts
val
bool
}
}
}
''')
assert executed == \
{'data': {'haystack':
{'histories': [[{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'm:', 'bool': True},
{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': False,
'bool': False},
{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'n:1.000000',
'bool': True},
{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'n:1.000000',
'bool': True},
{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 's:',
'bool': False}]]}}}
def test_merge_timeseries_with_same_ts():
ts = Grid(columns=["ts", "value"])
ts.append({
"ts": datetime(2020, 1, 1, 0, 0, 2, 0, tzinfo=pytz.UTC),
"value": 100
})
result_grid = merge_timeseries(ts, ts)
assert ts == result_grid
def test_diff_version():
left = Grid(version=VER_2_0)
right = Grid(version=VER_3_0)
diff = grid_diff(left, right)
assert diff.version == VER_3_0
assert grid_merge(left, diff) == right # A + (B - A) == B
def test_diff_with_same_grid():
left = Grid(columns={"a": {}, "b": {}})
left.append({"a": 1, "b": 2})
left.append({"a": 3, "b": 4})
diff = grid_diff(left, left)
assert len(diff) == 0
assert grid_merge(left.copy(), diff) == left
def test_diff_cols_move_col():
left = Grid(columns={"a": {'m': 1}, "b": {'m': 2}, "c": {'m': 3}})
right = Grid(columns={"b": {'m': 2}, "a": {'m': 1}, "c": {'m': 3}})
diff = grid_diff(left, right)
assert len(diff) == 0
assert len(diff.column) == 3
assert grid_merge(left, diff) == right
def test_merge_timeseries_with_diff_ts_same_values():
destination = Grid(columns=["ts", "value"])
destination.append({
"ts": datetime(2020, 1, 1, 0, 0, 2, 0, tzinfo=pytz.UTC),
"value": 100
})
source = destination.copy()
result_grid = merge_timeseries(source, destination)
assert source == result_grid
def test_diff_metadata_change_value():
left = Grid(metadata={"a": 1, "b": 2})
right = Grid(metadata={"a": 3, "b": 4})
diff = grid_diff(left, right)
assert len(diff) == 0
assert diff.metadata["a"] == 3
assert diff.metadata["b"] == 4
assert grid_merge(left, diff) == right
def test_diff_cols_add_value_in_map():
left = Grid(columns={"a": {'m': 1}, "b": {'m': 2}})
right = Grid(columns={"a": {'m': 1, 'n': 2}, "b": {'m': 4}})
diff = grid_diff(left, right)
assert len(diff) == 0
assert diff.column["a"] == {"n": 2}
assert diff.column["b"] == {"m": 4}
assert grid_merge(left, diff) == right
def test_diff_metadata_remove_value():
left = Grid(metadata={"a": 1, "b": 2})
right = Grid(metadata={"b": 2})
diff = grid_diff(left, right)
assert len(diff) == 0
assert diff.metadata["a"] == REMOVE
assert "b" not in diff.metadata
assert grid_merge(left, diff) == right
def test_diff_cols_remove_value_in_map():
left = Grid(columns={"a": {'m': 1, 'n': 2}, "b": {'m': 2}})
right = Grid(columns={"a": {'n': 2}, "b": {'m': 2}})
diff = grid_diff(left, right)
assert len(diff) == 0
assert diff.column["a"] == {"m": REMOVE}
assert diff.column["b"] == {}
assert grid_merge(left, diff) == right
def test_diff_cols_remove_first_col():
left = Grid(columns={"a": {'m': 1}, "b": {'m': 2}, "c": {'m': 3}})
right = Grid(columns={"b": {'m': 4}, "c": {'m': 3}})
diff = grid_diff(left, right)
assert len(diff) == 0
assert diff.column["a"] == {'remove_': REMOVE}
assert len(diff.column) == 3
assert grid_merge(left, diff) == right
def test_filter_and_limit():
grid = Grid(columns={'id': {}, 'site': {}})
grid.append({
'id': Ref('id1'),
'site': MARKER,
'equip': 'Chicago',
'geoPostalCode': "78280",
'ahu': MARKER,
'geoCity': 'Chicago',
'curVal': 76
})
grid.append({
'id': Ref('id2'),
'hvac': MARKER,
'geoPostalCode': "23220",
'curVal': 75
})
grid.append({
'equip': 'Chicago',
'hvac': MARKER,
'siteRef': Ref('id1'),
'curVal': 74
})
assert len(grid.filter('not acme', limit=1)) == 1
def test_diff_metadata_add_value():
left = Grid(metadata={"a": 1, "b": 2})
right = Grid(metadata={"a": 1, "b": 2, "c": 3})
diff = grid_diff(left, right)
assert len(diff) == 0
assert diff.metadata["c"] == 3
assert "a" not in diff.metadata
assert "b" not in diff.metadata
assert grid_merge(left, diff) == right
def test_diff_same_record_without_id_in_target():
left = Grid(columns={"a": {}, "b": {}})
left.append({"a": 1, "b": 2})
left.append({"a": 3, "b": 4})
right = Grid(columns={"a": {}, "b": {}})
right.append({"a": 1, "b": 2})
right.append({"a": 3, "b": 4})
right.append({"a": 3, "b": 4})
diff = grid_diff(left, right)
assert len(diff) == 1
def test_diff_remove_record_without_id():
left = Grid(columns={"a": {}, "b": {}})
left.append({"a": 1, "b": 2})
left.append({"a": 3, "b": 4})
right = Grid(columns={"a": {}, "b": {}})
right.append({"a": 5, "b": 6})
diff = grid_diff(left, right)
assert len(diff) == 3
assert len(diff[0].keys()) == 3
assert diff[0]['a'] == 1
assert diff[0]['b'] == 2
assert diff[0]['remove_'] == REMOVE
assert len(diff[1].keys()) == 3
assert diff[1]['a'] == 3
assert diff[1]['b'] == 4
assert diff[1]['remove_'] == REMOVE
assert len(diff[2].keys()) == 2
assert diff[2]['a'] == 5
assert diff[2]['b'] == 6
assert 'id' not in diff.column
assert 'a' in diff.column
assert 'b' in diff.column
assert grid_merge(left, diff) == right
def test_import_ts_grid_in_db_with_a_lot_of_records(mock1, mock2):
mock1.return_value = "customer"
mock2.return_value = "customer"
envs = {'HAYSTACK_DB': HAYSTACK_DB,
'HAYSTACK_TS': HAYSTACK_TS,
'AWS_PROFILE': os.environ['AWS_PROFILE'],
'AWS_REGION': os.environ['AWS_REGION']
}
with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider:
# Check TS with all types
entity_id = Ref("abc")
# Insert an antity for the TS, with an attribut "kind"
provider.purge_db()
grid = Grid(columns=["id", "kind"])
grid.append({"id": entity_id, "kind": "Number"}) # Without "kind", the default is "Number" or "float"
version = datetime.datetime.now(tz=pytz.UTC)
provider.update_grid(diff_grid=grid, version=version, customer_id="customer")
# WARNING: timestream accept only datetime in the memory retention period.
# Not before and not after.
# It's not possible to extend the memory retention temporary to inject an old value
provider.purge_ts()
provider.create_db()
grid = Grid(columns=["ts", "val"])
# You must use utcnow() and a retention
for i in range(0, 200):
grid.append({"ts": datetime.datetime.utcnow().replace(microsecond=i * 1000), "val": i})
provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
def test_update_grid():
try:
envs = {'HAYSTACK_DB': HAYSTACK_DB}
with cast(SQLProvider, get_provider("shaystack.providers.sql", envs,
use_cache=False)) as provider:
provider.purge_db()
provider.create_db()
grid = Grid(metadata={"dis": "hello"},
columns=[("id", {}), ("a", {"dis": "a"}), ("b", {"dis": "b"})])
grid.append({"id": Ref("1"), "a": "a", "b": "b"})
grid.append({"id": Ref("2"), "a": "c", "b": "d"})
provider.update_grid(grid, None, "customer", FAKE_NOW)
except psycopg2.OperationalError as ex:
raise SkipTest("Postgres db not started") from ex
def test_point_write_write_with_zinc(mock) -> None:
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = Grid(version=VER_3_0,
columns=["level", "levelDis", "val", "who"])
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = shaystack.Grid(columns=['id', "level", "val", "who", "duration"])
grid.append({
"id": Ref("1234"),
"level": 1,
"val": 100.0,
"who": "PPR",
"duration": Quantity(1, "min")
})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.point_write(envs, request, "dev")
# THEN
mock.assert_called_once_with(Ref("1234"), 1, 100, "PPR",
Quantity(1, "min"), None)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, mime_type) is not None
def test_watch_sub_with_args(mock):
# GIVEN
"""
Args:
mock:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mock.return_value = Grid(version=VER_3_0,
metadata={
"watchId": "0123456789ABCDEF",
"lease": 1
},
columns=["empty"])
mock.return_value.append({})
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
request.headers["Accept"] = mime_type
request.args["watchDis"] = "myWatch"
request.args["watchId"] = "myid"
request.args["lease"] = "1"
ids = ["@id1", "@id2"]
request.args["ids"] = str(ids)
# WHEN
response = shaystack.watch_sub(envs, request, "dev")
# THEN
mock.assert_called_once_with("myWatch", "myid",
[Ref("id1"), Ref("id2")], 1)
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
assert shaystack.parse(response.body, shaystack.MODE_ZINC) is not None
def test_negociation_with_complex_accept() -> None:
# GIVEN
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = Grid(columns={'filter': {}, 'limit': {}})
grid.append({'filter': '', 'limit': -1})
request.headers["Accept"] = "text/json;q=0.9,text/zinc;q=1"
request.headers["Content-Type"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.read(envs, request, "dev")
# THEN
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
def test_negociation_without_accept() -> None:
# GIVEN
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mime_type = shaystack.MODE_CSV
request = HaystackHttpRequest()
grid = Grid(columns={'filter': {}, 'limit': {}})
grid.append({'filter': '', 'limit': -1})
del request.headers["Accept"]
request.headers["Content-Type"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
try:
shaystack.read(envs, request, "dev")
assert False, "Must generate exception"
except HttpError as ex:
assert ex.error == 406
def test_negociation_zinc_without_content_type() -> None:
# GIVEN
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mime_type = shaystack.MODE_CSV
request = HaystackHttpRequest()
grid = Grid(columns={'id': {}})
grid.append({"id": Ref("1234")})
del request.headers["Content-Type"]
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.read(envs, request, "dev")
# THEN
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
shaystack.parse(response.body, mime_type)
def test_his_read_with_datetime(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
his = Grid(version=VER_3_0, columns=["ts", "val"])
his.append({
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": datetime(2020, 1, 1, tzinfo=pytz.utc)
})
mock_s3.return_value.history = his
mock_s3.return_value.his_count = 300
mock_get_url.return_value = "s3://bucket/grid.zinc"
envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"}
with cast(URLProvider, get_provider("shaystack.providers.url",
envs)) as Provider:
Provider.cache_clear()
client = Client(schema)
executed = client.execute('''
{
haystack
{
histories(ids:"@id1")
{
ts
val
date
time
#datetime
}
}
}
''')
assert executed == \
{'data':
{'haystack':
{'histories':
[[{'ts': '2020-01-01T00:00:00+00:00 UTC',
'val': 't:2020-01-01T00:00:00+00:00 UTC',
'date': '2020-01-01T00:00:00+00:00',
'time': '00:00:00'}]]}}}
def test_diff_with_no_column():
left = Grid(columns={"a": {}, "b": {}})
left.append({"a": 1, "b": 2})
left.append({"a": 3, "b": 4})
left.append({"a": 1, "b": 2})
left.append({"a": 1, "b": 2})
right = Grid()
diff = grid_diff(left, right)
assert len(diff) == 4
assert grid_merge(left.copy(), diff) == right
def test_import_ts_grid_in_db_and_his_read(mock1, mock2):
mock1.return_value = "customer"
mock2.return_value = "customer"
envs = {'HAYSTACK_DB': HAYSTACK_DB,
'HAYSTACK_TS': HAYSTACK_TS,
'AWS_PROFILE': os.environ['AWS_PROFILE'],
'AWS_REGION': os.environ['AWS_REGION']
}
with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider:
values = [
(XStr("hex", "deadbeef"), "Str"),
("100", "Str"),
(100.0, "Number"), (Quantity(1, "m"), "Number"), (100, "Number"),
(True, "Bool"), (False, "Bool"),
(MARKER, "Marker"), (None, "Marker"),
(REMOVE, "Remove"), (None, "Remove"),
(NA, "NA"), (None, "NA"),
(Ref("abc"), "Ref"),
(datetime.datetime.utcnow().replace(microsecond=0), "DateTime"),
(datetime.date.today(), "Date"),
(datetime.datetime.utcnow().time(), "Time"),
(datetime.time(16, 58, 57, 994), "Time"),
(Coordinate(100.0, 200.0), "Coord"),
]
# Check TS with all types
entity_id = Ref("abc")
for val, kind in values:
# Clean DB for the specific kind
provider.purge_db()
provider.create_db()
# Insert an entity for the TS, with an attribut "kind"
grid = Grid(columns=["id", "kind"])
grid.append({"id": entity_id, "kind": kind}) # Without "kind", the default is "Number" or "float"
version = datetime.datetime.now(tz=pytz.UTC)
provider.update_grid(diff_grid=grid, version=version, customer_id="customer")
# WARNING: timestream accept only datetime in the memory retention period.
# Not before and not after.
log.debug("Test %s", type(val))
grid = Grid(columns=["ts", "val"])
# You must use utcnow() and a retention
grid.append({"ts": datetime.datetime.utcnow(), "val": val})
provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
grid_ts = provider.his_read(entity_id, parse_date_range("today", provider.get_tz()), None)
assert grid_ts[0]['val'] == val, f"with kind={kind} and val={val}"
def get_object(self, Bucket, Key, IfNoneMatch): # pylint: disable=unused-argument, no-self-use
if Key == 'sample/carytown.zinc':
grid = Grid(columns=["hisURI"])
grid.append({"hisURL": "ts.zinc"})
else:
grid = Grid(columns=["tz", "value"])
grid.append({
"ts":
datetime(2020, 1, 1, 0, 0, 2, 0, tzinfo=pytz.UTC),
"value":
100
})
data = dump(grid, MODE_ZINC).encode("UTF8")
return {"Body": BytesIO(data)}
def test_negociation_with_navigator_accept() -> None:
# GIVEN
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
mime_type = shaystack.MODE_CSV
request = HaystackHttpRequest()
grid = Grid(columns={'filter': {}, 'limit': {}})
grid.append({'filter': '', 'limit': -1})
request.headers[
"Accept"] = "Accept:text/html,application/xhtml+xml," \
"application/xml;q=0.9," \
"image/webp,image/apng," \
"*/*;q=0.8," \
"application/signed-exchange;v=b3;q=0.9"
request.headers["Content-Type"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.read(envs, request, "dev")
# THEN
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
shaystack.parse(response.body, mime_type)
def test_negociation_with_zinc(no_cache) -> None:
# GIVEN
"""
Args:
no_cache:
"""
envs = {'HAYSTACK_PROVIDER': 'shaystack.providers.ping'}
no_cache.return_value = True
mime_type = shaystack.MODE_ZINC
request = HaystackHttpRequest()
grid = Grid(columns={'filter': {}, "limit": {}})
grid.append({"filter": "id==@me", "limit": 1})
request.headers["Content-Type"] = mime_type
request.headers["Accept"] = mime_type
request.body = shaystack.dump(grid, mode=mime_type)
# WHEN
response = shaystack.read(envs, request, "dev")
# THEN
assert response.status_code == 200
assert response.headers["Content-Type"].startswith(mime_type)
shaystack.parse(response.body, shaystack.MODE_ZINC)
