def test_update_grid():
try:
envs = {'HAYSTACK_DB': HAYSTACK_DB}
with cast(MongoProvider,
get_provider("shaystack.providers.mongodb",
envs)) as provider:
provider.purge_db()
provider.create_db()
grid = Grid(metadata={"dis": "hello"},
columns=[("id", {}), ("a", {
"dis": "a"
}), ("b", {
"dis": "b"
})])
grid.append({"id": Ref("1"), "a": "a", "b": "b"})
grid.append({"id": Ref("2"), "a": "c", "b": "d"})
provider.update_grid(grid, None, "customer", FAKE_NOW)
in_table = [
_conv_row_to_entity(row['entity'])
for row in provider.get_collection().find()
]
assert len(in_table) == len(grid)
assert in_table[0] == grid[0]
assert in_table[1] == grid[1]
except ServerSelectionTimeoutError as ex:
raise SkipTest("Mongo db not started") from ex
def test_import_ts_grid_in_db_with_a_lot_of_records(mock1, mock2):
mock1.return_value = "customer"
mock2.return_value = "customer"
envs = {'HAYSTACK_DB': HAYSTACK_DB,
'HAYSTACK_TS': HAYSTACK_TS,
'AWS_PROFILE': os.environ['AWS_PROFILE'],
'AWS_REGION': os.environ['AWS_REGION']
}
with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider:
# Check TS with all types
entity_id = Ref("abc")
# Insert an antity for the TS, with an attribut "kind"
provider.purge_db()
grid = Grid(columns=["id", "kind"])
grid.append({"id": entity_id, "kind": "Number"}) # Without "kind", the default is "Number" or "float"
version = datetime.datetime.now(tz=pytz.UTC)
provider.update_grid(diff_grid=grid, version=version, customer_id="customer")
# WARNING: timestream accept only datetime in the memory retention period.
# Not before and not after.
# It's not possible to extend the memory retention temporary to inject an old value
provider.purge_ts()
provider.create_db()
grid = Grid(columns=["ts", "val"])
# You must use utcnow() and a retention
for i in range(0, 200):
grid.append({"ts": datetime.datetime.utcnow().replace(microsecond=i * 1000), "val": i})
provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
def test_versions(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"}
with get_provider("shaystack.providers.url", envs) as _:
client = Client(schema)
executed = client.execute('''
{ haystack
{
versions
}
}
''')
assert executed == \
{'data':
{'haystack':
{'versions':
['2020-10-01T00:00:03+00:00 UTC',
'2020-10-01T00:00:02+00:00 UTC',
'2020-10-01T00:00:01+00:00 UTC']}}}
def test_entities_with_id(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"}
with get_provider("shaystack.providers.url", envs) as _:
client = Client(schema)
executed = client.execute('''
{ haystack
{
entities(ids:["@id1","@id2"])
}
}
''')
assert executed == \
{
'data':
{
'haystack':
{
'entities':
[
{'id': 'r:id1', 'col': 'n:1.000000', 'hisURI': 's:his0.zinc'},
{'id': 'r:id2', 'col': 'n:2.000000', 'hisURI': 's:his1.zinc'}
]
}
}
}
def _test_update_grid_in_db(provider_name: str, db: str):
with cast(
DBHaystackInterface,
get_provider(provider_name, {'HAYSTACK_DB': db},
use_cache=False)) as provider:
provider.purge_db()
provider.create_db()
left = Grid(columns={"id": {}, "a": {}, "b": {}, "c": {}})
left.append({"id": Ref("id1"), "a": 1, "b": 2})
left.append({"id": Ref("id2"), "a": 2, "b": 2})
left.append({"id": Ref("id3"), "a": 3, "b": 2})
left.append({"id": Ref("id4"), "a": 4, "b": 2})
left.append({"id": Ref("old_id"), "a": 1, "b": 2})
right = Grid(columns={"id": {}, "a": {}, "b": {}, "c": {}})
right.append({"id": Ref("id1"), "a": 3, "c": 5})
provider.update_grid(left,
version=None,
customer_id="customer",
now=FAKE_NOW)
next_fake = FAKE_NOW + datetime.timedelta(minutes=1)
provider.update_grid(right - left,
version=None,
customer_id="customer",
now=next_fake)
grid = provider.read_grid("customer", None)
assert len(grid) == 1, f"with {db}"
grid = provider.read_grid("customer", FAKE_NOW)
assert len(grid) == 5, f"with {db}"
def test_his_read_with_boolean(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
his = Grid(version=VER_3_0, columns=["ts", "val"])
his.extend([
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": MARKER
},
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": False
},
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": 1
},
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": 1.0
},
{
"ts": datetime(2020, 1, 1, tzinfo=pytz.utc),
"val": ""
},
])
mock_s3.return_value.history = his
mock_get_url.return_value = "s3://bucket/grid.zinc"
envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"}
with get_provider("shaystack.providers.url", envs) as _:
client = Client(schema)
executed = client.execute('''
{
haystack
{
histories(ids:"@id1")
{
ts
val
bool
}
}
}
''')
assert executed == \
{'data': {'haystack':
{'histories': [[{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'm:', 'bool': True},
{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': False,
'bool': False},
{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'n:1.000000',
'bool': True},
{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'n:1.000000',
'bool': True},
{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 's:',
'bool': False}]]}}}
def test_create_db():
try:
envs = {'HAYSTACK_DB': HAYSTACK_DB}
with cast(SQLProvider, get_provider("shaystack.providers.sql", envs,
use_cache=False)) as provider:
provider.create_db()
except psycopg2.OperationalError as ex:
raise SkipTest("Postgres db not started") from ex
def test_create_db():
envs = {'HAYSTACK_DB': HAYSTACK_DB,
'HAYSTACK_TS': HAYSTACK_TS,
'AWS_PROFILE': os.environ['AWS_PROFILE'],
'AWS_REGION': os.environ['AWS_REGION']
}
with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider:
provider.create_db()
def _test_version(provider_name: str, db: str):
with cast(
DBHaystackInterface,
get_provider(provider_name, {'HAYSTACK_DB': db},
use_cache=False)) as provider:
_populate_db(provider)
versions = provider.versions()
assert len(versions) == 3, f"with {db}"
def _test_values_for_tag_col(provider_name: str, db: str):
with cast(
DBHaystackInterface,
get_provider(provider_name, {'HAYSTACK_DB': db},
use_cache=False)) as provider:
_populate_db(provider)
values = provider.values_for_tag("col")
assert len(values) > 1, f"with {db}"
def _test_values_for_tag_dis(provider_name: str, db: str):
with cast(
DBHaystackInterface,
get_provider(provider_name, {'HAYSTACK_DB': db},
use_cache=False)) as provider:
_populate_db(provider)
values = provider.values_for_tag("dis")
assert values == ['Dis 1', 'Dis 2'], f"with {db}"
def _check_mongodb(
hs_filter: str, # pylint: disable=unused-argument,unused-variable
mongo_request: List[Dict[str, Any]]):
if os.environ.get('HAYSTACK_DB', '').startswith("mongodb"):
envs = {'HAYSTACK_DB': os.environ['HAYSTACK_DB']}
provider = cast(MongoProvider,
get_provider("shaystack.providers.mongodb", envs))
collection = provider.get_collection()
result = list(collection.aggregate(mongo_request)) # pylint: disable=unused-variable
def test_ops_without_implementation():
# GIVEN
provider = get_provider('tests.tstprovider_no_implementation', {})
# WHEN
ops = provider.ops()
# THEN
assert len(ops) == 2
def test_about():
envs = {'HAYSTACK_DB': HAYSTACK_DB,
'HAYSTACK_TS': HAYSTACK_TS,
'AWS_PROFILE': os.environ['AWS_PROFILE'],
'AWS_REGION': os.environ['AWS_REGION']
}
with get_provider("shaystack.providers.timestream", envs) as provider:
result = provider.about("http://localhost")
assert result[0]['moduleName'] == 'SQLProvider'
def test_about():
try:
with get_provider("shaystack.providers.sql",
{'HAYSTACK_DB': HAYSTACK_DB},
use_cache=False) as provider:
result = provider.about("http://localhost")
assert result[0]['moduleName'] == 'SQLProvider'
except psycopg2.OperationalError as ex:
raise SkipTest("Postgres db not started") from ex
def test_about(mock_get_url):
"""
Args:
mock_get_url:
"""
mock_get_url.return_value = "s3://bucket/grid.zinc"
with get_provider("shaystack.providers.url", {}) as provider:
result = provider.about("http://localhost")
assert result[0]['moduleName'] == 'URLProvider'
def _check_sqlite(sql_request: str) -> None:
if os.environ.get('HAYSTACK_DB', '').startswith("sqlite"):
envs = {'HAYSTACK_DB': os.environ['HAYSTACK_DB']}
provider = cast(SQLProvider,
get_provider("shaystack.providers.sql", envs))
conn = provider.get_connect()
try:
conn.execute(sql_request)
finally:
conn.rollback()
def test_ops_with_invoke_action():
# GIVEN
provider = get_provider('tests.tstprovider_invokeaction', {})
# WHEN
ops = provider.ops()
# THEN
assert len(ops) == 3
assert ops[2]['name'] == 'invokeAction'
def test_ops_with_nav():
# GIVEN
provider = get_provider('tests.tstprovider_nav', {})
# WHEN
ops = provider.ops()
# THEN
assert len(ops) == 3
assert ops[2]['name'] == 'nav'
def _check_pg(sql_request: str):
if os.environ.get('HAYSTACK_DB', '').startswith("postgres"):
envs = {'HAYSTACK_DB': os.environ['HAYSTACK_DB']}
provider = cast(SQLProvider,
get_provider("shaystack.providers.sql", envs))
conn = provider.get_connect()
try:
list(conn.cursor().execute(sql_request))
finally:
conn.rollback()
def test_ops_with_readonly():
# GIVEN
provider = get_provider('tests.tstprovider_readonly', {})
# WHEN
ops = provider.ops()
# THEN
assert len(ops) == 4
assert ops[2]['name'] == 'read'
assert ops[3]['name'] == 'hisRead'
def test_ops_with_writeonly():
# GIVEN
provider = get_provider('tests.tstprovider_write', {})
# WHEN
ops = provider.ops()
# THEN
assert len(ops) == 4
assert ops[2]['name'] == 'pointWrite'
assert ops[3]['name'] == 'hisWrite'
def test_version(mock, mock_get_url):
"""
Args:
mock:
mock_get_url:
"""
mock.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
with get_provider("shaystack.providers.url", {}) as provider:
versions = provider.versions()
assert len(versions) == 3
def test_ops_with_subscribe():
# GIVEN
provider = get_provider('tests.tstprovider_subscribe', {})
# WHEN
ops = provider.ops()
# THEN
assert len(ops) == 5
assert ops[2]['name'] == 'watchSub'
assert ops[3]['name'] == 'watchUnsub'
assert ops[4]['name'] == 'watchPoll'
def test_read_version_without_filter(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
with get_provider("shaystack.providers.url", {}) as provider:
version_2 = datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC)
result = provider.read(0, None, None, None, date_version=version_2)
assert result.metadata["v"] == "2"
def test_read_last_without_filter(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
with cast(URLProvider, get_provider("shaystack.providers.url",
{})) as provider:
provider.cache_clear()
result = provider.read(0, None, None, None, None)
assert result.metadata["v"] == "3"
def _test_read_version_with_filter_and_select(provider_name: str, db: str):
# caplog.set_level(logging.DEBUG)
with cast(
DBHaystackInterface,
get_provider(provider_name, {'HAYSTACK_DB': db},
use_cache=False)) as provider:
_populate_db(provider)
version_2 = datetime.datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC)
result = provider.read(0, "id,other", None, "id==@id1", version_2)
assert len(result) == 1, f"with {db}"
assert len(result.column) == 2, f"with {db}"
assert "id" in result.column, f"with {db}"
assert "other" in result.column, f"with {db}"
def test_values_for_tag(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
with get_provider("shaystack.providers.url", {}) as provider:
result = provider.values_for_tag("col")
assert result == [1.0, 2.0]
result = provider.values_for_tag("id")
assert result == [Ref("id1"), Ref("id2")]
def test_import_ts_grid_in_db_and_his_read(mock1, mock2):
mock1.return_value = "customer"
mock2.return_value = "customer"
envs = {'HAYSTACK_DB': HAYSTACK_DB,
'HAYSTACK_TS': HAYSTACK_TS,
'AWS_PROFILE': os.environ['AWS_PROFILE'],
'AWS_REGION': os.environ['AWS_REGION']
}
with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider:
values = [
(XStr("hex", "deadbeef"), "Str"),
("100", "Str"),
(100.0, "Number"), (Quantity(1, "m"), "Number"), (100, "Number"),
(True, "Bool"), (False, "Bool"),
(MARKER, "Marker"), (None, "Marker"),
(REMOVE, "Remove"), (None, "Remove"),
(NA, "NA"), (None, "NA"),
(Ref("abc"), "Ref"),
(datetime.datetime.utcnow().replace(microsecond=0), "DateTime"),
(datetime.date.today(), "Date"),
(datetime.datetime.utcnow().time(), "Time"),
(datetime.time(16, 58, 57, 994), "Time"),
(Coordinate(100.0, 200.0), "Coord"),
]
# Check TS with all types
entity_id = Ref("abc")
for val, kind in values:
# Clean DB for the specific kind
provider.purge_db()
provider.create_db()
# Insert an entity for the TS, with an attribut "kind"
grid = Grid(columns=["id", "kind"])
grid.append({"id": entity_id, "kind": kind}) # Without "kind", the default is "Number" or "float"
version = datetime.datetime.now(tz=pytz.UTC)
provider.update_grid(diff_grid=grid, version=version, customer_id="customer")
# WARNING: timestream accept only datetime in the memory retention period.
# Not before and not after.
log.debug("Test %s", type(val))
grid = Grid(columns=["ts", "val"])
# You must use utcnow() and a retention
grid.append({"ts": datetime.datetime.utcnow(), "val": val})
provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
grid_ts = provider.his_read(entity_id, parse_date_range("today", provider.get_tz()), None)
assert grid_ts[0]['val'] == val, f"with kind={kind} and val={val}"
def test_read_version_with_ids(mock_s3, mock_get_url):
"""
Args:
mock_s3:
mock_get_url:
"""
mock_s3.return_value = _get_mock_s3()
mock_get_url.return_value = "s3://bucket/grid.zinc"
with get_provider("shaystack.providers.url", {}) as provider:
version_2 = datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC)
result = provider.read(0, None, [Ref("id1")], None, version_2)
assert result.metadata["v"] == "2"
assert len(result) == 1
assert result[0]['id'] == Ref("id1")
