def test_update_grid(): try: envs = {'HAYSTACK_DB': HAYSTACK_DB} with cast(MongoProvider, get_provider("shaystack.providers.mongodb", envs)) as provider: provider.purge_db() provider.create_db() grid = Grid(metadata={"dis": "hello"}, columns=[("id", {}), ("a", { "dis": "a" }), ("b", { "dis": "b" })]) grid.append({"id": Ref("1"), "a": "a", "b": "b"}) grid.append({"id": Ref("2"), "a": "c", "b": "d"}) provider.update_grid(grid, None, "customer", FAKE_NOW) in_table = [ _conv_row_to_entity(row['entity']) for row in provider.get_collection().find() ] assert len(in_table) == len(grid) assert in_table[0] == grid[0] assert in_table[1] == grid[1] except ServerSelectionTimeoutError as ex: raise SkipTest("Mongo db not started") from ex
def test_import_ts_grid_in_db_with_a_lot_of_records(mock1, mock2): mock1.return_value = "customer" mock2.return_value = "customer" envs = {'HAYSTACK_DB': HAYSTACK_DB, 'HAYSTACK_TS': HAYSTACK_TS, 'AWS_PROFILE': os.environ['AWS_PROFILE'], 'AWS_REGION': os.environ['AWS_REGION'] } with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider: # Check TS with all types entity_id = Ref("abc") # Insert an antity for the TS, with an attribut "kind" provider.purge_db() grid = Grid(columns=["id", "kind"]) grid.append({"id": entity_id, "kind": "Number"}) # Without "kind", the default is "Number" or "float" version = datetime.datetime.now(tz=pytz.UTC) provider.update_grid(diff_grid=grid, version=version, customer_id="customer") # WARNING: timestream accept only datetime in the memory retention period. # Not before and not after. # It's not possible to extend the memory retention temporary to inject an old value provider.purge_ts() provider.create_db() grid = Grid(columns=["ts", "val"]) # You must use utcnow() and a retention for i in range(0, 200): grid.append({"ts": datetime.datetime.utcnow().replace(microsecond=i * 1000), "val": i}) provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW)
def test_versions(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"} with get_provider("shaystack.providers.url", envs) as _: client = Client(schema) executed = client.execute(''' { haystack { versions } } ''') assert executed == \ {'data': {'haystack': {'versions': ['2020-10-01T00:00:03+00:00 UTC', '2020-10-01T00:00:02+00:00 UTC', '2020-10-01T00:00:01+00:00 UTC']}}}
def test_entities_with_id(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"} with get_provider("shaystack.providers.url", envs) as _: client = Client(schema) executed = client.execute(''' { haystack { entities(ids:["@id1","@id2"]) } } ''') assert executed == \ { 'data': { 'haystack': { 'entities': [ {'id': 'r:id1', 'col': 'n:1.000000', 'hisURI': 's:his0.zinc'}, {'id': 'r:id2', 'col': 'n:2.000000', 'hisURI': 's:his1.zinc'} ] } } }
def _test_update_grid_in_db(provider_name: str, db: str): with cast( DBHaystackInterface, get_provider(provider_name, {'HAYSTACK_DB': db}, use_cache=False)) as provider: provider.purge_db() provider.create_db() left = Grid(columns={"id": {}, "a": {}, "b": {}, "c": {}}) left.append({"id": Ref("id1"), "a": 1, "b": 2}) left.append({"id": Ref("id2"), "a": 2, "b": 2}) left.append({"id": Ref("id3"), "a": 3, "b": 2}) left.append({"id": Ref("id4"), "a": 4, "b": 2}) left.append({"id": Ref("old_id"), "a": 1, "b": 2}) right = Grid(columns={"id": {}, "a": {}, "b": {}, "c": {}}) right.append({"id": Ref("id1"), "a": 3, "c": 5}) provider.update_grid(left, version=None, customer_id="customer", now=FAKE_NOW) next_fake = FAKE_NOW + datetime.timedelta(minutes=1) provider.update_grid(right - left, version=None, customer_id="customer", now=next_fake) grid = provider.read_grid("customer", None) assert len(grid) == 1, f"with {db}" grid = provider.read_grid("customer", FAKE_NOW) assert len(grid) == 5, f"with {db}"
def test_his_read_with_boolean(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() his = Grid(version=VER_3_0, columns=["ts", "val"]) his.extend([ { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": MARKER }, { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": False }, { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": 1 }, { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": 1.0 }, { "ts": datetime(2020, 1, 1, tzinfo=pytz.utc), "val": "" }, ]) mock_s3.return_value.history = his mock_get_url.return_value = "s3://bucket/grid.zinc" envs = {'HAYSTACK_PROVIDER': "shaystack.providers.url"} with get_provider("shaystack.providers.url", envs) as _: client = Client(schema) executed = client.execute(''' { haystack { histories(ids:"@id1") { ts val bool } } } ''') assert executed == \ {'data': {'haystack': {'histories': [[{'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'm:', 'bool': True}, {'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': False, 'bool': False}, {'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'n:1.000000', 'bool': True}, {'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 'n:1.000000', 'bool': True}, {'ts': '2020-01-01T00:00:00+00:00 UTC', 'val': 's:', 'bool': False}]]}}}
def test_create_db(): try: envs = {'HAYSTACK_DB': HAYSTACK_DB} with cast(SQLProvider, get_provider("shaystack.providers.sql", envs, use_cache=False)) as provider: provider.create_db() except psycopg2.OperationalError as ex: raise SkipTest("Postgres db not started") from ex
def test_create_db(): envs = {'HAYSTACK_DB': HAYSTACK_DB, 'HAYSTACK_TS': HAYSTACK_TS, 'AWS_PROFILE': os.environ['AWS_PROFILE'], 'AWS_REGION': os.environ['AWS_REGION'] } with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider: provider.create_db()
def _test_version(provider_name: str, db: str): with cast( DBHaystackInterface, get_provider(provider_name, {'HAYSTACK_DB': db}, use_cache=False)) as provider: _populate_db(provider) versions = provider.versions() assert len(versions) == 3, f"with {db}"
def _test_values_for_tag_col(provider_name: str, db: str): with cast( DBHaystackInterface, get_provider(provider_name, {'HAYSTACK_DB': db}, use_cache=False)) as provider: _populate_db(provider) values = provider.values_for_tag("col") assert len(values) > 1, f"with {db}"
def _test_values_for_tag_dis(provider_name: str, db: str): with cast( DBHaystackInterface, get_provider(provider_name, {'HAYSTACK_DB': db}, use_cache=False)) as provider: _populate_db(provider) values = provider.values_for_tag("dis") assert values == ['Dis 1', 'Dis 2'], f"with {db}"
def _check_mongodb( hs_filter: str, # pylint: disable=unused-argument,unused-variable mongo_request: List[Dict[str, Any]]): if os.environ.get('HAYSTACK_DB', '').startswith("mongodb"): envs = {'HAYSTACK_DB': os.environ['HAYSTACK_DB']} provider = cast(MongoProvider, get_provider("shaystack.providers.mongodb", envs)) collection = provider.get_collection() result = list(collection.aggregate(mongo_request)) # pylint: disable=unused-variable
def test_ops_without_implementation(): # GIVEN provider = get_provider('tests.tstprovider_no_implementation', {}) # WHEN ops = provider.ops() # THEN assert len(ops) == 2
def test_about(): envs = {'HAYSTACK_DB': HAYSTACK_DB, 'HAYSTACK_TS': HAYSTACK_TS, 'AWS_PROFILE': os.environ['AWS_PROFILE'], 'AWS_REGION': os.environ['AWS_REGION'] } with get_provider("shaystack.providers.timestream", envs) as provider: result = provider.about("http://localhost") assert result[0]['moduleName'] == 'SQLProvider'
def test_about(): try: with get_provider("shaystack.providers.sql", {'HAYSTACK_DB': HAYSTACK_DB}, use_cache=False) as provider: result = provider.about("http://localhost") assert result[0]['moduleName'] == 'SQLProvider' except psycopg2.OperationalError as ex: raise SkipTest("Postgres db not started") from ex
def test_about(mock_get_url): """ Args: mock_get_url: """ mock_get_url.return_value = "s3://bucket/grid.zinc" with get_provider("shaystack.providers.url", {}) as provider: result = provider.about("http://localhost") assert result[0]['moduleName'] == 'URLProvider'
def _check_sqlite(sql_request: str) -> None: if os.environ.get('HAYSTACK_DB', '').startswith("sqlite"): envs = {'HAYSTACK_DB': os.environ['HAYSTACK_DB']} provider = cast(SQLProvider, get_provider("shaystack.providers.sql", envs)) conn = provider.get_connect() try: conn.execute(sql_request) finally: conn.rollback()
def test_ops_with_invoke_action(): # GIVEN provider = get_provider('tests.tstprovider_invokeaction', {}) # WHEN ops = provider.ops() # THEN assert len(ops) == 3 assert ops[2]['name'] == 'invokeAction'
def test_ops_with_nav(): # GIVEN provider = get_provider('tests.tstprovider_nav', {}) # WHEN ops = provider.ops() # THEN assert len(ops) == 3 assert ops[2]['name'] == 'nav'
def _check_pg(sql_request: str): if os.environ.get('HAYSTACK_DB', '').startswith("postgres"): envs = {'HAYSTACK_DB': os.environ['HAYSTACK_DB']} provider = cast(SQLProvider, get_provider("shaystack.providers.sql", envs)) conn = provider.get_connect() try: list(conn.cursor().execute(sql_request)) finally: conn.rollback()
def test_ops_with_readonly(): # GIVEN provider = get_provider('tests.tstprovider_readonly', {}) # WHEN ops = provider.ops() # THEN assert len(ops) == 4 assert ops[2]['name'] == 'read' assert ops[3]['name'] == 'hisRead'
def test_ops_with_writeonly(): # GIVEN provider = get_provider('tests.tstprovider_write', {}) # WHEN ops = provider.ops() # THEN assert len(ops) == 4 assert ops[2]['name'] == 'pointWrite' assert ops[3]['name'] == 'hisWrite'
def test_version(mock, mock_get_url): """ Args: mock: mock_get_url: """ mock.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" with get_provider("shaystack.providers.url", {}) as provider: versions = provider.versions() assert len(versions) == 3
def test_ops_with_subscribe(): # GIVEN provider = get_provider('tests.tstprovider_subscribe', {}) # WHEN ops = provider.ops() # THEN assert len(ops) == 5 assert ops[2]['name'] == 'watchSub' assert ops[3]['name'] == 'watchUnsub' assert ops[4]['name'] == 'watchPoll'
def test_read_version_without_filter(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" with get_provider("shaystack.providers.url", {}) as provider: version_2 = datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC) result = provider.read(0, None, None, None, date_version=version_2) assert result.metadata["v"] == "2"
def test_read_last_without_filter(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" with cast(URLProvider, get_provider("shaystack.providers.url", {})) as provider: provider.cache_clear() result = provider.read(0, None, None, None, None) assert result.metadata["v"] == "3"
def _test_read_version_with_filter_and_select(provider_name: str, db: str): # caplog.set_level(logging.DEBUG) with cast( DBHaystackInterface, get_provider(provider_name, {'HAYSTACK_DB': db}, use_cache=False)) as provider: _populate_db(provider) version_2 = datetime.datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC) result = provider.read(0, "id,other", None, "id==@id1", version_2) assert len(result) == 1, f"with {db}" assert len(result.column) == 2, f"with {db}" assert "id" in result.column, f"with {db}" assert "other" in result.column, f"with {db}"
def test_values_for_tag(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" with get_provider("shaystack.providers.url", {}) as provider: result = provider.values_for_tag("col") assert result == [1.0, 2.0] result = provider.values_for_tag("id") assert result == [Ref("id1"), Ref("id2")]
def test_import_ts_grid_in_db_and_his_read(mock1, mock2): mock1.return_value = "customer" mock2.return_value = "customer" envs = {'HAYSTACK_DB': HAYSTACK_DB, 'HAYSTACK_TS': HAYSTACK_TS, 'AWS_PROFILE': os.environ['AWS_PROFILE'], 'AWS_REGION': os.environ['AWS_REGION'] } with cast(DBTSProvider, get_provider("shaystack.providers.timestream", envs)) as provider: values = [ (XStr("hex", "deadbeef"), "Str"), ("100", "Str"), (100.0, "Number"), (Quantity(1, "m"), "Number"), (100, "Number"), (True, "Bool"), (False, "Bool"), (MARKER, "Marker"), (None, "Marker"), (REMOVE, "Remove"), (None, "Remove"), (NA, "NA"), (None, "NA"), (Ref("abc"), "Ref"), (datetime.datetime.utcnow().replace(microsecond=0), "DateTime"), (datetime.date.today(), "Date"), (datetime.datetime.utcnow().time(), "Time"), (datetime.time(16, 58, 57, 994), "Time"), (Coordinate(100.0, 200.0), "Coord"), ] # Check TS with all types entity_id = Ref("abc") for val, kind in values: # Clean DB for the specific kind provider.purge_db() provider.create_db() # Insert an entity for the TS, with an attribut "kind" grid = Grid(columns=["id", "kind"]) grid.append({"id": entity_id, "kind": kind}) # Without "kind", the default is "Number" or "float" version = datetime.datetime.now(tz=pytz.UTC) provider.update_grid(diff_grid=grid, version=version, customer_id="customer") # WARNING: timestream accept only datetime in the memory retention period. # Not before and not after. log.debug("Test %s", type(val)) grid = Grid(columns=["ts", "val"]) # You must use utcnow() and a retention grid.append({"ts": datetime.datetime.utcnow(), "val": val}) provider._import_ts_in_db(grid, entity_id, "customer", FAKE_NOW) grid_ts = provider.his_read(entity_id, parse_date_range("today", provider.get_tz()), None) assert grid_ts[0]['val'] == val, f"with kind={kind} and val={val}"
def test_read_version_with_ids(mock_s3, mock_get_url): """ Args: mock_s3: mock_get_url: """ mock_s3.return_value = _get_mock_s3() mock_get_url.return_value = "s3://bucket/grid.zinc" with get_provider("shaystack.providers.url", {}) as provider: version_2 = datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC) result = provider.read(0, None, [Ref("id1")], None, version_2) assert result.metadata["v"] == "2" assert len(result) == 1 assert result[0]['id'] == Ref("id1")