def test_HtmlSiteStore_S3_backend(): bucket = "test_validation_store_bucket" prefix = "test/prefix" # create a bucket in Moto's mock AWS environment conn = boto3.resource("s3", region_name="us-east-1") conn.create_bucket(Bucket=bucket) my_store = HtmlSiteStore(store_backend={ "class_name": "TupleS3StoreBackend", "bucket": bucket, "prefix": prefix, }) with pytest.raises(TypeError): my_store.get("not_a_ValidationResultIdentifier") ns_1 = SiteSectionIdentifier( site_section_name="validations", resource_identifier=ValidationResultIdentifier( expectation_suite_identifier=ExpectationSuiteIdentifier( expectation_suite_name="asset.quarantine", ), run_id="20191007T151224.1234Z_prod_100", batch_identifier="1234", ), ) my_store.set(ns_1, "aaa") ns_2 = SiteSectionIdentifier( site_section_name="expectations", resource_identifier=ExpectationSuiteIdentifier( expectation_suite_name="asset.quarantine", ), ) my_store.set(ns_2, "bbb") assert set(my_store.list_keys()) == { ns_1.resource_identifier, ns_2.resource_identifier, } # This is a special un-store-like method exposed by the HtmlSiteStore my_store.write_index_page("index_html_string_content") # Verify that internals are working as expected, including the default filepath # paths below should include the batch_parameters assert { s3_object_info["Key"] for s3_object_info in boto3.client("s3").list_objects_v2( Bucket=bucket, Prefix=prefix)["Contents"] } == { "test/prefix/index.html", "test/prefix/expectations/asset/quarantine.html", "test/prefix/validations/asset/quarantine/20191007T151224.1234Z_prod_100/20190926T134241.000000Z/1234.html", } index_content = (boto3.client("s3").get_object( Bucket=bucket, Key="test/prefix/index.html")["Body"].read().decode("utf-8")) assert index_content == "index_html_string_content"
def test_HtmlSiteStore_filesystem_backend(tmp_path_factory): path = str(tmp_path_factory.mktemp('test_HtmlSiteStore_with_TupleFileSystemStoreBackend__dir')) my_store = HtmlSiteStore( store_backend={ "class_name": "TupleFilesystemStoreBackend", "base_directory": "my_store" }, runtime_environment={ "root_directory": path } ) with pytest.raises(TypeError): my_store.get("not_a_ValidationResultIdentifier") with pytest.raises(ValidationError): my_store.get(validationResultIdentifierSchema.load({}).data) ns_1 = SiteSectionIdentifier( site_section_name="validations", resource_identifier=ValidationResultIdentifier.from_tuple(('a', 'b', 'c', 'quarantine', 'prod-100')) ) my_store.set(ns_1, "aaa") # assert my_store.get(ns_1) == "aaa" ns_2 = SiteSectionIdentifier( site_section_name="validations", resource_identifier=ValidationResultIdentifier.from_tuple(('a', 'b', 'c', 'quarantine', 'prod-20')) ) my_store.set(ns_2, "bbb") # assert my_store.get(ns_2) == {"B": "bbb"} print(my_store.list_keys()) # WARNING: OBSERVE THAT SITE_SECTION_NAME IS LOST IN THE CALL TO LIST_KEYS assert set(my_store.list_keys()) == { ns_1.resource_identifier, ns_2.resource_identifier, } print(gen_directory_tree_str(path)) assert gen_directory_tree_str(path) == """\
def _convert_tuple_to_resource_identifier(self, tuple_): if tuple_[0] == "expectations": resource_identifier = ExpectationSuiteIdentifier(*tuple_[1]) elif tuple_[0] == "validations": resource_identifier = ValidationResultIdentifier(*tuple_[1]) else: raise Exception("unknown section name: " + tuple_[0]) new_identifier = SiteSectionIdentifier( site_section_name=tuple_[0], resource_identifier=resource_identifier) return new_identifier
def test_HtmlSiteStore_filesystem_backend(tmp_path_factory): path = str( tmp_path_factory.mktemp( "test_HtmlSiteStore_with_TupleFileSystemStoreBackend__dir")) my_store = HtmlSiteStore( store_backend={ "class_name": "TupleFilesystemStoreBackend", "base_directory": "my_store", }, runtime_environment={"root_directory": path}, ) with pytest.raises(TypeError): my_store.get("not_a_ValidationResultIdentifier") with pytest.raises(ValidationError): my_store.get(validationResultIdentifierSchema.load({})) ns_1 = SiteSectionIdentifier( site_section_name="validations", resource_identifier=ValidationResultIdentifier.from_tuple(( "a", "b", "c", "quarantine", datetime.datetime.now(datetime.timezone.utc), "prod-100", )), ) my_store.set(ns_1, "aaa") # assert my_store.get(ns_1) == "aaa" ns_2 = SiteSectionIdentifier( site_section_name="validations", resource_identifier=ValidationResultIdentifier.from_tuple(( "a", "b", "c", "quarantine", datetime.datetime.now(datetime.timezone.utc), "prod-20", )), ) my_store.set(ns_2, "bbb") # assert my_store.get(ns_2) == {"B": "bbb"} print(my_store.list_keys()) # WARNING: OBSERVE THAT SITE_SECTION_NAME IS LOST IN THE CALL TO LIST_KEYS assert set(my_store.list_keys()) == { ns_1.resource_identifier, ns_2.resource_identifier, } print(gen_directory_tree_str(path)) assert (gen_directory_tree_str(path) == """\ test_HtmlSiteStore_with_TupleFileSystemStoreBackend__dir0/ my_store/ validations/ a/ b/ c/ quarantine/ 20190926T134241.000000Z/ prod-100.html prod-20.html """)