def test_verify_dynamic_mapping(self):
doc1 = {
"manifest": {
"data": "hello world!"
},
"description":
"Scooby dooby do, where are you, we got some work to do now.",
"time1": "2017-11-02T09:50:20.123123Z",
"time2": "2017-11-02 09:55:12",
"time3": "2017-11-02",
}
bundle_uuid = str(uuid.uuid4())
version = get_version()
bundle_fqid = f"{bundle_uuid}.{version}"
es_client = ElasticsearchClient.get()
es_client.index(index=self.dss_index_name,
doc_type=ESDocType.doc.name,
id=bundle_fqid,
body=doc1)
mapping = es_client.indices.get_mapping(
self.dss_index_name)[self.dss_index_name]['mappings']
self.assertEqual(mapping['query']['properties']['query']['type'],
'percolator')
self.assertEqual(mapping['doc']['properties']['description']['type'],
'keyword')
self.assertEqual(
mapping['doc']['properties']['description']['fields']['text']
['type'], 'text')
self.assertEqual(mapping['doc']['properties']['time1']['type'], 'date')
self.assertEqual(mapping['doc']['properties']['time2']['type'], 'date')
self.assertEqual(mapping['doc']['properties']['time3']['type'], 'date')
def test_missing_library(self):
lambda_client = self.get_client()
response = lambda_client.invoke(
FunctionName=get_function_name("MissingLibraryFunction"),
Payload=json.dumps({'y': 1}),
)
raw_payload = response['Payload'].read().decode('utf-8')
json_payload = json.loads(raw_payload)
self.assertEqual('Unhandled', response['FunctionError'])
self.assertIn('there is no package called ‘Matrix’',
json_payload['errorMessage'])
error_type = 'packageNotFoundError' if get_version(
) == '3_6_0' else 'simpleError'
self.assertEqual(error_type, json_payload['errorType'])
def populate_search_index(self, index_document: dict, count: int) -> list:
es_client = ElasticsearchClient.get()
bundles = []
for i in range(count):
bundle_uuid = str(uuid.uuid4())
version = get_version()
index_document['manifest']['version'] = version
bundle_fqid = f"{bundle_uuid}.{version}"
bundle_url = (
f"https://127.0.0.1:{self.app._port}"
f"/v1/bundles/{bundle_uuid}?version={version}&replica={self.replica.name}"
)
es_client.index(index=self.dss_index_name,
doc_type=ESDocType.doc.name,
id=bundle_fqid,
body=index_document,
refresh=(i == count - 1))
bundles.append((bundle_fqid, bundle_url))
return bundles
def setUpClass(cls):
if is_local():
cls.api = start_local_api(
template_path="test-template.yaml",
parameter_overrides={'Version': get_version()},
)
def test_dependencies_exist(self):
file_uuid, file_version = str(uuid.uuid4()), get_version()
bundle_uuid, bundle_version = str(uuid.uuid4()), get_version()
collection_data = {
"contents": [{
"type": "bundle",
"uuid": bundle_uuid,
"version": bundle_version
}, {
"type": "file",
"uuid": file_uuid,
"version": file_version
}]
}
bundle_data = {
BundleMetadata.FILES: [{
BundleFileMetadata.UUID: file_uuid,
BundleFileMetadata.VERSION: file_version
}]
}
file_data = {
FileMetadata.SHA256: "sync_test",
FileMetadata.SHA1: "sync_test",
FileMetadata.S3_ETAG: "sync_test",
FileMetadata.CRC32C: str(uuid.uuid4())
}
with self.subTest("collection without deps"):
collection_key = "{}/{}".format(COLLECTION_PREFIX,
get_collection_fqid())
collection_blob = self.s3_bucket.Object(collection_key)
collection_blob.put(Body=json.dumps(collection_data).encode())
self.assertFalse(
sync.dependencies_exist(Replica.aws, Replica.aws,
collection_key))
with self.subTest("bundle without deps"):
bundle_key = "{}/{}".format(
BUNDLE_PREFIX,
BundleFQID(uuid=bundle_uuid, version=bundle_version))
bundle_blob = self.s3_bucket.Object(bundle_key)
bundle_blob.put(Body=json.dumps(bundle_data).encode())
self.assertFalse(
sync.dependencies_exist(Replica.aws, Replica.aws,
collection_key))
self.assertFalse(
sync.dependencies_exist(Replica.aws, Replica.aws, bundle_key))
with self.subTest("file without deps"):
file_key = "{}/{}".format(
FILE_PREFIX, FileFQID(uuid=file_uuid, version=file_version))
file_blob = self.s3_bucket.Object(file_key)
file_blob.put(Body=json.dumps(file_data).encode())
@eventually(timeout=8, interval=1, errors={Exception})
def check_file_revdeps():
self.assertTrue(
sync.dependencies_exist(Replica.aws, Replica.aws,
collection_key))
self.assertTrue(
sync.dependencies_exist(Replica.aws, Replica.aws,
bundle_key))
self.assertFalse(
sync.dependencies_exist(Replica.aws, Replica.aws,
file_key))
check_file_revdeps()
with self.subTest(
"blob presence causes all dependencies to be resolved"):
blob_key = compose_blob_key(file_data)
blob_blob = self.s3_bucket.Object(blob_key)
blob_blob.put(Body=b"sync_test")
@eventually(timeout=8, interval=1, errors={Exception})
def check_blob_revdeps():
self.assertTrue(
sync.dependencies_exist(Replica.aws, Replica.aws,
collection_key))
self.assertTrue(
sync.dependencies_exist(Replica.aws, Replica.aws,
bundle_key))
self.assertTrue(
sync.dependencies_exist(Replica.aws, Replica.aws,
file_key))
check_blob_revdeps()