def test_missing_manifest_files(self):
"""Ensures that if you try to read values from missing files, the right exceptions get raised"""
twine = Twine(source=self.VALID_MANIFEST_STRAND)
file = os.path.join(self.path, "not_a_file.json")
with self.assertRaises(exceptions.ConfigurationManifestFileNotFound):
twine.validate_configuration_manifest(source=file)
with self.assertRaises(exceptions.InputManifestFileNotFound):
twine.validate_input_manifest(source=file)
with self.assertRaises(exceptions.OutputManifestFileNotFound):
twine.validate_output_manifest(source=file)
def test_valid_manifest_files(self):
"""Ensures that a manifest file will validate."""
valid_configuration_manifest = """
{
"id": "3ead7669-8162-4f64-8cd5-4abe92509e17",
"datasets": {
"configuration_files_data": {
"id": "34ad7669-8162-4f64-8cd5-4abe92509e17",
"name": "configuration_files_data",
"tags": {},
"labels": ["the", "config", "labels"],
"files": [
{
"path": "configuration/datasets/7ead7669/file_1.csv",
"cluster": 0,
"sequence": 0,
"extension": "csv",
"tags": {},
"labels": [],
"posix_timestamp": 0,
"id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
"last_modified": "2019-02-28T22:40:30.533005Z",
"name": "file_1.csv",
"size_bytes": 59684813,
"sha-512/256": "somesha"
},
{
"path": "configuration/datasets/7ead7669/file_2.csv",
"cluster": 0,
"sequence": 1,
"extension": "csv",
"tags": {},
"labels": [],
"posix_timestamp": 0,
"id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45",
"last_modified": "2019-02-28T22:40:40.633001Z",
"name": "file_2.csv",
"size_bytes": 59684813,
"sha-512/256": "someothersha"
}
]
}
}
}
"""
valid_input_manifest = """
{
"id": "8ead7669-8162-4f64-8cd5-4abe92509e17",
"datasets": {
"met_mast_data": {
"id": "7ead7669-8162-4f64-8cd5-4abe92509e17",
"name": "met_mast_data",
"tags": {},
"labels": ["met", "mast", "wind"],
"files": [
{
"path": "input/datasets/7ead7669/file_1.csv",
"cluster": 0,
"sequence": 0,
"extension": "csv",
"tags": {},
"labels": [],
"posix_timestamp": 0,
"id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
"last_modified": "2019-02-28T22:40:30.533005Z",
"name": "file_1.csv",
"size_bytes": 59684813,
"sha-512/256": "somesha"
},
{
"path": "input/datasets/7ead7669/file_2.csv",
"cluster": 0,
"sequence": 1,
"extension": "csv",
"tags": {},
"labels": [],
"posix_timestamp": 0,
"id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45",
"last_modified": "2019-02-28T22:40:40.633001Z",
"name": "file_2.csv",
"size_bytes": 59684813,
"sha-512/256": "someothersha"
}
]
},
"scada_data": "gs://my-bucket/scada-data"
}
}
"""
valid_output_manifest = """
{
"id": "2ead7669-8162-4f64-8cd5-4abe92509e17",
"datasets": {
"output_files_data": {
"id": "1ead7669-8162-4f64-8cd5-4abe92509e17",
"name": "output_files_data",
"tags": {},
"labels": ["the", "output", "labels"],
"files": [
{
"path": "input/datasets/7ead7669/file_1.csv",
"cluster": 0,
"sequence": 0,
"extension": "csv",
"tags": {},
"labels": [],
"posix_timestamp": 0,
"id": "abff07bc-7c19-4ed5-be6d-a6546eae8e86",
"last_modified": "2019-02-28T22:40:30.533005Z",
"name": "file_1.csv",
"size_bytes": 59684813,
"sha-512/256": "somesha"
},
{
"path": "input/datasets/7ead7669/file_2.csv",
"cluster": 0,
"sequence": 1,
"extension": "csv",
"tags": {},
"labels": [],
"posix_timestamp": 0,
"id": "bbff07bc-7c19-4ed5-be6d-a6546eae8e45",
"last_modified": "2019-02-28T22:40:40.633001Z",
"name": "file_2.csv",
"size_bytes": 59684813,
"sha-512/256": "someothersha"
}
]
}
}
}
"""
twine = Twine(source=self.VALID_MANIFEST_STRAND)
twine.validate_configuration_manifest(source=valid_configuration_manifest)
twine.validate_input_manifest(source=valid_input_manifest)
twine.validate_output_manifest(source=valid_output_manifest)