def mocked_dsg(dataset_file, configuration_data, mocker):
with mock_sts():
config = Config()
config.config = configuration_data
dataset_group = config.dataset_group(dataset_file)
dsg = DatasetGroup(
dataset_group_name=dataset_group.dataset_group_name,
dataset_domain=dataset_group.dataset_group_domain,
)
dsg.cli = mocker.MagicMock()
dsg.cli.describe_dataset_group.return_value = {
"DatasetArns": ["arn::1", "arn::2", "arn::3"]
}
dsg.cli.describe_dataset.return_value = {"DatasetArn": 'arn::1', "Status": "ACTIVE", 'LastModificationTime': datetime.now()}
dsg.cli.get_paginator().paginate.return_value = [{
'DatasetImportJobs': [
{
"DatasetImportJobArn": f"arn::{i}",
"Status": "ACTIVE",
"LastModificationTime": datetime.now()
}
for i in range(3)
]
}]
yield dsg
def _valid_dataset_group(self, config_key, resource, config_data, errors):
try:
DatasetGroup.validate_config(DatasetGroupName="placeholder",
**config_data)
except ParamValidationError as excinfo:
errors.append(
f"configuration issue for {config_key}.{resource}: {str(excinfo)}"
)
def test_status(dataset_file, configuration_data, mock_forecast_dsg_exists):
config = Config()
config.config = configuration_data
dataset_group = config.dataset_group(dataset_file)
dsg = DatasetGroup(
dataset_group_name=dataset_group.dataset_group_name,
dataset_domain=dataset_group.dataset_group_domain,
)
dsg.cli = mock_forecast_dsg_exists
assert dsg.status == Status.ACTIVE
def dataset_groups(self, dataset_file: DatasetFile) -> List[DatasetGroup]:
"""
Get the dataset groups that depend on this file
:param dataset_file: The dataset file to use
:return: The dataset Groups
"""
dsg_names = self.dependent_dataset_groups(dataset_file)
dsgs = []
logger.info("%s depend on %s" %
(", ".join(dsg_names), dataset_file.prefix))
for dsg_name in dsg_names:
dsg = DatasetGroup(
dataset_group_name=dsg_name,
dataset_domain=self.dataset_group_domain(dataset_file),
)
ds = self.dataset(dataset_file)
if ds.dataset_domain != dsg.dataset_group_domain:
raise ValueError(
f"The dataset group domain ({dsg.dataset_group_domain}) and dataset domain ({ds.dataset_domain}) must match."
)
dsgs.append(dsg)
return dsgs
def dataset_group(self,
dataset_file: DatasetFile,
dataset_group_name: str = None):
"""
Get the dataset group
:param dataset_file: The dataset file to use
:param dataset_group_name:
:return: The dataset group
"""
name = dataset_group_name if dataset_group_name else dataset_file.prefix
domain = self.dataset_group_domain(dataset_file, dataset_group_name)
dsg = DatasetGroup(
dataset_group_name=name,
dataset_domain=domain,
)
ds = self.dataset(dataset_file)
if ds.dataset_domain != dsg.dataset_group_domain:
raise ValueError(
f"The dataset group domain ({dsg.dataset_group_domain}) and dataset domain ({ds.dataset_domain}) must match."
)
return dsg
def test_create(dataset_file, configuration_data):
config = Config()
config.config = configuration_data
dataset_group = config.dataset_group(dataset_file)
dsg = DatasetGroup(
dataset_group_name=dataset_group.dataset_group_name,
dataset_domain=dataset_group.dataset_group_domain,
)
assert dsg.arn == "arn:aws:forecast:us-east-1:abcdefghijkl:dataset-group/data"
def test_schema(domain, identifier, metric, fields):
dsg = DatasetGroup(
dataset_domain=DatasetDomain[domain],
dataset_group_name=DatasetGroupName("some_name"),
)
assert dsg.schema.dataset_group == dsg
assert dsg.schema.identifier == identifier
assert dsg.schema.metric == metric
for field in dsg.schema.fields:
assert field in fields
def schema(request):
with mock_sts():
dsg = DatasetGroup(DatasetGroupName("any"), request.param)
dsg_schema = dsg.schema
user_attributes = []
for field in dsg_schema.fields:
if field == dsg_schema.metric:
typ = "float"
elif field == dsg_schema.identifier:
typ = "string"
elif field == "timestamp":
typ = "timestamp"
else:
raise ValueError(
f"Could not prepare attributes! Unexpected attribute {field}"
)
user_attributes.append({
"AttributeName": field,
"AttributeType": typ
})
yield dsg_schema, user_attributes