def generate_datasets(spec_path):
try:
generator = \
GeneratorsHandler(arguments={'config_file': spec_path})
for dset_name, dset_format, dset_path in generator.generate():
click.echo("| Finished!\n"
f"| Dataset name: {dset_name}\n"
f"| Dataset format: {dset_format}\n"
f"| Dataset: {dset_path}\n")
except Exception as err:
click.echo(f"Error: {err}")
def test_timestamp_dataframe(self, mocker, timestamp_specification):
mock = mocker.patch \
.object(GeneratorsHandler, 'valid_specification_dataset')
mock.return_value = timestamp_specification
handler = GeneratorsHandler({"config_file": None})
specification = handler.valid_specification_dataset()
dataframe = handler.generate_dataframe(specification)
assert isinstance(dataframe, DataFrame) is True
assert dataframe.shape[0] == timestamp_specification['size']
for field in timestamp_specification['fields']:
assert dataframe[field['name']].dtype.name == field['expected']
def test_bool_dataframe(self, mocker, bool_specification):
with patch('builtins.open',
mock_open(read_data=json.dumps(bool_specification))):
dataset = bool_specification['datasets']['sample']
expected_size = dataset['size']
expected_fields = dataset['fields']
handler = GeneratorsHandler({"config_file": None})
dataframe = handler.generate_dataframe(dataset)
assert isinstance(dataframe, DataFrame) is True
assert dataframe.shape[0] == expected_size
for field in expected_fields:
assert dataframe[field['name']].dtype.name == field['expected']
def test_invalid_no_ids_spec_handler(self, mocker, invalid_no_ids_dataset):
with patch('builtins.open',
mock_open(
read_data=json.dumps(invalid_no_ids_dataset))) as mock:
with pytest.raises(ValueError):
GeneratorsHandler({"config_file": ""})
mock.assert_called()
def test_without_dataset(self, mocker, no_datasets_specification):
data = json.dumps(no_datasets_specification)
with patch('builtins.open', mock_open(read_data=data)) as mock:
with raises(ValueError):
GeneratorsHandler(arguments={'config_file': None})
mock.assert_called()
def test_generate(self, mocker, valid_specification):
mock_write = mocker.patch \
.object(GeneratorsHandler, 'write_dataframe')
mock_write.return_value = 'file_path'
mock_config = mocker.patch \
.object(GeneratorsHandler,
'valid_specification_dataset')
mock_config.return_value = valid_specification
handler = GeneratorsHandler(arguments={'config_file': None})
for key, format_, path in handler.generate():
assert key is not None
assert isinstance(key, str)
assert format_ is not None
assert isinstance(format_, str)
assert path is not None
assert isinstance(path, str)
mock_write.assert_called()
def test_write(self, mocker, valid_specification):
mock_config = mocker.patch \
.object(GeneratorsHandler,
'valid_specification_dataset')
mock_config.return_value = valid_specification
handler = GeneratorsHandler(arguments={'config_file': None})
for key in valid_specification['datasets'].keys():
dataset = valid_specification['datasets'][key]
dataset_format = dataset['format']
dataframe = handler.generate_dataframe(dataset)
for destination in dataset['serializers']['to']:
file_type = destination['type']
writer = writers[file_type]
mock = mocker.patch.object(writer, 'write')
mock.return_value = 'file_path'
file_path = handler.write_dataframe(dataframe=dataframe,
destination=destination,
format_=dataset_format)
mock.assert_called()
assert file_path == 'file_path'
def test_invalid_no_dataset_info(self, mocker,
invalid_dataset_specification):
from os.path import abspath
from uuid import uuid4
absolute_path = abspath(".")
file_name = uuid4()
with open(f"{absolute_path}/{file_name}", "w") as f:
json.dump(invalid_dataset_specification, f)
with pytest.raises(ValueError):
GeneratorsHandler({"config_file": f"{absolute_path}/{file_name}"})
remove(f"{absolute_path}/{file_name}")
def test_valid_spec_handler(self, mocker, valid_specification):
from os.path import abspath
from uuid import uuid4
absolute_path = abspath(".")
file_name = uuid4()
with open(f"{absolute_path}/{file_name}", "w") as f:
json.dump(valid_specification, f)
handler = \
GeneratorsHandler({"config_file": f"{absolute_path}/{file_name}"})
remove(f"{absolute_path}/{file_name}")
assert handler
def test_valid_specification(self, mocker, valid_specification):
data = json.dumps(valid_specification)
with patch('builtins.open', mock_open(read_data=data)) as mock:
GeneratorsHandler(arguments={'config_file': None})
mock.assert_called()
