def test_values_list_order():
data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
spec = builder.single_field('field', data).build()
supplier = Loader(spec).get('field')
values = [supplier.next(i) for i in range(10)]
assert values == data
def test_geo_spec_pair_lat_first():
spec = _geo_pair_spec(precision=2, lat_first="yes")
supplier = Loader(spec).get('pair')
value = supplier.next(0)
parts = value.split(',')
_verify_long(parts[1], -2)
_verify_lat(parts[0], -2)
def test_range_wrap_around():
data = [1, 3]
spec = builder.single_field("field:range", data).build()
supplier = Loader(spec).get('field')
vals = [supplier.next(i) for i in range(4)]
assert vals == [1, 2, 3, 1]
def test_unicode_range_single_range_as_hex():
field_spec = builder.unicode_range(data=[0x3040, 0x309f], count=5)
spec = builder.single_field("text", field_spec).build()
supplier = Loader(spec).get('text')
first = supplier.next(0)
for c in first:
assert 0x3040 <= ord(c) <= 0x309f
def test_config_ref_for_values():
""" verifies that the values ref inherits the config from the config_ref """
spec = builder.single_field("name?config_ref=quoteit", ["bob", "joe", "ann", "sue"]) \
.add_ref("quoteit", builder.config_ref(quote="\"")) \
.build()
supplier = Loader(spec).get('name')
assert supplier.next(0) == '"bob"'
def test_geo_spec_pair_default_order():
spec = _geo_pair_spec(precision=1)
supplier = Loader(spec).get('pair')
value = supplier.next(0)
parts = value.split(',')
_verify_long(parts[0], -1)
_verify_lat(parts[1], -1)
def test_uuid_valid_schema(key, spec):
# for coverage
spec = builder.single_field(key, spec).build()
loader = Loader(spec, enforce_schema=True)
supplier = loader.get('foo')
value1 = supplier.next(0)
assert UUID_REGEX.match(value1)
def test_values_count_as_list():
data = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
spec = builder.single_field('field', builder.values(data,
count=[2, 3])).build()
supplier = Loader(spec).get('field')
first = supplier.next(0)
assert isinstance(first, list) and len(first) == 2
second = supplier.next(1)
assert isinstance(second, list) and len(second) == 3
def test_unicode_multiple_ranges():
data = [['0x0590', '0x05ff'], ['0x3040', '0x309f']]
field_spec = builder.unicode_range(data=data, min=3, max=7)
spec = builder.single_field("text", field_spec).build()
supplier = Loader(spec).get('text')
first = supplier.next(0)
assert 3 <= len(first) <= 7
for c in first:
assert 0x0590 <= ord(c) <= 0x05ff or 0x3040 <= ord(c) <= 0x309f
def test_char_class_multiple_classes():
exclude = "CUSTOM"
spec = _char_class_spec(data=["lower", "digits", "CUSTOM"],
exclude=exclude)
supplier = Loader(spec).get('name')
value = supplier.next(0)
assert isinstance(value, str)
for char in value:
assert char in string.ascii_lowercase or char in string.digits
def test_rand_range():
spec = builder.spec_builder() \
.add_field("field", builder.rand_range([100.9, 109.9], cast="int")) \
.build()
supplier = Loader(spec).get('field')
first = supplier.next(0)
assert str(first).isnumeric()
# occasionally gets rounded down to 100
assert 100 <= first <= 110
def test_nested_range_lists_mixed_types_and_step_cast():
data = [
[0.5, 2.5, 0.5],
[20.01234, 30.56789]
]
spec = builder.single_field("field:range?cast=str&precision=2", data).build()
supplier = Loader(spec).get('field')
assert supplier.next(0) == '0.5'
assert supplier.next(1) == '20.01'
def test_uuid_spec():
spec = builder.single_field("foo:uuid", {}).build()
loader = Loader(spec)
supplier = loader.get('foo')
value1 = supplier.next(0)
assert UUID_REGEX.match(value1)
value2 = supplier.next(1)
assert UUID_REGEX.match(value2)
assert value1 != value2
def test_nested_range_lists_simple():
data = [
[0, 10],
[20, 30]
]
spec = builder.single_field("field:range", data).build()
supplier = Loader(spec).get('field')
first = supplier.next(0)
assert 0 <= first <= 10
second = supplier.next(1)
assert 20 <= second <= 30
def test_unicode_range_single_range_as_hex_strings():
field_spec = builder.unicode_range(data=[0x3040, 0x309f],
mean=5,
stddev=2,
min=2,
max=7)
spec = builder.single_field("text", field_spec).build()
supplier = Loader(spec).get('text')
first = supplier.next(0)
assert 2 <= len(first) <= 7
for c in first:
assert 0x3040 <= ord(c) <= 0x309f
def test_geo_spec_pair_reduced_ranges_bbox():
start_lat = -90
end_lat = -45.0
start_long = 90.0
end_long = 180.0
spec = _geo_pair_spec(bbox=[start_long, start_lat, end_long, end_lat])
supplier = Loader(spec).get('pair')
value = supplier.next(0)
parts = value.split(',')
_verify_in_range_and_has_precision(parts[0], start_long, end_long, -4)
_verify_in_range_and_has_precision(parts[1], start_lat, end_lat, -4)
def test_nested_range_lists_mixed_types_and_step():
data = [
[0, 10, 2],
[20.0, 30.0]
]
spec = builder.single_field("field:range", data).build()
supplier = Loader(spec).get('field')
first = supplier.next(0)
assert first % 2 == 0
assert 0 <= first <= 10
second = supplier.next(1)
assert 20.0 <= second <= 30.0
def test_geo_pair_as_list():
start_lat = -45.0
end_lat = 45.0
start_long = 50.0
end_long = 60.0
spec = _geo_pair_spec(bbox=[start_long, start_lat, end_long, end_lat],
as_list=True)
supplier = Loader(spec).get('pair')
value = supplier.next(0)
assert isinstance(value, list)
_verify_in_range_and_has_precision(value[0], start_long, end_long, -4)
_verify_in_range_and_has_precision(value[1], start_lat, end_lat, -4)
def test_geo_spec_pair_reduced_ranges():
start_lat = 0.0
end_lat = 75.0
start_long = -180.0
end_long = -90.0
spec = _geo_pair_spec(start_lat=start_lat,
end_lat=end_lat,
start_long=start_long,
end_long=end_long)
supplier = Loader(spec).get('pair')
value = supplier.next(0)
parts = value.split(',')
_verify_in_range_and_has_precision(parts[0], start_long, end_long, -4)
_verify_in_range_and_has_precision(parts[1], start_lat, end_lat, -4)
def test_single_nested():
# Geo
# - Place
# - Coord
geo_spec = builder.spec_builder() \
.add_field("place_id:uuid", {}) \
.add_field("coordinates", builder.geo_pair(as_list=True))
spec = builder.spec_builder() \
.add_field("id:uuid", {}) \
.add_field("geo", builder.nested(fields=geo_spec.build())) \
.build()
supplier = Loader(spec).get('geo')
first = supplier.next(0)
assert isinstance(first, dict)
assert list(first.keys()) == ['place_id', 'coordinates']
def test_unicode_no_data_element():
spec = builder.single_field("field",
builder.unicode_range(data=None)).build()
spec['field'].pop('data')
with pytest.raises(SpecException):
Loader(spec).get("field")
def test_buffered_supplier_from_spec():
"""
Tests interpreting specs for buffering
"""
values_spec = builder.values(['a', 'b', 'c', 'd', 'e', 'f', 'g'],
sample=True,
buffer_size="20")
assert suppliers._is_buffered(**values_spec['config'])
data_spec = builder.single_field('field', values_spec).build()
loader = Loader(data_spec)
supplier = loader.get('field')
for i in range(10):
value = supplier.next(i)
assert value == supplier.next(i)
def test_weighed_ref_count_as_list():
ref_weights = {
'one': 0.5,
'two': 0.4,
'tre': 0.1,
}
spec = builder.single_field('field', builder.weighted_ref(ref_weights, count=3)) \
.add_ref('one', 'uno') \
.add_ref('two', 'dos') \
.add_ref('tre', 'tres') \
.build()
loader = Loader(spec)
supplier = loader.get('field')
first = supplier.next(0)
assert isinstance(first, list)
assert len(first) == 3
def test_char_class_abbreviations():
abbreviations = [
'cc-' + key for key in _registered_types._CLASS_MAPPING.keys()
]
for abbreviation in abbreviations:
spec = _cc_abbrev_spec(abbrev=abbreviation, count=7)
supplier = Loader(spec).get('name')
_verify_values(supplier, 7, 7)
def _configure_ref_supplier(field_spec: dict, loader: datacraft.Loader):
""" configures supplier for ref type """
key = None
if 'data' in field_spec:
key = field_spec.get('data')
if 'ref' in field_spec:
key = field_spec.get('ref')
if key is None:
raise datacraft.SpecException('No key found for spec: ' +
json.dumps(field_spec))
return loader.get(key)
def test_multi_nested():
# User
# - Geo
# - - Place
# - - Coord
geo_spec = builder.spec_builder() \
.add_field("place_id:uuid", {}) \
.add_field("coordinates", builder.geo_pair(as_list=True))
user_spec = builder.spec_builder() \
.add_field("user_id:uuid", {}) \
.add_field("geo", builder.nested(fields=geo_spec.build()))
spec = builder.spec_builder() \
.add_field("id:uuid", {}) \
.add_field("user", builder.nested(fields=user_spec.build())) \
.build()
supplier = Loader(spec).get('user')
first = supplier.next(0)
assert isinstance(first, dict)
assert list(first.keys()) == ['user_id', 'geo']
second = first['geo']
assert isinstance(second, dict)
assert list(second.keys()) == ['place_id', 'coordinates']
def build_suppliers_map(field_spec: dict, loader: datacraft.Loader) -> dict:
"""uses refs or fields to build a map for those suppliers"""
if 'refs' not in field_spec and 'fields' not in field_spec:
raise datacraft.SpecException(
f'Must define one of fields or refs. {json.dumps(field_spec)}')
if 'refs' in field_spec and 'fields' in field_spec:
raise datacraft.SpecException(
f'Must define only one of fields or refs. {json.dumps(field_spec)}'
)
mappings = _get_mappings(field_spec, 'refs')
mappings.update(_get_mappings(field_spec, 'fields'))
if len(mappings) < 1:
raise datacraft.SpecException(
f'fields or refs empty: {json.dumps(field_spec)}')
suppliers_map = {}
for field_or_ref, alias in mappings.items():
supplier = loader.get(field_or_ref)
suppliers_map[alias] = supplier
return suppliers_map
def test_unicode_data_is_not_list():
spec = builder.single_field(
"field", builder.unicode_range(data="0x3040,0x309f")).build()
with pytest.raises(SpecException):
Loader(spec).get("field")
def test_char_class_no_data_element():
spec = _char_class_spec(data="special", count=4)
spec['name'].pop('data')
with pytest.raises(SpecException):
Loader(spec).get('name')
def test_char_class_printable():
spec = _cc_abbrev_spec(abbrev="printable", mean=3, stddev=2, min=1, max=5)
supplier = Loader(spec).get('name')
_verify_values(supplier, 1, 5)