def test_write_read_geojson_with_props(gjson, props):
byte_stream = io.BytesIO()
writer = geostream.writer(byte_stream, props)
writer.write_feature(gjson)
byte_stream.seek(0)
header1, header2, prop_len = struct.unpack(
'Bii', byte_stream.read(struct.calcsize('Bii')))
assert header1 in GEOSTREAM_SCHEMA_VERSIONS
assert header2 == GEOJSON_EPSG_SRID
assert prop_len > 0
p = byte_stream.read(prop_len)
header_props = cbor2.loads(p)
assert header_props["unit"] == "something"
length = struct.unpack('i', byte_stream.read(struct.calcsize('i')))[0]
zipped_data = byte_stream.read(length)
trailing_length = struct.unpack('i',
byte_stream.read(struct.calcsize('i')))[0]
assert len(zipped_data) == length
assert length == trailing_length
byte_stream.seek(0)
reader = geostream.reader(byte_stream)
assert reader.srid == GEOJSON_EPSG_SRID
assert reader.schema_version in GEOSTREAM_SCHEMA_VERSIONS
assert reader.properties == props
read_features = [f for f in reader]
assert len(read_features) == 1
assert gjson == read_features[0]
byte_stream.seek(0)
rev_reader = geostream.reader(byte_stream, reverse=True)
assert rev_reader.srid == GEOJSON_EPSG_SRID
assert rev_reader.schema_version in GEOSTREAM_SCHEMA_VERSIONS
assert rev_reader.properties == props
rev_features = [f for f in rev_reader]
assert len(rev_features) == 1
assert gjson == read_features[0]
def test_read_invalid_schema_from_stream_raises_exception(feat_collection_2):
with pytest.raises(ValueError):
collection = feat_collection_2
collection_props = {"unit": "something", "key": "uuid"}
byte_stream = io.BytesIO()
writer = geostream.writer(byte_stream, collection_props)
writer.write_feature_collection(collection)
byte_stream.seek(0)
byte_stream.write(struct.pack("B", 0))
byte_stream.seek(0)
geostream.reader(byte_stream)
def test_read_truncated_after_feature_length(feat_collection_2):
collection = feat_collection_2
collection_props = {}
byte_stream = io.BytesIO()
writer = geostream.writer(byte_stream, collection_props)
writer.write_feature_collection(collection)
byte_stream.write(struct.pack(
"i",
2)) # append the length of a second feature, but don't append data
byte_stream.seek(0)
reader = geostream.reader(byte_stream)
read_features = [f for f in reader]
assert len(read_features) == 1
def test_read_truncated_feature_data(feat_collection_2):
collection = feat_collection_2
byte_stream = io.BytesIO()
writer = geostream.writer(byte_stream)
writer.write_feature_collection(collection)
byte_stream.seek(0)
good_data = byte_stream.read()
bad_byte_stream = io.BytesIO()
bad_byte_stream.write(
good_data[:-5]) # truncate trailing length and 1 byte of feature data
bad_byte_stream.seek(0)
reader = geostream.reader(bad_byte_stream)
read_features = [f for f in reader]
assert len(read_features) == 0
def test_reverse_read_gjson_features_from_longer_stream(feat_collection_3):
collection = feat_collection_3
collection_props = {"unit": "something", "key": "uuid"}
byte_stream = io.BytesIO()
writer = geostream.writer(byte_stream, collection_props)
for i in range(0, 100):
writer.write_feature_collection(collection)
byte_stream.seek(0)
reader = geostream.reader(byte_stream, reverse=True)
assert reader.srid == GEOJSON_EPSG_SRID
assert reader.schema_version in GEOSTREAM_SCHEMA_VERSIONS
assert reader.properties == collection_props
read_features = [f for f in reader]
assert len(read_features) == 300
feature_list = feat_collection_3["features"]
assert feature_list[0] == read_features[2]
assert feature_list[1] == read_features[1]
assert feature_list[2] == read_features[0]
def test_reverse_read_empty_stream_raises_exception():
with pytest.raises(struct.error):
byte_stream = io.BytesIO()
geostream.reader(byte_stream, reverse=True)
def cli():
args = parse_args()
output_one_named_file = False
output_path = None
if args.out:
output_path = Path(args.out)
if output_path.suffix == ".json":
if len(args.inputs) == 1 and Path(args.inputs[0]).is_file():
output_one_named_file = True
else:
sys.stderr.write(
f"Output path must be a directory for multiple input files, not: {args.out}\n"
)
exit(1)
elif not output_path.exists():
try:
output_path.mkdir(parents=True)
except Exception as e:
sys.stderr.write(
f"Un-writeable output path: {args.out}. Error: {e}")
elif not output_path.is_dir():
sys.stderr.write(
f"Existing output path must be a directory: {args.out}\n")
exit(1)
select = {}
if args.select:
try:
select = json.loads(args.select)
except Exception as e:
sys.stderr.write(
f"Invalid select text, must be valid JSON string: {args.select}. Error: {e}\n"
)
exit(1)
for gjz_file in [f for i in args.inputs for f in iglob(i)]:
input_gjz = Path(gjz_file)
if not input_gjz.is_file() or not input_gjz.exists():
print(
f"Skipped: {gjz_file}, which is not a file or does not exist")
continue
if input_gjz.suffix != ".gjz":
print(
f"Skipped: {gjz_file}, which does not have the name extension: .gjz"
)
continue
if output_one_named_file:
output_file = output_path
elif output_path is not None:
output_file = output_path.joinpath(input_gjz.stem + ".json")
else:
output_file = input_gjz.parent.joinpath(input_gjz.stem + ".json")
if args.verbose:
print(f"Unpacking: {gjz_file} into: {output_file}")
with input_gjz.open("rb") as bf:
try:
reader = geostream.reader(bf, reverse=args.reverse)
except Exception as e:
print(f"...failed to open {gjz_file}, error: {e}")
continue
if select:
selected = filter(
lambda feat: all(item in feat.properties.items()
for item in select.items()), reader)
else:
selected = reader
if args.verbose:
_feature_count.append(0)
selected = map(_count_features, selected)
geojson_collection = FeatureCollection(
features=selected,
properties=reader.properties,
srid=reader.srid)
try:
with output_file.open("w") as tf:
dump_fn = partial(
json.dump,
geojson_collection,
tf,
allow_nan=False,
iterable_as_array=True,
default=_cbor2types_to_json,
)
if args.pretty:
dump_fn(indent=4, sort_keys=True)
else:
dump_fn()
except Exception as e:
sys.stderr.write(
f"...bad out directory path, failed to open: {output_file}, error: {e}"
)
exit(1)
if args.verbose:
filtered = f"selected by: {select}" if select else "selected all"
print(f"...unpacked {_feature_count[0]} Features, {filtered}")