def output_geojson_bzipped(index, routes):
'''
'''
try:
ids = [id for (id, t, g) in routes]
geometries = [cascaded_union(geoms) for (i, t, geoms) in routes]
geometries = [geom.__geo_interface__ for geom in geometries if geom]
properties = [tags for (i, tags, g) in routes]
features = [dict(type='Feature', id=id, properties=p, geometry=g)
for (id, p, g) in zip(ids, properties, geometries)]
geojson = dict(type='FeatureCollection', features=features)
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(geojson)
output = BZ2File('routes-%06d.json.bz2' % index, 'w')
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + '%.6f' % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write('%.6f' % float(token))
else:
output.write(token)
output.close()
except Exception, e:
return index, len(routes), e
def to_output(self, output: TextIOBase, **kwargs):
"""
Outputs the given object to the given output stream in
the JSON data representation format.
Args:
output: The output stream to serialize object to.
kwargs: Additional arguments to be passed to the
JSON encoder.
"""
assert output.writable()
default = JSONEncoder.default
def json_encoder(value):
if isinstance(value, IOable):
return value.to_dict()
elif hasattr(value, 'to_json'):
return value.to_json()
else:
raise TypeError(f'{value}')
if 'indent' not in kwargs:
kwargs['indent'] = 2
encoder = JSONEncoder(default=json_encoder, **kwargs)
for chunk in encoder.iterencode(self):
output.write(chunk)
def output_geojson_bzipped(index, streets):
'''
'''
try:
ids = [str(uuid1()) for (n, k, h, g) in streets]
geometries = [geom.__geo_interface__ for (n, k, h, geom) in streets]
properties = [dict(name=short_street_name(n), long_name=n, kind=k, highway=h)
for (n, k, h, g) in streets]
features = [dict(type='Feature', id=id, properties=p, geometry=g)
for (id, p, g) in zip(ids, properties, geometries)]
geojson = dict(type='FeatureCollection', features=features)
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(geojson)
output = BZ2File('streets-%06d.json.bz2' % index, 'w')
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + '%.6f' % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write('%.6f' % float(token))
else:
output.write(token)
output.close()
except Exception, e:
return index, len(streets), e
def json_encode(data, pretty=False):
''' Encode stream of JSON with 7-digits floating point precision.
'''
if pretty:
separators = (', ', ': ')
indent = 2
else:
separators = (',', ':')
indent = None
encoder = JSONEncoder(indent=indent, separators=separators)
encoded = encoder.iterencode(data)
output = StringIO()
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + '%.7f' % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write('%.7f' % float(token))
else:
output.write(token)
return output.getvalue()
class JsonEncoder(object):
"""JSON Encoder."""
def __init__(self, stream, **kwargs):
"""Create JSON Encoder.
:param stream: Output stream
:type stream: file or io.IOBase
"""
self.stream = stream
self.encoder = JSONEncoder(**kwargs)
def write_dict(self, items):
"""Write dictionary to stream.
:param items: Items
:type items: generator
"""
for chunk in self.encoder.iterencode(DictionaryEmitter(items)):
if six.PY3:
chunk = bytes(chunk, encoding='utf8')
self.stream.write(chunk)
def write_list(self, items):
"""Write list to stream.
:param items: Items
:type items: generator
"""
raise NotImplementedError
def dump(iterator, f=sys.stdout):
""" Converts a Python generator (iterator) to a JSON list and
outputs to a file handle (f) (defaults to sys.stdout)"""
encoder = JSONEncoder(indent=2)
for chunk in encoder.iterencode(
StreamingJsonListPrinter.SerializableGenerator(iterator)):
f.write(chunk)
f.flush()
class JsonFormatter(Formatter):
def begin(self, fields):
self.encoder = JSONEncoder(indent=2, separators=(', ', ': '))
def iterout(self, iterable):
self.begin([])
for chunk in self.encoder.iterencode(list(iterable)):
self.fp.write(chunk)
def out(self, d):
self.fp.write(self.encoder.encode(d))
def json_encode(data):
''' Encode stream of JSON with 7-digits floating point precision.
'''
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(data)
output = StringIO()
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + '%.7f' % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write('%.7f' % float(token))
else:
output.write(token)
return output.getvalue()
def iterencode(self, o: object, *args, **kwargs):
'''
Overriding the iterencode method in json.JSONEncoder class
keyword argument:
o -- A python object
*args and **kwargs -- Cover any other positional arguments
'''
def is_namedtuple() -> bool:
'''
Returns true if object is a namedtuple for the typing or collections library.
'''
return (isinstance(o, tuple)
and getattr(o, '_fields', None) is not None)
if is_namedtuple():
ntuple_odict: 'OrderedDict[str, Any]' = o._asdict()
return str({k: v
for k, v in ntuple_odict.items()}).replace("'", '"')
return JSONEncoder.iterencode(o, *args, **kwargs)
def write_liljson(data, output, precision):
args = {'separators': (',', ':')}
encoder = JSONEncoder(**args)
encoded = encoder.iterencode(data)
format = '%.' + str(precision) + 'f'
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + format % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write(format % float(token))
else:
output.write(token)
def write_liljson(data, output, precision):
args = { 'separators': (',', ':') }
encoder = JSONEncoder(**args)
encoded = encoder.iterencode(data)
format = '%.' + str(precision) + 'f'
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + format % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write(format % float(token))
else:
output.write(token)
def to_output(cls, object: 'IOable',
output: TextIOBase, format: str = 'json',
options: Dict = {}, **kwargs):
"""
Outputs the given object to the given output stream in
the specified output data representation format.
Args:
object: The object to serialize.
output: The output stream to serialize object to.
format: The output serialization format: 'json'.
options: The options to be passed to to_object
via json_encoder_default, used to customize
and tweak the object generation process.
kwargs: Additional arguments to be passed to the
specific output format encoder.
Raises:
ValueError: If output format is unsupported.
"""
assert output.writable()
default = JSONEncoder.default
def json_encoder(value):
if isinstance(value, IOable):
return value.__class__.to_object(value, **options)
elif hasattr(value, 'to_json'):
return getattr(value, 'to_json', default)(value, **options)
else:
raise TypeError(f'{value}')
if format == 'json':
if 'indent' not in kwargs:
kwargs['indent'] = 2
encoder = JSONEncoder(default=json_encoder, **kwargs)
for chunk in encoder.iterencode(object):
output.write(chunk)
else:
raise ValueError(f'Unsupported "{format}" output format')
def merge_geojson(geojsonPath, outfile):
float_pat = compile(r'^-?\d+\.\d+(e-?\d+)?$')
charfloat_pat = compile(r'^[\[,\,]-?\d+\.\d+(e-?\d+)?$')
precision = 6
infiles = glob.glob(geojsonPath)
outjson = dict(type='FeatureCollection', features=[])
for infile in infiles:
injson = load(open(infile))
if injson.get('type', None) != 'FeatureCollection':
raise Exception('Sorry, "%s" does not look like GeoJSON' % infile)
if type(injson.get('features', None)) != list:
raise Exception('Sorry, "%s" does not look like GeoJSON' % infile)
outjson['features'] += injson['features']
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(outjson)
format = '%.' + str(precision) + 'f'
output = open(outfile, 'w')
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + format % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write(format % float(token))
else:
output.write(token)
float_pat = compile(r'^-?\d+\.\d+(e-?\d+)?$')
charfloat_pat = compile(r'^[\[,\,]-?\d+\.\d+(e-?\d+)?$')
if __name__ == '__main__':
features = []
for line in stdin:
try:
features.extend(hadoop_line_features(line))
except Exception, e:
logging.error(str(e))
continue
geojson = dict(type='FeatureCollection', features=features)
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(geojson)
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
stdout.write(token[0] + '%.5f' % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
stdout.write('%.5f' % float(token))
else:
stdout.write(token)
outjson = dict(type='FeatureCollection', features=[])
for infile in infiles:
injson = load(open(infile))
if injson.get('type', None) != 'FeatureCollection':
raise Exception('Sorry, "%s" does not look like GeoJSON' % infile)
if type(injson.get('features', None)) != list:
raise Exception('Sorry, "%s" does not look like GeoJSON' % infile)
outjson['features'] += injson['features']
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(outjson)
format = '%.' + str(options.precision) + 'f'
output = open(outfile, 'w')
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + format % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write(format % float(token))
else:
output.write(token)
outjson = dict(type='FeatureCollection', features=[])
for infile in infiles:
injson = load(open(infile))
if injson.get('type', None) != 'FeatureCollection':
raise Exception('Sorry, "%s" does not look like GeoJSON' % infile)
if type(injson.get('features', None)) != list:
raise Exception('Sorry, "%s" does not look like GeoJSON' % infile)
outjson['features'] += injson['features']
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(outjson)
format = '%.' + str(options.precision) + 'f'
output = open(outfile, 'w')
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + format % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write(format % float(token))
else:
output.write(token)
type='int', help='Digits of precision, default %(precision)d.' % defaults)
if __name__ == '__main__':
options, args = parser.parse_args()
#
# Read!
#
input = len(args) and open(args[0]) or stdin
data = load(input)
#
# Write!
#
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(data)
format = '%.' + str(options.precision) + 'f'
output = len(args) == 2 and open(args[1], 'w') or stdout
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + format % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write(format % float(token))
else:
output.write(token)
from re import compile
import encode_postgis
float_pat = compile(r'^-?\d+\.\d+(e-?\d+)?$')
charfloat_pat = compile(r'^[\[,\,]-?\d+\.\d+(e-?\d+)?$')
#input = { "type": "FeatureCollection","features": [{ "type": "Feature", "geometry": { "type": "Polygon", "coordinates": [ [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] ] }, "properties": { "prop0": "value0", "prop1": {"this": "that"} } } ]}
input = { "type": "FeatureCollection", "features": [ { "type": "Feature", "geometry": {"type": "Point", "coordinates": [102.0, 0.5]}, "properties": {"prop0": "value0"} }, { "type": "Feature", "geometry": { "type": "LineString", "coordinates": [ [102.0, 0.0], [103.0, 1.0], [104.0, 0.0], [105.0, 1.0] ] }, "properties": { "prop0": "value0", "prop1": 0.0 } }, { "type": "Feature", "geometry": { "type": "Polygon", "coordinates": [ [ [100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0] ] ] }, "properties": { "prop0": "value0", "prop1": {"this": "that"} } } ]}
data = dumps(input)
print data
#data = loads(data)
#print data
#
# Write!
#
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(input)
# format = '%.' + str(options.precision) + 'f'
# output = len(args) == 2 and open(args[1], 'w') or stdout
prev_lat = 0
prev_lng = 0
out = ""
char_lat = True
# p = ""
# for feature in input['features']:
# print feature['geometry']['type']
# print feature['geometry']['coordinates']
# print encode_postgis._encode_geometry(feature['geometry'])
# feature['geometry']['coordinates'] = encode_postgis._encode_geometry(feature['geometry'])
"prop0": "value0",
"prop1": {
"this": "that"
}
}
}]
}
data = dumps(input)
print data
#data = loads(data)
#print data
#
# Write!
#
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(input)
# format = '%.' + str(options.precision) + 'f'
# output = len(args) == 2 and open(args[1], 'w') or stdout
prev_lat = 0
prev_lng = 0
out = ""
char_lat = True
# p = ""
# for feature in input['features']:
# print feature['geometry']['type']
# print feature['geometry']['coordinates']
# print encode_postgis._encode_geometry(feature['geometry'])
# feature['geometry']['coordinates'] = encode_postgis._encode_geometry(feature['geometry'])
type='int', help='Digits of precision, default %(precision)d.' % defaults)
if __name__ == '__main__':
options, args = parser.parse_args()
#
# Read!
#
input = len(args) and open(args[0]) or stdin
data = load(input)
#
# Write!
#
encoder = JSONEncoder(separators=(',', ':'))
encoded = encoder.iterencode(data)
format = '%.' + str(options.precision) + 'f'
output = len(args) == 2 and open(args[1], 'w') or stdout
for token in encoded:
if charfloat_pat.match(token):
# in python 2.7, we see a character followed by a float literal
output.write(token[0] + format % float(token[1:]))
elif float_pat.match(token):
# in python 2.6, we see a simple float literal
output.write(format % float(token))
else:
output.write(token)
