def add_geometries(opts, doc):
datadir = os.path.join(aerotropolii, 'data', 'geojson')
tree = utils.woeid2path(doc['woeid'])
fname = "%s.json" % doc['woeid']
path = os.path.join(datadir, tree, fname)
fh = open(path)
data = json.load(fh)
feature = data['features'][0]
geom = feature['geometry']
geom = shapely.geometry.asShape(geom)
centroid = geom.centroid
lat = centroid.y
lon = centroid.x
bbox = feature['bbox']
doc['centroid'] = "%s,%s" % (lat,lon)
doc['sw_corner'] = "%s,%s" % (bbox[1], bbox[0])
doc['ne_corner'] = "%s,%s" % (bbox[3], bbox[2])
doc['geometry_default'] = utils.geometry2carbonite(feature['geometry'])
props = feature['properties']
area = props.get('area_sqkm', None)
rank = props.get('scalerank', None)
if area:
doc['area'] = float(area)
if rank:
doc['scale_rank'] = int(rank)
doc['provider_geometry'] = 'natural earth 2.0'
return doc
def import_planets(options):
lookup = generate_lookup_table(options)
for root, dirs, files in os.walk(options.exoplanets):
for f in files:
if not f.endswith(".xml") :
continue
path = os.path.join(root, f)
path = os.path.abspath(path)
fh = open(path, 'r')
# this is a total hack... but it does mean you
# can ignore all that foofy xml :D :D :D
data = xmltodict.parse(fh)
data = json.loads(json.dumps(data))
props = {}
for k, v in data['planet'].items():
if k == 'star':
for _k, _v in data['planet']['star'].items():
props[ 'oep-star:%s' % _k ] = _v
elif k == 'properties':
props['oep:mass'] = data['planet']['properties']['mass']
props['oep:radius'] = data['planet']['properties']['radius']
for _k, _v in data['planet']['properties']['orbit'].items():
props[ 'oep-orbit:%s' % _k ] = _v
else:
k = "oep:%s" % k
props[k] = v
oepid = props['oep:id']
woeid = lookup.get(oepid, 0)
if woeid != 0:
woe_root = utils.woeid2path(woeid)
woe_root = os.path.join(os.path.abspath(options.woeplanets), woe_root)
fname = "%s.json" % woeid
woe_path = os.path.join(woe_root, fname)
woe_fh = open(woe_path, 'r')
woe_data = json.load(woe_fh)
woe_features = woe_data['features'][0]
old_props = woe_features['properties']
for k in ('woe:id', 'placetype', 'artisanal:id', 'artisanal:foundry'):
props[k] = old_props[k]
else:
woeid, foundry = get_artisanal_int()
props['woe:id'] = woeid
props['placetype'] = 'planet'
props['artisanal:id'] = woeid
props['artisanal:foundry'] = foundry
feature = {
'id': woeid,
'properties': props
}
woe_data = {
'type': 'FeatureCollection',
'features': [ feature ],
'geometry': { 'type': 'Polygon', 'coordinates': [] }
}
woe_root = utils.woeid2path(woeid)
woe_root = os.path.join(os.path.abspath(options.woeplanets), woe_root)
if not os.path.exists(woe_root):
os.makedirs(woe_root)
fname = "%s.json" % woeid
woe_path = os.path.join(woe_root, fname)
woe_fh = open(woe_path, 'w')
woe_data = json.dump(woe_data, woe_fh, indent=2)
woe_fh.close()
print woe_path
if not f.get('bbox', False):
geom = f['geometry']
geom = shapely.geometry.asShape(geom)
f['bbox'] = geom.bounds
geojson = {
'type': 'FeatureCollection',
'features': [ f ]
}
woeid = f['id']
tree = utils.woeid2path(woeid)
fname = "%s.json" % woeid
root = os.path.join(opts.outdir, tree)
path = os.path.join(root, fname)
if not os.path.exists(root):
logging.info("create %s" % root)
os.makedirs(root)
logging.info("write %s" % path)
out = open(path, 'w')
utils.write_json(geojson, out)
logging.debug("done")
props['ne_latitude'],
props['sw_longitude'],
props['sw_latitude'],
props['ne_longitude'],
props['woe:id'],
props['name'].encode('utf-8'),
props['iso'],
props.get('icao:code', ''),
props.get('iata:code', '')
))
#
woeid = props['woe:id']
root = utils.woeid2path(woeid)
path = "/%s/%s.json" % (root, woeid)
short = {
'geometry' : { 'type': 'Point', 'coordinates': centroid },
'id': feature['id'],
'properties' : {
'name': props['name'],
'woe:id': woeid,
'href': path
}
}
for code in ('iata:code', 'icao:code'):
if props.get(code, False):
short['properties'][code] = props[code]
else:
pass
props[k] = v
p = f['geometry']
p = shapely.geometry.asShape(p)
bbox = p.bounds
f['type'] = 'Feature'
f['properties'] = props
f['bbox'] = bbox
f['id'] = props['woe_id']
geojson = {'type': 'FeatureCollection', 'features': [f]}
tree = utils.woeid2path(f['id'])
fname = "%s.json" % f['id']
root = os.path.join(jsondir, tree)
path = os.path.join(root, fname)
if not os.path.exists(root):
os.makedirs(root)
fh = open(path, 'w')
indent = None
utils.write_json(geojson, fh, indent)
print path
def import_urbanareas(options):
# sudo make me work with multiprocessor...
lookup = generate_lookup_table(options)
areas = os.path.abspath(options.urbanareas)
fh = open(areas, 'r')
data = json.load(fh)
for f in data['features']:
ne_props = f['properties']
ne_geom = f['geometry']
ne_id = f['id']
print ne_id
props = {
'ne:fid' : ne_id
}
woeid = lookup.get(ne_id, 0)
if woeid != 0:
woe_root = utils.woeid2path(woeid)
woe_root = os.path.join(os.path.abspath(options.data), woe_root)
fname = "%s.json" % woeid
woe_path = os.path.join(woe_root, fname)
woe_fh = open(woe_path, 'r')
woe_data = json.load(woe_fh)
woe_features = woe_data['features'][0]
props = woe_features['properties']
else:
woeid, foundry = get_artisanal_int()
props['woe:id'] = woeid
props['placetype'] = 'metropolitan-area'
props['artisanal:id'] = woeid
props['artisanal:foundry'] = foundry
for k, v in ne_props.items():
k = "ne:%s" % k
props[k] = v
shp = shapely.geometry.asShape(ne_geom)
feature = {
'id': woeid,
'properties': props,
'geometry': ne_geom
}
woe_data = {
'type': 'FeatureCollection',
'features': [ feature ],
'bbox': shp.bounds
}
# print pprint.pformat(woe_data)
# sys.exit()
woe_root = utils.woeid2path(woeid)
woe_root = os.path.join(os.path.abspath(options.data), woe_root)
if not os.path.exists(woe_root):
os.makedirs(woe_root)
fname = "%s.json" % woeid
woe_path = os.path.join(woe_root, fname)
woe_fh = open(woe_path, 'w')
woe_data = json.dump(woe_data, woe_fh, indent=2)
woe_fh.close()
print woe_path
def import_urbanareas(options):
# sudo make me work with multiprocessor...
lookup = generate_lookup_table(options)
areas = os.path.abspath(options.urbanareas)
fh = open(areas, 'r')
data = json.load(fh)
for f in data['features']:
ne_props = f['properties']
ne_geom = f['geometry']
ne_id = f['id']
print ne_id
props = {'ne:fid': ne_id}
woeid = lookup.get(ne_id, 0)
if woeid != 0:
woe_root = utils.woeid2path(woeid)
woe_root = os.path.join(os.path.abspath(options.data), woe_root)
fname = "%s.json" % woeid
woe_path = os.path.join(woe_root, fname)
woe_fh = open(woe_path, 'r')
woe_data = json.load(woe_fh)
woe_features = woe_data['features'][0]
props = woe_features['properties']
else:
woeid, foundry = get_artisanal_int()
props['woe:id'] = woeid
props['placetype'] = 'metropolitan-area'
props['artisanal:id'] = woeid
props['artisanal:foundry'] = foundry
for k, v in ne_props.items():
k = "ne:%s" % k
props[k] = v
shp = shapely.geometry.asShape(ne_geom)
feature = {'id': woeid, 'properties': props, 'geometry': ne_geom}
woe_data = {
'type': 'FeatureCollection',
'features': [feature],
'bbox': shp.bounds
}
# print pprint.pformat(woe_data)
# sys.exit()
woe_root = utils.woeid2path(woeid)
woe_root = os.path.join(os.path.abspath(options.data), woe_root)
if not os.path.exists(woe_root):
os.makedirs(woe_root)
fname = "%s.json" % woeid
woe_path = os.path.join(woe_root, fname)
woe_fh = open(woe_path, 'w')
woe_data = json.dump(woe_data, woe_fh, indent=2)
woe_fh.close()
print woe_path
def import_planets(options):
lookup = generate_lookup_table(options)
for root, dirs, files in os.walk(options.exoplanets):
for f in files:
if not f.endswith(".xml"):
continue
path = os.path.join(root, f)
path = os.path.abspath(path)
fh = open(path, 'r')
# this is a total hack... but it does mean you
# can ignore all that foofy xml :D :D :D
data = xmltodict.parse(fh)
data = json.loads(json.dumps(data))
props = {}
for k, v in data['planet'].items():
if k == 'star':
for _k, _v in data['planet']['star'].items():
props['oep-star:%s' % _k] = _v
elif k == 'properties':
props['oep:mass'] = data['planet']['properties']['mass']
props['oep:radius'] = data['planet']['properties'][
'radius']
for _k, _v in data['planet']['properties']['orbit'].items(
):
props['oep-orbit:%s' % _k] = _v
else:
k = "oep:%s" % k
props[k] = v
oepid = props['oep:id']
woeid = lookup.get(oepid, 0)
if woeid != 0:
woe_root = utils.woeid2path(woeid)
woe_root = os.path.join(os.path.abspath(options.woeplanets),
woe_root)
fname = "%s.json" % woeid
woe_path = os.path.join(woe_root, fname)
woe_fh = open(woe_path, 'r')
woe_data = json.load(woe_fh)
woe_features = woe_data['features'][0]
old_props = woe_features['properties']
for k in ('woe:id', 'placetype', 'artisanal:id',
'artisanal:foundry'):
props[k] = old_props[k]
else:
woeid, foundry = get_artisanal_int()
props['woe:id'] = woeid
props['placetype'] = 'planet'
props['artisanal:id'] = woeid
props['artisanal:foundry'] = foundry
feature = {'id': woeid, 'properties': props}
woe_data = {
'type': 'FeatureCollection',
'features': [feature],
'geometry': {
'type': 'Polygon',
'coordinates': []
}
}
woe_root = utils.woeid2path(woeid)
woe_root = os.path.join(os.path.abspath(options.woeplanets),
woe_root)
if not os.path.exists(woe_root):
os.makedirs(woe_root)
fname = "%s.json" % woeid
woe_path = os.path.join(woe_root, fname)
woe_fh = open(woe_path, 'w')
woe_data = json.dump(woe_data, woe_fh, indent=2)
woe_fh.close()
print woe_path
props[k] = v
p = f['geometry']
p = shapely.geometry.asShape(p)
bbox = p.bounds
f['type'] = 'Feature'
f['properties'] = props
f['bbox'] = bbox
f['id'] = props['woe_id']
geojson = {
'type': 'FeatureCollection',
'features': [ f ]
}
tree = utils.woeid2path(f['id'])
fname = "%s.json" % f['id']
root = os.path.join(jsondir, tree)
path = os.path.join(root, fname)
if not os.path.exists(root):
os.makedirs(root)
fh = open(path, 'w')
indent = None
utils.write_json(geojson, fh, indent)
print path
props = f['properties']
name = props.get('NAME_1', '')
fips = props.get('FIPS_1', '')
iso = props.get('ISO', '')
label = "%s %s %s" % (name, iso, fips)
woeid = lookup.get(label, 0)
print "label: %s -> %s" % (label, woeid)
if not woeid:
continue
root = utils.woeid2path(woeid)
fname = '%s.json' % woeid
woe_root = os.path.join(root, fname)
woe_path = os.path.join(datadir, woe_root)
if not os.path.exists(woe_path):
continue
print "%s: %s" % (iso, woe_path)
geom = f['geometry']
shp = shapely.geometry.asShape(geom)
bbox = shp.bounds