Esempio n. 1
0
def add_geometries(opts, doc):

    datadir = os.path.join(aerotropolii, 'data', 'geojson')
    tree = utils.woeid2path(doc['woeid'])
    fname = "%s.json" % doc['woeid']

    path = os.path.join(datadir, tree, fname)

    fh = open(path)
    data = json.load(fh)

    feature = data['features'][0]

    geom = feature['geometry']
    geom = shapely.geometry.asShape(geom)

    centroid = geom.centroid
    lat = centroid.y
    lon = centroid.x

    bbox = feature['bbox']

    doc['centroid'] = "%s,%s" % (lat,lon)

    doc['sw_corner'] = "%s,%s" % (bbox[1], bbox[0])
    doc['ne_corner'] = "%s,%s" % (bbox[3], bbox[2])

    doc['geometry_default'] = utils.geometry2carbonite(feature['geometry'])

    props = feature['properties']

    area = props.get('area_sqkm', None)
    rank = props.get('scalerank', None)

    if area:
        doc['area'] = float(area)

    if rank:
        doc['scale_rank'] = int(rank)

    doc['provider_geometry'] = 'natural earth 2.0'
    return doc
def import_planets(options):

    lookup = generate_lookup_table(options)

    for root, dirs, files in os.walk(options.exoplanets):

        for f in files:

            if not f.endswith(".xml") :
                continue

            path = os.path.join(root, f)
            path = os.path.abspath(path)

            fh = open(path, 'r')

            # this is a total hack... but it does mean you
            # can ignore all that foofy xml :D :D :D

            data = xmltodict.parse(fh)
            data = json.loads(json.dumps(data))

            props = {}

            for k, v in data['planet'].items():

                if k == 'star':
                    for _k, _v in data['planet']['star'].items():
                        props[ 'oep-star:%s' % _k ] = _v
                elif k == 'properties':
                    props['oep:mass'] = data['planet']['properties']['mass']
                    props['oep:radius'] = data['planet']['properties']['radius']

                    for _k, _v in data['planet']['properties']['orbit'].items():
                        props[ 'oep-orbit:%s' % _k ] = _v

                else:
                    k = "oep:%s" % k
                    props[k] = v

            oepid = props['oep:id']
            woeid = lookup.get(oepid, 0)

            if woeid != 0:

                woe_root = utils.woeid2path(woeid)
                woe_root = os.path.join(os.path.abspath(options.woeplanets), woe_root)

                fname = "%s.json" % woeid
                woe_path = os.path.join(woe_root, fname)

                woe_fh = open(woe_path, 'r')
                woe_data = json.load(woe_fh)

                woe_features = woe_data['features'][0]
                old_props = woe_features['properties']
                
                for k in ('woe:id', 'placetype', 'artisanal:id', 'artisanal:foundry'):
                    props[k] = old_props[k]

            else:

                woeid, foundry = get_artisanal_int()

                props['woe:id'] =  woeid
                props['placetype'] = 'planet'
                props['artisanal:id'] = woeid
                props['artisanal:foundry'] = foundry

            feature = {
                'id': woeid,
                'properties': props
                }

            woe_data = {
                'type': 'FeatureCollection',
                'features': [ feature ],
                'geometry': { 'type': 'Polygon', 'coordinates': [] }
                }

            woe_root = utils.woeid2path(woeid)
            woe_root = os.path.join(os.path.abspath(options.woeplanets), woe_root)

            if not os.path.exists(woe_root):
                os.makedirs(woe_root)
            
            fname = "%s.json" % woeid

            woe_path = os.path.join(woe_root, fname)

            woe_fh = open(woe_path, 'w')
            woe_data = json.dump(woe_data, woe_fh, indent=2)
            woe_fh.close()
            
            print woe_path
Esempio n. 3
0
            if not f.get('bbox', False):

                geom = f['geometry']
                geom = shapely.geometry.asShape(geom)

                f['bbox'] = geom.bounds

            geojson = {
                'type': 'FeatureCollection',
                'features': [ f ]
            }

            woeid = f['id']

            tree = utils.woeid2path(woeid)
            fname = "%s.json" % woeid

            root = os.path.join(opts.outdir, tree)
            path = os.path.join(root, fname)

            if not os.path.exists(root):
                logging.info("create %s" % root)
                os.makedirs(root)

            logging.info("write %s" % path)
        
            out = open(path, 'w')
            utils.write_json(geojson, out)

    logging.debug("done")
                        props['ne_latitude'],
                        props['sw_longitude'],
                        props['sw_latitude'],
                        props['ne_longitude'],
                        props['woe:id'],
                        props['name'].encode('utf-8'),
                        props['iso'],
                        props.get('icao:code', ''),
                        props.get('iata:code', '')
                        ))

            #

            woeid = props['woe:id']

            root = utils.woeid2path(woeid)
            path = "/%s/%s.json" % (root, woeid)

            short = {
                'geometry' : { 'type': 'Point', 'coordinates': centroid },
                'id': feature['id'],
                'properties' : {
                    'name': props['name'],
                    'woe:id': woeid,
                    'href': path
                    }
                }

            for code in ('iata:code', 'icao:code'):
                if props.get(code, False):
                    short['properties'][code] = props[code]
            else:
                pass

            props[k] = v

        p = f['geometry']
        p = shapely.geometry.asShape(p)
        bbox = p.bounds

        f['type'] = 'Feature'
        f['properties'] = props
        f['bbox'] = bbox
        f['id'] = props['woe_id']

        geojson = {'type': 'FeatureCollection', 'features': [f]}

        tree = utils.woeid2path(f['id'])
        fname = "%s.json" % f['id']

        root = os.path.join(jsondir, tree)
        path = os.path.join(root, fname)

        if not os.path.exists(root):
            os.makedirs(root)

        fh = open(path, 'w')
        indent = None

        utils.write_json(geojson, fh, indent)
        print path
def import_urbanareas(options):

    # sudo make me work with multiprocessor...

    lookup = generate_lookup_table(options)

    areas = os.path.abspath(options.urbanareas)

    fh = open(areas, 'r')
    data = json.load(fh)

    for f in data['features']:

        ne_props = f['properties']
        ne_geom = f['geometry']
        ne_id = f['id']

        print ne_id

        props = {
            'ne:fid' : ne_id
            }

        woeid = lookup.get(ne_id, 0)

        if woeid != 0:

            woe_root = utils.woeid2path(woeid)
            woe_root = os.path.join(os.path.abspath(options.data), woe_root)

            fname = "%s.json" % woeid
            woe_path = os.path.join(woe_root, fname)

            woe_fh = open(woe_path, 'r')
            woe_data = json.load(woe_fh)

            woe_features = woe_data['features'][0]
            props = woe_features['properties']
                
        else:
            woeid, foundry = get_artisanal_int()
                
            props['woe:id'] =  woeid
            props['placetype'] = 'metropolitan-area'
            props['artisanal:id'] = woeid
            props['artisanal:foundry'] = foundry

        for k, v in ne_props.items():
            k = "ne:%s" % k
            props[k] = v

        shp = shapely.geometry.asShape(ne_geom)
        
        feature = {
            'id': woeid,
            'properties': props,
            'geometry': ne_geom
            }

        woe_data = {
            'type': 'FeatureCollection',
            'features': [ feature ],
            'bbox': shp.bounds
            }

        # print pprint.pformat(woe_data)
        # sys.exit()

        woe_root = utils.woeid2path(woeid)
        woe_root = os.path.join(os.path.abspath(options.data), woe_root)

        if not os.path.exists(woe_root):
            os.makedirs(woe_root)
            
        fname = "%s.json" % woeid

        woe_path = os.path.join(woe_root, fname)

        woe_fh = open(woe_path, 'w')
        woe_data = json.dump(woe_data, woe_fh, indent=2)
        woe_fh.close()
            
        print woe_path
Esempio n. 7
0
def import_urbanareas(options):

    # sudo make me work with multiprocessor...

    lookup = generate_lookup_table(options)

    areas = os.path.abspath(options.urbanareas)

    fh = open(areas, 'r')
    data = json.load(fh)

    for f in data['features']:

        ne_props = f['properties']
        ne_geom = f['geometry']
        ne_id = f['id']

        print ne_id

        props = {'ne:fid': ne_id}

        woeid = lookup.get(ne_id, 0)

        if woeid != 0:

            woe_root = utils.woeid2path(woeid)
            woe_root = os.path.join(os.path.abspath(options.data), woe_root)

            fname = "%s.json" % woeid
            woe_path = os.path.join(woe_root, fname)

            woe_fh = open(woe_path, 'r')
            woe_data = json.load(woe_fh)

            woe_features = woe_data['features'][0]
            props = woe_features['properties']

        else:
            woeid, foundry = get_artisanal_int()

            props['woe:id'] = woeid
            props['placetype'] = 'metropolitan-area'
            props['artisanal:id'] = woeid
            props['artisanal:foundry'] = foundry

        for k, v in ne_props.items():
            k = "ne:%s" % k
            props[k] = v

        shp = shapely.geometry.asShape(ne_geom)

        feature = {'id': woeid, 'properties': props, 'geometry': ne_geom}

        woe_data = {
            'type': 'FeatureCollection',
            'features': [feature],
            'bbox': shp.bounds
        }

        # print pprint.pformat(woe_data)
        # sys.exit()

        woe_root = utils.woeid2path(woeid)
        woe_root = os.path.join(os.path.abspath(options.data), woe_root)

        if not os.path.exists(woe_root):
            os.makedirs(woe_root)

        fname = "%s.json" % woeid

        woe_path = os.path.join(woe_root, fname)

        woe_fh = open(woe_path, 'w')
        woe_data = json.dump(woe_data, woe_fh, indent=2)
        woe_fh.close()

        print woe_path
Esempio n. 8
0
def import_planets(options):

    lookup = generate_lookup_table(options)

    for root, dirs, files in os.walk(options.exoplanets):

        for f in files:

            if not f.endswith(".xml"):
                continue

            path = os.path.join(root, f)
            path = os.path.abspath(path)

            fh = open(path, 'r')

            # this is a total hack... but it does mean you
            # can ignore all that foofy xml :D :D :D

            data = xmltodict.parse(fh)
            data = json.loads(json.dumps(data))

            props = {}

            for k, v in data['planet'].items():

                if k == 'star':
                    for _k, _v in data['planet']['star'].items():
                        props['oep-star:%s' % _k] = _v
                elif k == 'properties':
                    props['oep:mass'] = data['planet']['properties']['mass']
                    props['oep:radius'] = data['planet']['properties'][
                        'radius']

                    for _k, _v in data['planet']['properties']['orbit'].items(
                    ):
                        props['oep-orbit:%s' % _k] = _v

                else:
                    k = "oep:%s" % k
                    props[k] = v

            oepid = props['oep:id']
            woeid = lookup.get(oepid, 0)

            if woeid != 0:

                woe_root = utils.woeid2path(woeid)
                woe_root = os.path.join(os.path.abspath(options.woeplanets),
                                        woe_root)

                fname = "%s.json" % woeid
                woe_path = os.path.join(woe_root, fname)

                woe_fh = open(woe_path, 'r')
                woe_data = json.load(woe_fh)

                woe_features = woe_data['features'][0]
                old_props = woe_features['properties']

                for k in ('woe:id', 'placetype', 'artisanal:id',
                          'artisanal:foundry'):
                    props[k] = old_props[k]

            else:

                woeid, foundry = get_artisanal_int()

                props['woe:id'] = woeid
                props['placetype'] = 'planet'
                props['artisanal:id'] = woeid
                props['artisanal:foundry'] = foundry

            feature = {'id': woeid, 'properties': props}

            woe_data = {
                'type': 'FeatureCollection',
                'features': [feature],
                'geometry': {
                    'type': 'Polygon',
                    'coordinates': []
                }
            }

            woe_root = utils.woeid2path(woeid)
            woe_root = os.path.join(os.path.abspath(options.woeplanets),
                                    woe_root)

            if not os.path.exists(woe_root):
                os.makedirs(woe_root)

            fname = "%s.json" % woeid

            woe_path = os.path.join(woe_root, fname)

            woe_fh = open(woe_path, 'w')
            woe_data = json.dump(woe_data, woe_fh, indent=2)
            woe_fh.close()

            print woe_path
            props[k] = v

        p = f['geometry']
        p = shapely.geometry.asShape(p)
        bbox = p.bounds

        f['type'] = 'Feature'
        f['properties'] = props
        f['bbox'] = bbox
        f['id'] = props['woe_id']

        geojson = {
            'type': 'FeatureCollection',
            'features': [ f ]
            }

        tree = utils.woeid2path(f['id'])
        fname = "%s.json" % f['id']

        root = os.path.join(jsondir, tree)
        path = os.path.join(root, fname)

        if not os.path.exists(root):
            os.makedirs(root)
            
        fh = open(path, 'w')
        indent = None

        utils.write_json(geojson, fh, indent)
        print path
Esempio n. 10
0
        props = f['properties']

        name = props.get('NAME_1', '')
        fips = props.get('FIPS_1', '')
        iso = props.get('ISO', '')

        label = "%s %s %s" % (name, iso, fips)
        woeid = lookup.get(label, 0)

        print "label: %s -> %s" % (label, woeid)

        if not woeid:
            continue

        root = utils.woeid2path(woeid)
        fname = '%s.json' % woeid

        woe_root = os.path.join(root, fname)
        woe_path = os.path.join(datadir, woe_root)

        if not os.path.exists(woe_path):
            continue

        print "%s: %s" % (iso, woe_path)

        geom = f['geometry']

        shp = shapely.geometry.asShape(geom)
        bbox = shp.bounds