Exemple #1
0
 def test_calculation_iteration(self):
     field = self.get_field(with_value=True,month_count=2)
     field.variables.add_variable(Variable(value=field.variables['tmax'].value+5,
                                           name='tmin',alias='tmin'))
     field.temporal.name_uid = 'tid'
     field.level.name_uid = 'lid'
     field.spatial.geom.name_uid = 'gid'
     
     grouping = ['month']
     tgd = field.temporal.get_grouping(grouping)
     mu = Mean(field=field,tgd=tgd,alias='my_mean')
     ret = mu.execute()
     
     kwds = copy(field.__dict__)
     kwds.pop('_raw')
     kwds.pop('_variables')
     kwds['temporal'] = tgd
     kwds['variables'] = ret
     cfield = DerivedField(**kwds)
     cfield.temporal.name_uid = 'tid'
     cfield.temporal.name_value = 'time'
     cfield.spatial.name_uid = 'gid'
                     
     sc = ShpCabinet()
     meta = sc.get_meta('state_boundaries')
     sp = SpatialCollection(meta=meta,key='state_boundaries',headers=constants.calc_headers)
     for row in sc.iter_geoms('state_boundaries'):
         sp.add_field(row['properties']['UGID'],row['geom'],cfield.variables.keys()[0],
                      cfield,properties=row['properties'])
     for ii,row in enumerate(sp.get_iter_dict()):
         if ii == 0:
             self.assertEqual(row[0].bounds,(-100.5, 39.5, -99.5, 40.5))
             self.assertDictEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000, 'time': datetime.datetime(2000, 1, 16, 0, 0), 'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1, 'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean', 'tid': 1, 'level': 50, 'day': 16})
         self.assertEqual(len(row),2)
         self.assertEqual(len(row[1]),len(constants.calc_headers))
Exemple #2
0
def get_shp(request,key=None):
    query = helpers.parse_qs(request.META['QUERY_STRING'])
    
    select_ugid = SelectUgid()
    select_ugid.parse_query(query)
    
    prefix = Prefix()
    prefix.parse_query(query)
    
    unwrap = Unwrap()
    unwrap.parse_query(query)
    
    pm = PrimeMeridian()
    pm.parse_query(query)
    
    sc = ShpCabinet()
    geom_dict = sc.get_geom_dict(key,attr_filter=select_ugid.value)
    
    ## unwrap coordinates if requested
    if unwrap.value:
        unwrap_geoms(geom_dict,pm.value)
    
    dir_path = get_temp_path(nest=True,only_dir=True,wd=env.DIR_OUTPUT)
    if prefix.value is None:
        out_name = key
    else:
        out_name = prefix.value
    filename = '{0}.shp'.format(out_name)
    path = os.path.join(dir_path,filename)
    path = sc.write(geom_dict,path)
    path = os.path.split(path)[0]
    
    resp = helpers._zip_response_(path,filename=filename.replace('shp','zip'))
    return(resp)
Exemple #3
0
 def test_get_keys(self, dir_shpcabinet=None):
     ocgis.env.DIR_SHPCABINET = dir_shpcabinet or ocgis.env.DIR_SHPCABINET
     sc = ShpCabinet()
     ret = sc.keys()
     target_keys = ['state_boundaries', 'world_countries']
     self.assertEqual(len(set(target_keys).intersection(set(ret))),
                      len(target_keys))
Exemple #4
0
 def _write_(self):
     gid_file = OrderedDict()
     build = True
     is_aggregated = self.ops.aggregate
     with open(self.path,'w') as f:
         ocgis_lh(msg='opened csv file: {0}'.format(self.path),level=logging.DEBUG,
                  logger='conv.csv+')
         writer = csv.writer(f,dialect=OcgDialect)
         for coll in self:
             ocgis_lh('writing collection','conv.csv+',level=logging.DEBUG)
             if build:
                 ocgis_lh('starting build','conv.csv+',level=logging.DEBUG)
                 headers = coll.get_headers(upper=True)
                 if env.WRITE_TO_REFERENCE_PROJECTION:
                     projection = env.REFERENCE_PROJECTION
                 else:
                     projection = coll._archetype.spatial.projection
                 writer.writerow(headers)
                 build = False
                 ocgis_lh(msg='build finished'.format(self.path),level=logging.DEBUG,
                  logger='conv.csv+')
             for geom,row,geom_ids in coll.get_iter(with_geometry_ids=True):
                 if not is_aggregated:
                     ugid = geom_ids['ugid']
                     did = geom_ids['did']
                     gid = geom_ids['gid']
                     if ugid not in gid_file:
                         gid_file[ugid] = OrderedDict()
                     if did not in gid_file[ugid]:
                         gid_file[ugid][did] = OrderedDict()
                     gid_file[ugid][did][gid] = geom
                 writer.writerow(row)
             ocgis_lh('finished writing collection','conv.csv+',level=logging.DEBUG)
     
     if is_aggregated is True:
         ocgis_lh('creating a UGID-GID shapefile is not necessary for aggregated data. use UGID shapefile.',
                  'conv.csv+',
                  logging.WARN)
     else:
         ocgis_lh('writing UGID-GID shapefile','conv.csv+',logging.DEBUG)
         sc = ShpCabinet()
         shp_dir = os.path.join(self.outdir,'shp')
         try:
             os.mkdir(shp_dir)
         ## catch if the directory exists
         except OSError:
             if os.path.exists(shp_dir):
                 pass
             else:
                 raise
         shp_path = os.path.join(shp_dir,self.prefix+'_gid.shp')
         
         def iter_gid_file():
             for ugid,did_gid in gid_file.iteritems():
                 for did,gid_geom in did_gid.iteritems():
                     for gid,geom in gid_geom.iteritems():
                         yield({'geom':geom,'DID':did,
                                'UGID':ugid,'GID':gid})
         
         sc.write(iter_gid_file(),shp_path,sr=projection.sr)
Exemple #5
0
 def write(self,path):
     geoms = []
     uid = self.spatial.uid
     for ii,geom in iter_array(self.spatial.geom,return_value=True):
         geoms.append({'geom':geom,'ugid':uid[ii]})
     sc = ShpCabinet()
     sc.write(geoms,path,sr=self.spatial.projection.sr)
Exemple #6
0
 def write(self, path):
     geoms = []
     uid = self.spatial.uid
     for ii, geom in iter_array(self.spatial.geom, return_value=True):
         geoms.append({'geom': geom, 'ugid': uid[ii]})
     sc = ShpCabinet()
     sc.write(geoms, path, sr=self.spatial.projection.sr)
Exemple #7
0
 def test_shp(self):
     sc = ShpCabinet()
     keys = sc.keys()
     for key in keys:
         url = '/shp/{0}'.format(key)
         resp = self.c.get(url)
         self.assertTrue(len(resp.content) > 100)
Exemple #8
0
 def test_shp(self):
     sc = ShpCabinet()
     keys = sc.keys()
     for key in keys:
         url = '/shp/{0}'.format(key)
         resp = self.c.get(url)
         self.assertTrue(len(resp.content) > 100)
Exemple #9
0
    def test_multivariate_iteration(self):
        field = self.get_field(with_value=True, month_count=1)
        field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5,
                                              name='tmin', alias='tmin'))
        field.temporal.name_uid = 'tid'
        field.level.name_uid = 'lid'
        field.spatial.geom.name_uid = 'gid'

        div = Divide(field=field, parms={'arr1': 'tmin', 'arr2': 'tmax'}, alias='some_division',
                     dtype=np.float64)
        ret = div.execute()

        cfield = DerivedMultivariateField(variables=ret, realization=field.realization, temporal=field.temporal,
                                          level=field.level,
                                          spatial=field.spatial, meta=field.meta, uid=field.uid)
        cfield.spatial.name_uid = 'gid'

        sc = ShpCabinet()
        meta = sc.get_meta('state_boundaries')
        sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_MULTI)
        for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
            sp.add_field(cfield, ugeom=row)

        for ii, row in enumerate(sp.get_iter_dict(melted=True)):
            if ii == 0:
                self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'cid': 1, 'did': None, 'year': 2000,
                                              'time': datetime.datetime(2000, 1, 1, 12, 0),
                                              'calc_alias': 'some_division', 'value': 12.989774984574424, 'month': 1,
                                              'gid': 1, 'calc_key': 'divide', 'tid': 1, 'level': 50, 'day': 1})
        self.assertEqual(ii + 1, 2 * 31 * 2 * 3 * 4 * 51)
Exemple #10
0
def get_shp(request, key=None):
    query = helpers.parse_qs(request.META['QUERY_STRING'])

    select_ugid = SelectUgid()
    select_ugid.parse_query(query)

    prefix = Prefix()
    prefix.parse_query(query)

    unwrap = Unwrap()
    unwrap.parse_query(query)

    pm = PrimeMeridian()
    pm.parse_query(query)

    sc = ShpCabinet()
    geom_dict = sc.get_geom_dict(key, attr_filter=select_ugid.value)

    ## unwrap coordinates if requested
    if unwrap.value:
        unwrap_geoms(geom_dict, pm.value)

    dir_path = get_temp_path(nest=True, only_dir=True, wd=env.DIR_OUTPUT)
    if prefix.value is None:
        out_name = key
    else:
        out_name = prefix.value
    filename = '{0}.shp'.format(out_name)
    path = os.path.join(dir_path, filename)
    path = sc.write(geom_dict, path)
    path = os.path.split(path)[0]

    resp = helpers._zip_response_(path,
                                  filename=filename.replace('shp', 'zip'))
    return (resp)
Exemple #11
0
 def test_iter_all(self):
     raise(SkipTest('dev - long'))
     sc = ShpCabinet()
     for key in sc.keys():
         print key
         for geom in sc.iter_geoms(key):
             self.assertTrue(True)
Exemple #12
0
 def test_sql_subset(self):
     sc = ShpCabinet()
     path = sc.get_shp_path('state_boundaries')
     ds = ogr.Open(path)
     ret = ds.ExecuteSQL('select * from state_boundaries where state_name = "New Jersey"')
     ret.ResetReading()
     self.assertEqual(len(ret),1)
Exemple #13
0
 def test_sql_subset(self):
     sc = ShpCabinet()
     path = sc.get_shp_path('state_boundaries')
     ds = ogr.Open(path)
     ret = ds.ExecuteSQL(
         'select * from state_boundaries where state_name = "New Jersey"')
     ret.ResetReading()
     self.assertEqual(len(ret), 1)
Exemple #14
0
 def get_collection(self):
     field = self.get_field(with_value=True)
     sc = ShpCabinet()
     meta = sc.get_meta('state_boundaries')
     sp = SpatialCollection(meta=meta, key='state_boundaries')
     for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
         sp.add_field(field, ugeom=row)
     return sp
Exemple #15
0
 def test_unwrap(self):
     sc = ShpCabinet()
     _key = ['state_boundaries','world_countries']
     for key in _key:
         geoms = sc.get_geoms(key,unwrap=True)
         for geom in geoms:
             x = geom['geom'].centroid.x
             self.assertTrue(x > 0)
Exemple #16
0
 def test_iter_geoms_no_load_geoms(self):
     sc = ShpCabinet()
     it = sc.iter_geoms('state_boundaries',load_geoms=False)
     geoms = list(it)
     self.assertEqual(len(geoms),51)
     self.assertEqual(geoms[12]['properties']['STATE_NAME'],'New Hampshire')
     for geom in geoms:
         self.assertNotIn('geom',geom)
Exemple #17
0
 def test_iter_geoms(self):
     sc = ShpCabinet()
     it = sc.iter_geoms('state_boundaries')
     geoms = list(it)
     self.assertEqual(len(geoms),51)
     self.assertEqual(geoms[12]['properties']['STATE_NAME'],'New Hampshire')
     for geom in geoms:
         self.assertIn(type(geom['geom']),(Polygon,MultiPolygon))
Exemple #18
0
 def test_unwrap_pm(self):
     _pm = [-4.0,-10.0,-20.0,5.0]
     sc = ShpCabinet()
     for pm in _pm:
         geoms = sc.get_geoms('world_countries',unwrap=True,pm=pm)
         path = '/tmp/shp{0}.shp'.format(time.time())
         sc.write(geoms,path)
         for geom in geoms:
             bounds = geom['geom'].bounds
             self.assertTrue(bounds[0] >= pm)
Exemple #19
0
 def write(self,path):
     geoms = []
     uid = self.spatial.uid
     attrs = self.spatial.attrs
     for ii,geom in iter_array(self.spatial.geom,return_value=True):
         dct = {'geom':geom,'UGID':uid[ii]}
         for k,v in attrs.iteritems():
             dct[k] = v[ii]
         geoms.append(dct)
     sc = ShpCabinet()
     sc.write(geoms,path,sr=self.spatial.projection.sr)
Exemple #20
0
 def write(self, path):
     geoms = []
     uid = self.spatial.uid
     attrs = self.spatial.attrs
     for ii, geom in iter_array(self.spatial.geom, return_value=True):
         dct = {'geom': geom, 'UGID': uid[ii]}
         for k, v in attrs.iteritems():
             dct[k] = v[ii]
         geoms.append(dct)
     sc = ShpCabinet()
     sc.write(geoms, path, sr=self.spatial.projection.sr)
Exemple #21
0
 def test_shapefiles_not_in_folders(self):
     for dirpath,dirnames,filenames in os.walk(ocgis.env.DIR_SHPCABINET):
         for filename in filenames:
             if filename.startswith('state_boundaries') or filename.startswith('world_countries'):
                 dst = os.path.join(self._test_dir,filename)
                 src = os.path.join(dirpath,filename)
                 shutil.copy2(src,dst)
     self.test_get_keys(dir_shpcabinet=self._test_dir)
     
     sc = ShpCabinet(path=self._test_dir)
     path = sc.get_shp_path('world_countries')
     self.assertEqual(path,os.path.join(self._test_dir,'world_countries.shp'))
Exemple #22
0
 def test_constructor(self):
     field = self.get_field(with_value=True)
     sc = ShpCabinet()
     meta = sc.get_meta('state_boundaries')
     sp = SpatialCollection(meta=meta,key='state_boundaries')
     for row in sc.iter_geoms('state_boundaries'):
         sp.add_field(row['properties']['UGID'],row['geom'],field.variables.keys()[0],
                      field,properties=row['properties'])
     self.assertEqual(len(sp),51)
     self.assertIsInstance(sp.geoms[25],MultiPolygon)
     self.assertIsInstance(sp.properties[25],dict)
     self.assertEqual(sp[25]['tmax'].variables['tmax'].value.shape,(2, 31, 2, 3, 4))
Exemple #23
0
    def test_number_in_shapefile_name(self):
        """Test number in shapefile name."""

        sc = ShpCabinet()
        path = sc.get_shp_path('state_boundaries')
        out_path = os.path.join(self.current_dir_output, '51_states.shp')
        with fiona.open(path) as source:
            with fiona.open(out_path, mode='w', driver='ESRI Shapefile', schema=source.meta['schema'],
                            crs=source.meta['crs']) as sink:
                for record in source:
                    sink.write(record)
        ret = list(ShpCabinetIterator(select_uid=[23], path=out_path))
        self.assertEqual(len(ret), 1)
Exemple #24
0
 def _format_all_elements_(self,value):
     try:
         minx,miny,maxx,maxy = value
         geom = Polygon(((minx,miny),
                         (minx,maxy),
                         (maxx,maxy),
                         (maxx,miny)))
         assert(geom.is_valid)
         ret = [{'id':1,'geom':geom}]
     except ValueError:
         sc = ShpCabinet()
         ret = sc.get_geom_dict(value[0])
     return(ret)
Exemple #25
0
 def test_shp_process(self):
     copy_path = os.path.join(self._test_dir,'test_shp_process')
     shutil.copytree(self._test_path,copy_path)
     shp_path = os.path.join(copy_path,'wc_4326.shp')
     out_folder = tempfile.mkdtemp(dir=self._test_dir)
     sp = ShpProcess(shp_path,out_folder)
     sp.process(key='world_countries',ugid=None)
         
     sc = ShpCabinet(path=out_folder)
     select_ugid = [33,126,199]
     geoms = list(sc.iter_geoms('world_countries',select_ugid=select_ugid))
     self.assertEqual(len(geoms),3)
     names = [item['properties']['NAME'] for item in geoms]
     self.assertEqual(set(names),set(['Canada','Mexico','United States']))
Exemple #26
0
    def test_iter_geoms(self):
        sc = ShpCabinet()
        it = sc.iter_geoms('state_boundaries')
        geoms = list(it)
        self.assertEqual(len(geoms), 51)
        self.assertEqual(geoms[12]['properties']['STATE_NAME'], 'New Hampshire')
        for geom in geoms:
            self.assertIn(type(geom['geom']), (Polygon, MultiPolygon))

        # test with a shapefile not having a unique identifier
        env.DEFAULT_GEOM_UID = 'ggidd'
        new = self.get_shapefile_path_with_no_ugid()
        sc = ShpCabinet()
        target = list(sc.iter_geoms(path=new))
        self.assertEqual(len(target), 11)
        self.assertEqual(target[0]['properties'][env.DEFAULT_GEOM_UID], 1)
        self.assertEqual(target[3]['properties'][env.DEFAULT_GEOM_UID], 4)

        target = list(sc.iter_geoms(path=new, uid='ID'))
        self.assertNotIn(env.DEFAULT_GEOM_UID, target[9]['properties'])
        self.assertEqual(int, type(target[7]['properties']['ID']))

        target = list(sc.iter_geoms(path=new, uid='ID', as_spatial_dimension=True))
        ref = target[4]
        self.assertEqual(ref.uid[0, 0], 10)
        self.assertNotIn(env.DEFAULT_GEOM_UID, ref.properties.dtype.names)

        # test with a different geometry unique identifier
        path = self.get_shapefile_path_with_no_ugid()
        geom_select_uid = [12, 15]
        geom_uid = 'ID'
        sc = ShpCabinet()
        records = list(sc.iter_geoms(path=path, uid=geom_uid, select_uid=geom_select_uid))
        self.assertEqual(len(records), 2)
        self.assertEqual([r['properties']['ID'] for r in records], geom_select_uid)
Exemple #27
0
    def test_process_name(self):
        copy_path = os.path.join(self.current_dir_output, 'test_shp_process')
        sc = ShpCabinet()
        test_path = os.path.split(sc.get_shp_path('wc_4326'))[0]
        shutil.copytree(test_path, copy_path)

        shp_path = os.path.join(copy_path, 'wc_4326.shp')
        out_folder = tempfile.mkdtemp(dir=self.current_dir_output)
        sp = ShpProcess(shp_path, out_folder)
        sp.process(key='world_countries', ugid=None, name='new_id')
        path = os.path.join(out_folder, 'world_countries.shp')
        with fiona.open(path, 'r') as sci:
            uids = [record['properties']['new_id'] for record in sci]
        self.assertEqual(uids, range(1, 212))
Exemple #28
0
    def test_calculation_iteration_two_calculations(self):
        field = self.get_field(with_value=True, month_count=2)
        field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5, name='tmin', alias='tmin'))
        field.temporal.name_uid = 'tid'
        field.level.name_uid = 'lid'
        field.spatial.geom.name_uid = 'gid'

        grouping = ['month']
        tgd = field.temporal.get_grouping(grouping)
        mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64, add_parents=True)
        ret = mu.execute()
        thresh = Threshold(field=field, vc=ret, tgd=tgd, alias='a_treshold', add_parents=True,
                           parms={'operation': 'gte', 'threshold': 0.5})
        ret = thresh.execute()

        kwds = copy(field.__dict__)
        kwds.pop('_raw')
        kwds.pop('_variables')
        kwds.pop('_should_regrid')
        kwds.pop('_has_assigned_coordinate_system')
        kwds.pop('_attrs')
        kwds['name'] = kwds.pop('_name')
        kwds['temporal'] = tgd
        kwds['variables'] = ret
        cfield = DerivedField(**kwds)
        cfield.temporal.name_uid = 'tid'
        cfield.temporal.name_value = 'time'
        cfield.spatial.name_uid = 'gid'

        sc = ShpCabinet()
        meta = sc.get_meta('state_boundaries')
        sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_CALC)
        for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
            sp.add_field(cfield, ugeom=row)

        cids = set()
        for ii, row in enumerate(sp.get_iter_dict(melted=True)):
            cids.update([row[1]['cid']])
            if ii == 0:
                self.assertEqual(row[0].bounds, (-100.5, 39.5, -99.5, 40.5))
                self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000,
                                              'time': datetime.datetime(2000, 1, 16, 0, 0),
                                              'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1,
                                              'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean',
                                              'tid': 1, 'level': 50, 'day': 16})
            self.assertEqual(len(row), 2)
            self.assertEqual(len(row[1]), len(constants.HEADERS_CALC))
        self.assertEqual(ii + 1, 2 * 2 * 2 * 3 * 4 * 51 * 4)
        self.assertEqual(len(cids), 4)
Exemple #29
0
 def _format_string_element_(self, value):
     elements = value.split('|')
     try:
         elements = [float(e) for e in elements]
         minx, miny, maxx, maxy = elements
         geom = Polygon(
             ((minx, miny), (minx, maxy), (maxx, maxy), (maxx, miny)))
         self._assert_(geom.is_valid)
         ret = [{'ugid': 1, 'geom': geom}]
         self._bounds = elements
     except ValueError:
         from ocgis.util.shp_cabinet import ShpCabinet
         sc = ShpCabinet()
         ret = sc.get_geom_dict(value)
     return (ret)
Exemple #30
0
 def test_shp_process(self):
     test_path = '/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/test_data/test_shp_process'
     copy_path = os.path.join(self._test_dir,'test_shp_process')
     shutil.copytree(test_path,copy_path)
     shp_path = os.path.join(copy_path,'wc_4326.shp')
     sp = ShpProcess(shp_path)
     out_folder = tempfile.mkdtemp(dir=self._test_dir)
     sp.process(out_folder,'world_countries',ugid=None)
         
     sc = ShpCabinet(path=out_folder)
     select_ugid = [33,126,199]
     geoms = list(sc.iter_geoms('world_countries',select_ugid=select_ugid))
     self.assertEqual(len(geoms),3)
     names = [item['properties']['NAME'] for item in geoms]
     self.assertEqual(set(names),set(['Canada','Mexico','United States']))
Exemple #31
0
    def test_shapefiles_not_in_folders(self):
        for dirpath, dirnames, filenames in os.walk(ocgis.env.DIR_SHPCABINET):
            for filename in filenames:
                if filename.startswith(
                        'state_boundaries') or filename.startswith(
                            'world_countries'):
                    dst = os.path.join(self._test_dir, filename)
                    src = os.path.join(dirpath, filename)
                    shutil.copy2(src, dst)
        self.test_get_keys(dir_shpcabinet=self._test_dir)

        sc = ShpCabinet(path=self._test_dir)
        path = sc.get_shp_path('world_countries')
        self.assertEqual(path,
                         os.path.join(self._test_dir, 'world_countries.shp'))
Exemple #32
0
 def _run_(s, func):
     try:
         ds = ogr.Open(path)
         obj = ShpCabinet._get_features_object_(ds, select_sql_where=s)
         func(obj)
     finally:
         ds.Destroy()
Exemple #33
0
    def test_shp_process(self):
        test_path = '/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/test_data/test_shp_process'
        copy_path = os.path.join(self._test_dir, 'test_shp_process')
        shutil.copytree(test_path, copy_path)
        shp_path = os.path.join(copy_path, 'wc.shp')
        sp = ShpProcess(shp_path)
        out_folder = tempfile.mkdtemp(dir=self._test_dir)
        sp.process(out_folder, 'world_countries', ugid=None)

        sc = ShpCabinet(path=out_folder)
        select_ugid = [33, 126, 199]
        geoms = sc.get_geoms('world_countries', select_ugid=select_ugid)
        self.assertEqual(len(geoms), 3)
        names = [item['NAME'] for item in geoms]
        self.assertEqual(set(names), set(['Canada', 'Mexico',
                                          'United States']))
Exemple #34
0
    def test_geom(self):
        geom = make_poly((37.762, 38.222), (-102.281, -101.754))

        g = Geom(geom)
        self.assertEqual(type(g.value), GeometryDataset)
        g.value = None
        self.assertEqual(None, g.value)

        g = Geom(None)
        self.assertEqual(g.value, None)
        self.assertEqual(str(g), 'geom=None')

        g = Geom('-120|40|-110|50')
        self.assertEqual(g.value.spatial.geom[0].bounds,
                         (-120.0, 40.0, -110.0, 50.0))
        self.assertEqual(str(g), 'geom=-120.0|40.0|-110.0|50.0')

        g = Geom('mi_watersheds')
        self.assertEqual(str(g), 'geom=mi_watersheds')

        geoms = ShpCabinet().get_geoms('mi_watersheds')
        g = Geom('mi_watersheds')
        self.assertEqual(len(g.value), len(geoms))

        su = SelectUgid([1, 2, 3])
        g = Geom('mi_watersheds', select_ugid=su)
        self.assertEqual(len(g.value), 3)

        geoms = GeometryDataset(uid=[1, 2], geom=[geom, geom])
        g = Geom(geoms)

        bbox = [-120, 40, -110, 50]
        g = Geom(bbox)
        self.assertEqual(g.value.spatial.geom[0].bounds,
                         tuple(map(float, bbox)))
Exemple #35
0
 def _format_string_element_(self,value):
     elements = value.split('|')
     try:
         elements = [float(e) for e in elements]
         minx,miny,maxx,maxy = elements
         geom = Polygon(((minx,miny),
                         (minx,maxy),
                         (maxx,maxy),
                         (maxx,miny)))
         self._assert_(geom.is_valid)
         ret = [{'ugid':1,'geom':geom}]
         self._bounds = elements
     except ValueError:
         from ocgis.util.shp_cabinet import ShpCabinet
         sc = ShpCabinet()
         ret = sc.get_geom_dict(value)
     return(ret)
Exemple #36
0
 def test_none_path(self):
     try:
         ocgis.env.DIR_SHPCABINET = None
         with self.assertRaises(ValueError):
             ShpCabinet().get_geoms('state_boundaries')
             
     finally:
         ocgis.env.reset()
Exemple #37
0
 def test_iteration(self):
     field = self.get_field(with_value=True)
     
     field.temporal.name_uid = 'tid'
     field.level.name_uid = 'lid'
     field.spatial.geom.name_uid = 'gid'
     field.spatial.name_uid = 'gid'
     
     sc = ShpCabinet()
     meta = sc.get_meta('state_boundaries')
     sp = SpatialCollection(meta=meta,key='state_boundaries')
     for row in sc.iter_geoms('state_boundaries'):
         sp.add_field(row['properties']['UGID'],row['geom'],field.variables.keys()[0],
                      field,properties=row['properties'])
     for ii,row in enumerate(sp.get_iter_dict()):
         if ii == 1:
             self.assertEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'alias': 'tmax', 'did': 1, 'year': 2000, 'value': 0.7203244934421581, 'month': 1, 'variable': 'tmax', 'gid': 2, 'time': datetime.datetime(2000, 1, 1, 12, 0), 'tid': 1, 'level': 50, 'day': 1})
         self.assertIsInstance(row[0],MultiPolygon)
         self.assertEqual(len(row),2)
         self.assertEqual(len(row[1]),len(constants.raw_headers))
Exemple #38
0
 def test_attribute_flags(self):
     attr_flags = ['none','all']
     for attr_flag in attr_flags:
         ## make temporary directory
         path = get_temp_path(only_dir=True,nest=True)
         try:
             ## the shapefile key to use
             key = 'state_boundaries'
             ## get path to shapefile and copy directory to temporary location
             sc = ShpCabinet()
             origin = os.path.split(sc.get_shp_path(key))[0]
             shutil.copytree(origin,os.path.join(path,key))
             ## remove original config file
             os.remove(os.path.join(path,key,key+'.cfg'))
             ## write config file
             config = ConfigParser.ConfigParser()
             config.add_section('mapping')
             config.set('mapping','ugid','none')
             config.set('mapping','attributes',attr_flag)
             with open(os.path.join(path,key,key+'.cfg'),'w') as f:
                 config.write(f)
             ## load data
             sc = ShpCabinet(path)
             geoms = sc.get_geoms(key)
             
             for geom in geoms:
                 if attr_flag == 'none':
                     self.assertEqual(set(['ugid','geom']),set(geom.keys()))
                 else:
                     self.assertEqual(set(['ugid', 'state_name', 'state_fips', 'geom', 'state_abbr', 'id']),
                                      set(geom.keys()))
         finally:
             shutil.rmtree(path)
Exemple #39
0
    def test_iteration_methods(self):
        field = self.get_field(with_value=True)

        field.temporal.name_uid = 'tid'
        field.level.name_uid = 'lid'
        field.spatial.geom.name_uid = 'gid'
        field.spatial.name_uid = 'gid'

        sc = ShpCabinet()
        meta = sc.get_meta('state_boundaries')
        sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_RAW)
        for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True):
            sp.add_field(field, ugeom=row)
        for ii, row in enumerate(sp.get_iter_dict(melted=True)):
            if ii == 1:
                self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'alias': 'tmax', 'did': 1, 'year': 2000,
                                              'value': 0.7203244934421581, 'month': 1, 'variable': 'tmax', 'gid': 2,
                                              'time': datetime.datetime(2000, 1, 1, 12, 0), 'tid': 1, 'level': 50,
                                              'day': 1})
            self.assertIsInstance(row[0], MultiPolygon)
            self.assertEqual(len(row), 2)
            self.assertEqual(len(row[1]), len(constants.HEADERS_RAW))
Exemple #40
0
 def _write_(self):
     gid_file = []
     build = True
     with open(self.path,'w') as f:
         writer = csv.writer(f,dialect=OcgDialect)
         for coll in self:
             if build:
                 headers = coll.get_headers(upper=True)
                 if env.WRITE_TO_REFERENCE_PROJECTION:
                     projection = constants.reference_projection
                 else:
                     projection = coll._archetype.spatial.projection
                 ugid_idx = headers.index('UGID')
                 gid_idx = headers.index('GID')
                 did_idx = headers.index('DID')
                 writer.writerow(headers)
                 build = False
             for geom,row in coll.get_iter():
                 gid_file.append({'geom':geom,'did':row[did_idx],
                                  'ugid':row[ugid_idx],'gid':row[gid_idx]})
                 writer.writerow(row)
     
     if self.ops.aggregate is True and self.ops.abstraction == 'point':
         if env.VERBOSE:
             print('creating a UGID-GID shapefile is not necessary for aggregated point data. use UGID shapefile.')
     else:
         sc = ShpCabinet()
         shp_dir = os.path.join(self.outdir,'shp')
         try:
             os.mkdir(shp_dir)
         ## catch if the directory exists
         except OSError:
             if os.path.exists(shp_dir):
                 pass
             else:
                 raise
         shp_path = os.path.join(shp_dir,self.prefix+'_gid.shp')
         sc.write(gid_file,shp_path,sr=projection.sr)
Exemple #41
0
    def test_get_features_object(self):
        # test with a shapefile not having the default unique geometry identifier
        path = self.get_shapefile_path_with_no_ugid()
        keywords = dict(uid=[None, 'ID'],
                        select_uid=[None, [8, 11, 13]],
                        select_sql_where=[None, 'STATE_NAME = "Wisconsin"'])

        for k in self.iter_product_keywords(keywords):
            ds = ogr.Open(path)
            try:
                try:
                    obj = ShpCabinet._get_features_object_(ds, uid=k.uid, select_uid=k.select_uid,
                                                           select_sql_where=k.select_sql_where)
                except RuntimeError:
                    self.assertIsNone(k.uid)
                    self.assertIsNotNone(k.select_uid)
                    continue

                if k.select_sql_where is not None:
                    length = 1
                elif k.select_uid is not None:
                    length = 3
                else:
                    length = 11
                self.assertEqual(len(obj), length)
                self.assertIsInstance(obj, Layer)
            finally:
                ds.Destroy()

        # test on a shapefile having the default unique geometry identifier
        path = ShpCabinet().get_shp_path('state_boundaries')
        ds = ogr.Open(path)
        try:
            obj = ShpCabinet._get_features_object_(ds, select_uid=[8, 11, 13])
            self.assertEqual(len(obj), 3)
        finally:
            ds.Destroy()
Exemple #42
0
 def test_load_all(self):
     sc = ShpCabinet()
     for key in sc.keys():
         geoms = sc.get_geoms(key)
         self.assertTrue(len(geoms) > 2)
Exemple #43
0
 def test_get_keys(self):
     sc = ShpCabinet()
     ret = sc.keys()
     self.assertTrue(len(ret) >= 1)
Exemple #44
0
 def test_bad_path(self):
     bp = '/a/bad/location'
     with self.assertRaises(ValueError):
         ShpCabinet(bp).get_geoms('state_boundaries')
Exemple #45
0
 def _sc(self):
     if self.__sc is None:
         self.__sc = ShpCabinet()
     return (self.__sc)
Exemple #46
0
 def test_load_all(self):
     raise (SkipTest('too long for no benefit'))
     sc = ShpCabinet()
     for key in sc.keys():
         geoms = sc.get_geoms(key)
         self.assertTrue(len(geoms) > 2)
Exemple #47
0
 def test_huc_loading(self):
     raise (SkipTest('tests loading time for HUC data'))
     sc = ShpCabinet()
     geoms = sc.get_geoms('WBDHU12_June2013', select_ugid=[2221, 5000])
     self.assertEqual(len(geoms), 2)
     self.assertEqual(geoms[0]['UGID'], 2221)
Exemple #48
0
    def _write_(self):
        gid_file = OrderedDict()
        build = True
        is_aggregated = self.ops.aggregate
        with open(self.path, 'w') as f:
            ocgis_lh(msg='opened csv file: {0}'.format(self.path),
                     level=logging.DEBUG,
                     logger='conv.csv+')
            writer = csv.writer(f, dialect=OcgDialect)
            for coll in self:
                ocgis_lh('writing collection',
                         'conv.csv+',
                         level=logging.DEBUG)
                if build:
                    ocgis_lh('starting build',
                             'conv.csv+',
                             level=logging.DEBUG)
                    headers = coll.get_headers(upper=True)
                    if env.WRITE_TO_REFERENCE_PROJECTION:
                        projection = env.REFERENCE_PROJECTION
                    else:
                        projection = coll._archetype.spatial.projection
                    writer.writerow(headers)
                    build = False
                    ocgis_lh(msg='build finished'.format(self.path),
                             level=logging.DEBUG,
                             logger='conv.csv+')
                for geom, row, geom_ids in coll.get_iter(
                        with_geometry_ids=True):
                    if not is_aggregated:
                        ugid = geom_ids['ugid']
                        did = geom_ids['did']
                        gid = geom_ids['gid']
                        if ugid not in gid_file:
                            gid_file[ugid] = OrderedDict()
                        if did not in gid_file[ugid]:
                            gid_file[ugid][did] = OrderedDict()
                        gid_file[ugid][did][gid] = geom
                    writer.writerow(row)
                ocgis_lh('finished writing collection',
                         'conv.csv+',
                         level=logging.DEBUG)

        if is_aggregated is True:
            ocgis_lh(
                'creating a UGID-GID shapefile is not necessary for aggregated data. use UGID shapefile.',
                'conv.csv+', logging.WARN)
        else:
            ocgis_lh('writing UGID-GID shapefile', 'conv.csv+', logging.DEBUG)
            sc = ShpCabinet()
            shp_dir = os.path.join(self.outdir, 'shp')
            try:
                os.mkdir(shp_dir)
            ## catch if the directory exists
            except OSError:
                if os.path.exists(shp_dir):
                    pass
                else:
                    raise
            shp_path = os.path.join(shp_dir, self.prefix + '_gid.shp')

            def iter_gid_file():
                for ugid, did_gid in gid_file.iteritems():
                    for did, gid_geom in did_gid.iteritems():
                        for gid, geom in gid_geom.iteritems():
                            yield ({
                                'geom': geom,
                                'DID': did,
                                'UGID': ugid,
                                'GID': gid
                            })

            sc.write(iter_gid_file(), shp_path, sr=projection.sr)
Exemple #49
0
 def california(self):
     sc = ShpCabinet()
     ret = sc.get_geom_dict('state_boundaries', {'ugid': [25]})
     return (ret)
Exemple #50
0
 def world_countries(self):
     sc = ShpCabinet()
     ret = sc.get_geom_dict('world_countries')
     return (ret)
Exemple #51
0
 def alaska(self):
     sc = ShpCabinet()
     ret = sc.get_geom_dict('state_boundaries', {'ugid': [51]})
     return (ret)
Exemple #52
0
 def state_boundaries(self):
     sc = ShpCabinet()
     ret = sc.get_geom_dict('state_boundaries')
     return (ret)