def test_iter_geoms(self): sc = ShpCabinet() it = sc.iter_geoms('state_boundaries') geoms = list(it) self.assertEqual(len(geoms), 51) self.assertEqual(geoms[12]['properties']['STATE_NAME'], 'New Hampshire') for geom in geoms: self.assertIn(type(geom['geom']), (Polygon, MultiPolygon)) # test with a shapefile not having a unique identifier env.DEFAULT_GEOM_UID = 'ggidd' new = self.get_shapefile_path_with_no_ugid() sc = ShpCabinet() target = list(sc.iter_geoms(path=new)) self.assertEqual(len(target), 11) self.assertEqual(target[0]['properties'][env.DEFAULT_GEOM_UID], 1) self.assertEqual(target[3]['properties'][env.DEFAULT_GEOM_UID], 4) target = list(sc.iter_geoms(path=new, uid='ID')) self.assertNotIn(env.DEFAULT_GEOM_UID, target[9]['properties']) self.assertEqual(int, type(target[7]['properties']['ID'])) target = list(sc.iter_geoms(path=new, uid='ID', as_spatial_dimension=True)) ref = target[4] self.assertEqual(ref.uid[0, 0], 10) self.assertNotIn(env.DEFAULT_GEOM_UID, ref.properties.dtype.names) # test with a different geometry unique identifier path = self.get_shapefile_path_with_no_ugid() geom_select_uid = [12, 15] geom_uid = 'ID' sc = ShpCabinet() records = list(sc.iter_geoms(path=path, uid=geom_uid, select_uid=geom_select_uid)) self.assertEqual(len(records), 2) self.assertEqual([r['properties']['ID'] for r in records], geom_select_uid)
def test_iter_all(self): raise(SkipTest('dev - long')) sc = ShpCabinet() for key in sc.keys(): print key for geom in sc.iter_geoms(key): self.assertTrue(True)
def test_multivariate_iteration(self): field = self.get_field(with_value=True, month_count=1) field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5, name='tmin', alias='tmin')) field.temporal.name_uid = 'tid' field.level.name_uid = 'lid' field.spatial.geom.name_uid = 'gid' div = Divide(field=field, parms={'arr1': 'tmin', 'arr2': 'tmax'}, alias='some_division', dtype=np.float64) ret = div.execute() cfield = DerivedMultivariateField(variables=ret, realization=field.realization, temporal=field.temporal, level=field.level, spatial=field.spatial, meta=field.meta, uid=field.uid) cfield.spatial.name_uid = 'gid' sc = ShpCabinet() meta = sc.get_meta('state_boundaries') sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_MULTI) for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True): sp.add_field(cfield, ugeom=row) for ii, row in enumerate(sp.get_iter_dict(melted=True)): if ii == 0: self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'cid': 1, 'did': None, 'year': 2000, 'time': datetime.datetime(2000, 1, 1, 12, 0), 'calc_alias': 'some_division', 'value': 12.989774984574424, 'month': 1, 'gid': 1, 'calc_key': 'divide', 'tid': 1, 'level': 50, 'day': 1}) self.assertEqual(ii + 1, 2 * 31 * 2 * 3 * 4 * 51)
def test_calculation_iteration(self): field = self.get_field(with_value=True,month_count=2) field.variables.add_variable(Variable(value=field.variables['tmax'].value+5, name='tmin',alias='tmin')) field.temporal.name_uid = 'tid' field.level.name_uid = 'lid' field.spatial.geom.name_uid = 'gid' grouping = ['month'] tgd = field.temporal.get_grouping(grouping) mu = Mean(field=field,tgd=tgd,alias='my_mean') ret = mu.execute() kwds = copy(field.__dict__) kwds.pop('_raw') kwds.pop('_variables') kwds['temporal'] = tgd kwds['variables'] = ret cfield = DerivedField(**kwds) cfield.temporal.name_uid = 'tid' cfield.temporal.name_value = 'time' cfield.spatial.name_uid = 'gid' sc = ShpCabinet() meta = sc.get_meta('state_boundaries') sp = SpatialCollection(meta=meta,key='state_boundaries',headers=constants.calc_headers) for row in sc.iter_geoms('state_boundaries'): sp.add_field(row['properties']['UGID'],row['geom'],cfield.variables.keys()[0], cfield,properties=row['properties']) for ii,row in enumerate(sp.get_iter_dict()): if ii == 0: self.assertEqual(row[0].bounds,(-100.5, 39.5, -99.5, 40.5)) self.assertDictEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000, 'time': datetime.datetime(2000, 1, 16, 0, 0), 'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1, 'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean', 'tid': 1, 'level': 50, 'day': 16}) self.assertEqual(len(row),2) self.assertEqual(len(row[1]),len(constants.calc_headers))
def get_collection(self): field = self.get_field(with_value=True) sc = ShpCabinet() meta = sc.get_meta('state_boundaries') sp = SpatialCollection(meta=meta, key='state_boundaries') for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True): sp.add_field(field, ugeom=row) return sp
def test_iter_geoms(self): sc = ShpCabinet() it = sc.iter_geoms('state_boundaries') geoms = list(it) self.assertEqual(len(geoms),51) self.assertEqual(geoms[12]['properties']['STATE_NAME'],'New Hampshire') for geom in geoms: self.assertIn(type(geom['geom']),(Polygon,MultiPolygon))
def test_iter_geoms_no_load_geoms(self): sc = ShpCabinet() it = sc.iter_geoms('state_boundaries',load_geoms=False) geoms = list(it) self.assertEqual(len(geoms),51) self.assertEqual(geoms[12]['properties']['STATE_NAME'],'New Hampshire') for geom in geoms: self.assertNotIn('geom',geom)
def test_constructor(self): field = self.get_field(with_value=True) sc = ShpCabinet() meta = sc.get_meta('state_boundaries') sp = SpatialCollection(meta=meta,key='state_boundaries') for row in sc.iter_geoms('state_boundaries'): sp.add_field(row['properties']['UGID'],row['geom'],field.variables.keys()[0], field,properties=row['properties']) self.assertEqual(len(sp),51) self.assertIsInstance(sp.geoms[25],MultiPolygon) self.assertIsInstance(sp.properties[25],dict) self.assertEqual(sp[25]['tmax'].variables['tmax'].value.shape,(2, 31, 2, 3, 4))
def test_shp_process(self): copy_path = os.path.join(self._test_dir,'test_shp_process') shutil.copytree(self._test_path,copy_path) shp_path = os.path.join(copy_path,'wc_4326.shp') out_folder = tempfile.mkdtemp(dir=self._test_dir) sp = ShpProcess(shp_path,out_folder) sp.process(key='world_countries',ugid=None) sc = ShpCabinet(path=out_folder) select_ugid = [33,126,199] geoms = list(sc.iter_geoms('world_countries',select_ugid=select_ugid)) self.assertEqual(len(geoms),3) names = [item['properties']['NAME'] for item in geoms] self.assertEqual(set(names),set(['Canada','Mexico','United States']))
def test_calculation_iteration_two_calculations(self): field = self.get_field(with_value=True, month_count=2) field.variables.add_variable(Variable(value=field.variables['tmax'].value + 5, name='tmin', alias='tmin')) field.temporal.name_uid = 'tid' field.level.name_uid = 'lid' field.spatial.geom.name_uid = 'gid' grouping = ['month'] tgd = field.temporal.get_grouping(grouping) mu = Mean(field=field, tgd=tgd, alias='my_mean', dtype=np.float64, add_parents=True) ret = mu.execute() thresh = Threshold(field=field, vc=ret, tgd=tgd, alias='a_treshold', add_parents=True, parms={'operation': 'gte', 'threshold': 0.5}) ret = thresh.execute() kwds = copy(field.__dict__) kwds.pop('_raw') kwds.pop('_variables') kwds.pop('_should_regrid') kwds.pop('_has_assigned_coordinate_system') kwds.pop('_attrs') kwds['name'] = kwds.pop('_name') kwds['temporal'] = tgd kwds['variables'] = ret cfield = DerivedField(**kwds) cfield.temporal.name_uid = 'tid' cfield.temporal.name_value = 'time' cfield.spatial.name_uid = 'gid' sc = ShpCabinet() meta = sc.get_meta('state_boundaries') sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_CALC) for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True): sp.add_field(cfield, ugeom=row) cids = set() for ii, row in enumerate(sp.get_iter_dict(melted=True)): cids.update([row[1]['cid']]) if ii == 0: self.assertEqual(row[0].bounds, (-100.5, 39.5, -99.5, 40.5)) self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'cid': 1, 'did': 1, 'year': 2000, 'time': datetime.datetime(2000, 1, 16, 0, 0), 'calc_alias': 'my_mean_tmax', 'value': 0.44808476666433006, 'month': 1, 'alias': 'tmax', 'variable': 'tmax', 'gid': 1, 'calc_key': 'mean', 'tid': 1, 'level': 50, 'day': 16}) self.assertEqual(len(row), 2) self.assertEqual(len(row[1]), len(constants.HEADERS_CALC)) self.assertEqual(ii + 1, 2 * 2 * 2 * 3 * 4 * 51 * 4) self.assertEqual(len(cids), 4)
def test_shp_process(self): test_path = '/home/local/WX/ben.koziol/Dropbox/nesii/project/ocg/bin/test_data/test_shp_process' copy_path = os.path.join(self._test_dir,'test_shp_process') shutil.copytree(test_path,copy_path) shp_path = os.path.join(copy_path,'wc_4326.shp') sp = ShpProcess(shp_path) out_folder = tempfile.mkdtemp(dir=self._test_dir) sp.process(out_folder,'world_countries',ugid=None) sc = ShpCabinet(path=out_folder) select_ugid = [33,126,199] geoms = list(sc.iter_geoms('world_countries',select_ugid=select_ugid)) self.assertEqual(len(geoms),3) names = [item['properties']['NAME'] for item in geoms] self.assertEqual(set(names),set(['Canada','Mexico','United States']))
def test_iteration(self): field = self.get_field(with_value=True) field.temporal.name_uid = 'tid' field.level.name_uid = 'lid' field.spatial.geom.name_uid = 'gid' field.spatial.name_uid = 'gid' sc = ShpCabinet() meta = sc.get_meta('state_boundaries') sp = SpatialCollection(meta=meta,key='state_boundaries') for row in sc.iter_geoms('state_boundaries'): sp.add_field(row['properties']['UGID'],row['geom'],field.variables.keys()[0], field,properties=row['properties']) for ii,row in enumerate(sp.get_iter_dict()): if ii == 1: self.assertEqual(row[1],{'lid': 1, 'ugid': 1, 'vid': 1, 'alias': 'tmax', 'did': 1, 'year': 2000, 'value': 0.7203244934421581, 'month': 1, 'variable': 'tmax', 'gid': 2, 'time': datetime.datetime(2000, 1, 1, 12, 0), 'tid': 1, 'level': 50, 'day': 1}) self.assertIsInstance(row[0],MultiPolygon) self.assertEqual(len(row),2) self.assertEqual(len(row[1]),len(constants.raw_headers))
def test_iteration_methods(self): field = self.get_field(with_value=True) field.temporal.name_uid = 'tid' field.level.name_uid = 'lid' field.spatial.geom.name_uid = 'gid' field.spatial.name_uid = 'gid' sc = ShpCabinet() meta = sc.get_meta('state_boundaries') sp = SpatialCollection(meta=meta, key='state_boundaries', headers=constants.HEADERS_RAW) for row in sc.iter_geoms('state_boundaries', as_spatial_dimension=True): sp.add_field(field, ugeom=row) for ii, row in enumerate(sp.get_iter_dict(melted=True)): if ii == 1: self.assertDictEqual(row[1], {'lid': 1, 'ugid': 1, 'vid': 1, 'alias': 'tmax', 'did': 1, 'year': 2000, 'value': 0.7203244934421581, 'month': 1, 'variable': 'tmax', 'gid': 2, 'time': datetime.datetime(2000, 1, 1, 12, 0), 'tid': 1, 'level': 50, 'day': 1}) self.assertIsInstance(row[0], MultiPolygon) self.assertEqual(len(row), 2) self.assertEqual(len(row[1]), len(constants.HEADERS_RAW))
def test_iter_geoms_select_ugid_is_sorted(self): sc = ShpCabinet() with self.assertRaises(ValueError): list(sc.iter_geoms('state_boundaries',select_ugid=[23,18]))
def test_iter_geoms_select_ugid(self): sc = ShpCabinet() it = sc.iter_geoms('state_boundaries',select_ugid=[13]) geoms = list(it) self.assertEqual(len(geoms),1) self.assertEqual(geoms[0]['properties']['STATE_NAME'],'New Hampshire')
def test_iter_geoms_select_sql_where(self): sc = ShpCabinet() sql = 'STATE_NAME = "New Hampshire"' self.assertEqual(len(list(sc.iter_geoms('state_boundaries', select_sql_where=sql))), 1)
def test_iter_geoms(self): sc = ShpCabinet() it = sc.iter_geoms('state_boundaries') geoms = list(it) self.assertEqual(len(geoms),51) self.assertEqual(geoms[12]['properties']['STATE_NAME'],'New Hampshire')