def test_cookiecutter_gain(self): """ Cookie cutter on int image with floating point gain """ bbox = np.array([0.0, 0.0, 1.0, 1.0]) geoimg = gp.GeoImage.create(xsz=1000, ysz=1000, bbox=bbox, dtype='int16') geoimg.set_gain(0.0001) arr = np.zeros((1000,1000)) + 0.0001 arr[0:500,:] = 0.0002 geoimg.write(deepcopy(arr)) res = geoimg.resolution() imgout = alg.cookie_cutter([geoimg], xres=res.x(), yres=res.y()) np.testing.assert_array_equal(arr, imgout.read())
def fetch(items, geometry, keys, path=config.DATADIR, filename=config.FILENAME, proj=None, res=None): """ This fetches data from just the AOI and clips it """ derived_item = create_derived_item(items, geometry) bands = [] for k in keys: bands += items[0].asset(k).get('eo:bands', []) filename = items[0].get_filename(path=path, filename=filename).replace( '.json', '.tif') os.makedirs(os.path.dirname(filename), exist_ok=True) derived_item._data['assets'] = { 'image': { 'type': 'image/vnd.stac.geotiff; cloud-optimized=true', 'title': 'Clipped image', 'href': filename, 'eo:bands': bands } } if os.path.exists(filename): return derived_item try: geoimgs = [] for item in items: geoimgs.append(open_image(item, keys)) # default to first image res and srs if res is None: res = geoimgs[0].resolution() res = [res.x(), res.y()] elif len(res) == 1: res = [res[0], res[0]] if proj is None: proj = geoimgs[0].srs() geovec = geometry_to_GeoVector(geometry) imgout = algs.cookie_cutter(geoimgs, filename, geovec[0], xres=res[0], yres=res[1], proj=proj, options=OPTS) logger.info("Created %s" % imgout.filename()) return derived_item except Exception as err: print('Error: ', str(err))
def test_cookiecutter_real_crop(self): """ Test cookie cutter with cropping """ geoimg = gpt.get_test_image().select(['red', 'green', 'blue']) vpath = os.path.join(os.path.dirname(__file__), 'vectors') feature = gp.GeoVector(os.path.join(vpath, 'aoi1_epsg32416.shp')) imgout = alg.cookie_cutter([geoimg], feature=feature[0], xres=30.0, yres=30.0, crop=True) extin = feature.extent() extout = imgout.extent() self.assertTrue(extout.x0() >= extin.x0()) self.assertTrue(extout.y0() >= extin.y0()) self.assertTrue(extout.x1() <= extin.x1()) self.assertTrue(extout.y1() <= extin.y1())
def test_cookiecutter_real_reproj(self): """ Test with different projection """ geoimg = gpt.get_test_image().select(['red', 'green', 'blue']) vpath = os.path.join(os.path.dirname(__file__), 'vectors') feature = gp.GeoVector(os.path.join(vpath, 'aoi1_epsg32416.shp')) extin = feature.extent() # test extent matches feature imgout = alg.cookie_cutter([geoimg], feature=feature[0], xres=30.0, yres=30.0) extout = imgout.extent() self.assertAlmostEqual(extout.x0(), extin.x0()) self.assertAlmostEqual(extout.y0(), extin.y0()) self.assertAlmostEqual(extout.x1(), extin.x1()) self.assertAlmostEqual(extout.y1(), extin.y1())
def test_cookiecutter_real(self): """ Cookie cutter on single real image """ geoimg = gpt.get_test_image().select(['red']) #, 'green', 'blue']) vpath = os.path.join(os.path.dirname(__file__), 'vectors') # test with feature of different projection feature = gp.GeoVector(os.path.join(vpath, 'aoi1_epsg4326.shp')) extin = feature.extent() imgout = alg.cookie_cutter([geoimg], feature=feature[0], xres=0.0003, yres=0.0003) extout = imgout.extent() self.assertAlmostEqual(extout.x0(), extin.x0()) self.assertAlmostEqual(extout.y0(), extin.y0()) self.assertAlmostEqual(extout.x1(), extin.x1()) self.assertAlmostEqual(extout.y1(), extin.y1())
def test_cookiecutter(self): """ Create mosaic from multiple images (cookie cutter) """ bbox1 = np.array([0.0, 0.0, 1.0, 1.0]) geoimg1 = gp.GeoImage.create(xsz=1000, ysz=1000, bbox=bbox1) bbox2 = np.array([1.0, 0.0, 1.0, 1.0]) geoimg2 = gp.GeoImage.create(xsz=1000, ysz=1000, bbox=bbox2) res = geoimg1.resolution() imgout = alg.cookie_cutter([geoimg1, geoimg2], xres=res.x(), yres=res.y()) ext = imgout.extent() self.assertEqual(ext.x0(), 0.0) self.assertEqual(ext.y0(), 0.0) self.assertEqual(ext.width(), 2.0) self.assertEqual(ext.height(), 1.0)
def test_cookiecutter_gain(self): """ Cookie cutter on int image with floating point gain """ bbox = np.array([0.0, 0.0, 1.0, 1.0]) geoimg = gp.GeoImage.create(xsz=1000, ysz=1000, bbox=bbox, dtype='int16') geoimg.set_gain(0.0001) arr = np.zeros((1000, 1000)) arr[0:500, :] = 0.0002 geoimg.write(deepcopy(arr)) res = geoimg.resolution() imgout = alg.cookie_cutter([geoimg], xres=res.x(), yres=res.y()) np.testing.assert_array_almost_equal(arr, imgout.read())
def test_cookiecutter(self): """ Create mosaic from multiple images (cookie cutter) """ bbox1 = np.array([0.0, 0.0, 1.0, 1.0]) geoimg1 = gp.GeoImage.create(xsz=1000, ysz=1000, bbox=bbox1) bbox2 = np.array([1.0, 0.0, 1.0, 1.0]) geoimg2 = gp.GeoImage.create(xsz=1000, ysz=1000, bbox=bbox2) res = geoimg1.resolution() imgout = alg.cookie_cutter([geoimg1, geoimg2], xres=res.x(), yres=res.y()) ext = imgout.extent() self.assertEqual(ext.x0(), 0.0) self.assertEqual(ext.y0(), 0.0) self.assertEqual(ext.width(), 2.0) self.assertEqual(ext.height(), 1.0)
def test_cookiecutter_real(self): """ Cookie cutter on single real image """ geoimg = gpt.get_test_image().select(['red']) #, 'green', 'blue']) vpath = os.path.join(os.path.dirname(__file__), 'vectors') # test with feature of different projection feature = gp.GeoVector(os.path.join(vpath, 'aoi1_epsg4326.shp')) extin = feature.extent() imgout = alg.cookie_cutter([geoimg], feature=feature[0], xres=0.0003, yres=0.0003) extout = imgout.extent() self.assertAlmostEqual(extout.x0(), extin.x0()) self.assertAlmostEqual(extout.y0(), extin.y0()) self.assertAlmostEqual(extout.x1(), extin.x1()) self.assertAlmostEqual(extout.y1(), extin.y1())
def test_cookiecutter_real_crop(self): """ Test cookie cutter with cropping """ geoimg = gpt.get_test_image().select(['red', 'green', 'blue']) vpath = os.path.join(os.path.dirname(__file__), 'vectors') feature = gp.GeoVector(os.path.join(vpath, 'aoi1_epsg32416.shp')) imgout = alg.cookie_cutter([geoimg], feature=feature[0], xres=30.0, yres=30.0, crop=True) extin = feature.extent() extout = imgout.extent() self.assertTrue(extout.x0() >= extin.x0()) self.assertTrue(extout.y0() >= extin.y0()) self.assertTrue(extout.x1() <= extin.x1()) self.assertTrue(extout.y1() <= extin.y1())
def test_cookiecutter_real_reproj(self): """ Test with different projection """ geoimg = gpt.get_test_image().select(['red', 'green', 'blue']) vpath = os.path.join(os.path.dirname(__file__), 'vectors') feature = gp.GeoVector(os.path.join(vpath, 'aoi1_epsg32416.shp')) extin = feature.extent() # test extent matches feature imgout = alg.cookie_cutter([geoimg], feature=feature[0], xres=30.0, yres=30.0) extout = imgout.extent() self.assertAlmostEqual(extout.x0(), extin.x0()) self.assertAlmostEqual(extout.y0(), extin.y0()) self.assertAlmostEqual(extout.x1(), extin.x1()) self.assertAlmostEqual(extout.y1(), extin.y1())
def fetch(scenes, assets, geovector, basename='image'): """ This fetches data from just the AOI and clips it """ opts = { 'COMPRESS': 'DEFLATE', 'PREDICTOR': '2', 'INTERLEAVE': 'BAND', 'TILED': 'YES', 'BLOCKXSIZE': '512', 'BLOCKYSIZE': '512' } path = scenes[0].get_path() fname = scenes[0].get_filename() # TODO - different paths for sat-fetch ? fout = os.path.join(path, fname + '.tif') # create derived scene metadata derived_scene = Scene.create_derived(scenes) derived_scene.feature['geometry'] = json.loads(geovector.json_geometry()) # add asset(s) to derived scene bands = [] for a in assets: bands += scenes[0].asset(a)['eo:bands'] derived_scene.feature['assets'][basename] = { 'href': fout, 'eo:bands': bands } if os.path.exists(fout): return derived_scene try: geoimgs = [] for s in scenes: geoimgs.append(open_image(s, assets)) # default to first image res and srs res = geoimgs[0].resolution() imgout = algs.cookie_cutter(geoimgs, fout, geovector, xres=res.x(), yres=res.y(), proj=geoimgs[0].srs(), options=opts) logger.info("Created %s" % imgout.filename()) return derived_scene except Exception as err: print('Error: ', str(err))
def test_cookiecutter(self): """ Create mosaic from multiple images (cookie cutter) """ bbox1 = np.array([0.0, 0.0, 1.0, 1.0]) geoimg1 = gp.GeoImage.create(xsz=1000, ysz=1000, bbox=bbox1) bbox2 = np.array([1.0, 0.0, 1.0, 1.0]) geoimg2 = gp.GeoImage.create(xsz=1000, ysz=1000, bbox=bbox2) res = geoimg1.resolution() imgout = alg.cookie_cutter([geoimg1, geoimg2], xres=res.x(), yres=res.y()) ext = imgout.extent() # This appears to be accurate to 7 decimal places. # Is something getting converted from a double to a float somewhere? self.assertAlmostEqual(ext.x0(), 0.0) self.assertAlmostEqual(ext.y0(), 0.0) self.assertAlmostEqual(ext.width(), 2.0, places=6) self.assertAlmostEqual(ext.height(), 1.0) # '' self.assertAlmostEqual(imgout.resolution().x(), res.x()) self.assertAlmostEqual(imgout.resolution().y(), res.y())
def test_cookiecutter_real_reproj(self): """ Test with different projection """ geoimg = gpt.get_test_image().select(['red', 'green', 'blue']) vpath = os.path.join(os.path.dirname(__file__), 'vectors') feature = gp.GeoVector(os.path.join(vpath, 'aoi1_epsg32416.shp')) extin = feature.extent() # test extent matches feature imgout = alg.cookie_cutter([geoimg], feature=feature[0], xres=30.0, yres=30.0) extout = imgout.extent() self.assertAlmostEqual(extout.x0() + 15, extin.x0()) self.assertAlmostEqual(extout.y0() + 15, extin.y0()) # cookie cutter will never add more than a pixel and a half in width self.assertTrue(extout.x1() - extin.x1() < 45.0) self.assertTrue(extout.y1() - extin.y1() < 45.0) self.assertEqual(imgout.resolution().x(), 30.0) self.assertEqual(imgout.resolution().y(), -30.0)
def generate_stats(): geotifs = [ op.join(os.getcwd(), f) for f in os.listdir('.') if f.endswith('.tif') ] geojsons = [ op.join(os.getcwd(), geoj) for geoj in os.listdir('.') if geoj.endswith('.geojson') ] output_path = op.join(os.getcwd(), 'outputs') output_files = [] if not op.isdir(output_path): makedirs(output_path) for geotif in geotifs: for geojson in geojsons: fname = op.splitext(geotif)[0] basename = fname.split('/')[-1] filename = op.splitext(geojson)[0] geo_fname = filename.split('/')[-1] date = basename[10:16] tile = basename[28:35] geoimg = gippy.GeoImage.open([geotif]) geoimg.set_nodata(0) geovec = gippy.GeoVector(geojson) res = geoimg.resolution() output_filename = '{}_{}_{}.csv'.format(geo_fname, date, tile) f = open(output_filename, 'w') f.write('min,max,mean,stddev,skew,count\n') fout = geoimg.basename() + '_{}.tif'.format(str(geo_fname)) imgout = alg.cookie_cutter([geoimg], fout, geovec[0], xres=res.x(), yres=res.y(), proj=geoimg.srs()) stats = imgout[0].stats() f.write(','.join([str(s) for s in stats]) + '\n') f.close() if output_filename not in output_files: output_files.append(output_filename) shutil.move(os.path.join('.', fout), os.path.join(output_path, fout)) print('{} site stats extracted!'.format(geojson)) return output_files
def test_cookiecutter_real(self): """ Cookie cutter on single real image """ geoimg = gpt.get_test_image().select(['red']) #, 'green', 'blue']) iext = geoimg.extent() vpath = os.path.join(os.path.dirname(__file__), 'vectors') # test with feature of different projection feature = gp.GeoVector(os.path.join(vpath, 'aoi1_epsg4326.shp')) extin = feature.extent() imgout = alg.cookie_cutter([geoimg], feature=feature[0], xres=0.0003, yres=0.0003) extout = imgout.extent() self.assertAlmostEqual(extout.x0() + 0.00015, extin.x0()) self.assertAlmostEqual(extout.y0() + 0.00015, extin.y0()) # cookie cutter will never add more than a pixel and a half in width self.assertTrue(extout.x1() - extin.x1() < 0.0045) self.assertTrue(extout.y1() - extin.y1() < 0.0045) self.assertAlmostEqual(imgout.resolution().x(), 0.0003) self.assertAlmostEqual(imgout.resolution().y(), -0.0003)
def main(scenes, aoi, datadir='./', bands=None): scenes = Scenes.load(scenes) print(scenes.text_calendar()) bname = os.path.splitext(os.path.basename(aoi))[0] features = gippy.GeoVector(aoi) if not os.path.exists(datadir): os.makedirs(datadir) gippy.Options.set_verbose(5) #opts = {'COMPRESS': 'DEFLATE', 'PREDICTOR': '2', 'TILED': 'YES', 'BLOCKXSIZE': '512', 'BLOCKYSIZE': '512'} #opts = {'COMPRESS': 'LZW', 'TILED': 'YES', 'BLOCKXSIZE': '512', 'BLOCKYSIZE': '512'} opts = {'TILED': 'YES', 'BLOCKXSIZE': '512', 'BLOCKYSIZE': '512'} for date in scenes.dates(): _scenes = [s for s in scenes if s.date == date] outname = '%s_%s.tif' % (_scenes[0].date, _scenes[0].platform) fout = os.path.join(datadir, outname) if not os.path.exists(fout): try: geoimgs = [] for s in _scenes: links = s.links() if bands is None: bands = links.keys() filenames = [links[k].replace('https:/', '/vsicurl') for k in sorted(links) if k in bands] geoimg = gippy.GeoImage.open(filenames) geoimg.set_nodata(0) geoimgs.append(geoimg) # default to first image res and srs res = geoimgs[0].resolution() imgout = algs.cookie_cutter(geoimgs, fout, features[0], xres=res.x(), yres=res.y(), proj=geoimgs[0].srs(), options=opts) except Exception as err: print('Error: ', str(err))
def mask_with_vector(geoimg, vector, filename=''): """ Mask geoimage with a vector """ ext = geoimg.geo_extent() ds, layer = open_vector(vector[0], vector[1]) geovec = get_features(layer, bbox=[ext.x0(), ext.y0(), ext.x1(), ext.y1()], union=True) res = geoimg.resolution() logger.info('Saving to file %s' % filename, action='Save file', actee=filename, actor=__name__) imgout = alg.cookie_cutter([geoimg], filename=filename, feature=geovec[0], proj=geoimg.srs(), xres=res.x(), yres=res.y()) return imgout
def main(args=None): print('started') if args is None: args = sys.argv if isinstance(args, str): args = args.split(' ') maxRows = '100' outDir = args[1] #where to put the mosaic zipDir = outDir + '/zip/' #where to put downloads unzipDir = outDir + '/unzip/' #where to put working files #create directory to store downloads, output, and working files if not os.path.isdir(outDir): os.mkdir(outDir) if not os.path.isdir(zipDir): os.mkdir(zipDir) if not os.path.isdir(unzipDir): os.mkdir(unzipDir) bbox = args[2] #corners of the bounding box username = args[3] password = args[4] product = 'GRD' mission = "1" #min sense time end = "NOW" #max sense time beginning = ABS_START i = 5 #argument index while i < len(args): arg = args[i] if arg == '-m': i += 1 mission = args[i] elif arg == '-e': i += 1 end = args[i] + MIDNIGHT elif arg == '-b': i += 1 beginning = args[i] + MIDNIGHT elif arg == '-p': i += 1 maxRows = args[i] elif arg == '-t': i += 1 type = args[i].upper() else: print('Unrecognized flag ' + arg) print(CODE_2_MEANING) sys.exit(2) i += 1 #create strings that will be used as options for querying scihub ingestOption = INGESTION_KEYWORD + ':[' + beginning + ' TO ' + end + ']' p = parse_bbox(bbox) #Point 0 appears twice because the polygon must be a closed loop geographicType = POLYGON_KEYWORD + p[0] + POINT_DELIM + p[1] + POINT_DELIM\ + p[2] + POINT_DELIM + p[3] + POINT_DELIM + p[0] + '))' footprintOption = FOOTPRINT_KEYWORD + geographicType + ')"' platformOption = 'platformname:Sentinel-' + mission if mission == '1': productOption = PRODUCT_KEYWORD + product + QUERY_DELIM else: productOption = '' #there's no QUERY_DELIM between product option and platform option because product option is sometime blank queryURL = URL_START + ingestOption + QUERY_DELIM + footprintOption + QUERY_DELIM + productOption\ + platformOption + '&rows=' + maxRows + '&start=0&format=json' queryURL = queryURL.replace(' ', '%20') session.auth = (username, password) print queryURL response = session.post(queryURL, auth=session.auth) try: json_feed = response.json()['feed'] total_results = int(json_feed['opensearch:totalResults']) except (ValueError, KeyError): print(response) print 'API response not valid. JSON decoding failed. Exiting with code 3.' exit(3) entries = json_feed.get('entry', []) download_all(entries, zipDir, unzipDir) #add the images to the list for dir in os.listdir(unzipDir): if dir[len(dir) - 5:] == '.SAFE': dir = dir + '/measurement/' dir = unzipDir + dir for file in os.listdir(dir): if file.endswith('tiff'): warp(file, dir) thisFile = dir + file files.append(thisFile) geoimgs = [] for file in files: img = gippy.GeoImage(file) img.set_nodata(0) geoimgs.append(img) print files algorithms.cookie_cutter(geoimgs, outDir + '/mosaic', xres=10.0, yres=10.0, proj='EPSG:3857')
def fetch(self, key, aoi, pansharpen=False, acomp=False, dra=False, **kwargs): if self.order(): # create tempfile for AOI with tempfile.NamedTemporaryFile(suffix='.geojson', mode='w', delete=False) as f: aoiname = f.name aoistr = json.dumps(aoi) f.write(aoistr) geovec = gippy.GeoVector(aoiname) ext = geovec.extent() bbox = [ext.x0(), ext.y0(), ext.x1(), ext.y1()] # defaults spec = '' pansharpen = False acomp = False dra = False nodata = 0 if self['eo:platform'] in ['GEOEYE01', 'QUICKBIRD02' ] else -1e10 opts = COG # set options if key == 'rgb': spec = 'rgb' nodata = 0 #opts = JPEG_COG elif key == 'rgb-pan': pansharpen = True spec = 'rgb' nodata = 0 elif key == 'visual': pansharpen = True dra = True nodata = 0 #opts = JPEG_COG elif key == 'analytic': acomp = True fout = os.path.join(self.get_path(), self.get_filename(suffix='_%s' % key)) + '.tif' with TemporaryDirectory() as temp_dir: try: if not os.path.exists(fout): logger.info('Fetching %s: %s' % (key, fout)) # TODO - allow for other projections img = CatalogImage( self['id'], pansharpen=pansharpen, acomp=acomp, dra=dra, bbox=bbox) #, proj=utm_epsg(scenes.center())) tmp_fout1 = os.path.join( temp_dir, '%s_%s_1.tif' % (self['id'], key)) tmp_fout2 = os.path.join( temp_dir, '%s_%s_2.tif' % (self['id'], key)) tif = img.geotiff(path=tmp_fout1, proj='EPSG:4326', spec=spec) # clip and save geoimg = gippy.GeoImage(tif, True) # workaround for gbdxtools scaling if key in ['rgb', 'visual']: geoimg = geoimg.autoscale(1, 255).save(tmp_fout2) geoimg.set_nodata(0) # this clips the image to the AOI res = geoimg.resolution() imgout = alg.cookie_cutter([geoimg], fout, geovec[0], xres=res.x(), yres=res.y(), proj=geoimg.srs(), options=opts) imgout.add_overviews([2, 4, 8, 16], resampler='average') imgout = None except Exception as e: logger.warning('Error fetching: %s' % str(e)) #logger.warning('Traceback: %s', traceback.format_exc()) os.remove(aoiname) return fout