class DGImage:
def __init__(self, image_id):
self.image_id = image_id
self.image = None
self.image_path = os.path.join(GBDX_DIR,
'{}.tif'.format(random_word(10)))
self.bc = None
def __getattr__(self, item):
return getattr(self.image, item)
def fetch(self, *args, **kwargs):
self.image = CatalogImage(self.image_id, *args, **kwargs)
def load(self, *args, **kwargs):
if self.image is not None:
self.image.geotiff(self.image_path, *args, **kwargs)
else:
raise AttributeError('Fetch image before loading')
def transform(self, dst_path, channels):
if os.path.exists(self.image_path):
# split raster to separate bands
split_raster(self.image, dst_path, channels)
# extract meta information and save
meta_fc = get_meta(self.image)
meta_fc.save(os.path.join(dst_path, 'meta.geojson'))
else:
raise FileExistsError('Load GBDX Image before transform')
def test_cat_image_aoi(self):
_id = '104001002838EC00'
img = CatalogImage(_id)
aoi = img.aoi(bbox=[
-85.81455230712892, 10.416235163695223, -85.77163696289064,
10.457089934231618
])
assert aoi.shape == (8, 3037, 3189)
def test_rda_materialize(self):
catid = '881e61e5-5c56-44bf-a0d0-c4f5260a8aff-inv'
img = CatalogImage(catid)
aoi = img.randwindow((1000, 1000))
assert aoi.shape == (4, 1000, 1000)
job_id = img.materialize(bounds=aoi.bounds)
status = img.materialize_status(job_id)
assert status['jobStatus'] == 'processing'
def test_rda_materialize(self):
# TODO: Fix this test
catid = '8c3c4fc6-abcb-4f5f-bce6-d496c1a91676'
img = CatalogImage(catid)
aoi = img.randwindow((1000, 1000))
assert aoi.shape == (4, 1000, 1000)
job_id = img.materialize(bounds=aoi.bounds)
status = img.materialize_status(job_id)
assert status['jobStatus'] == 'processing'
def analyze_area(area_name, bbox, catids, buildings_geojson):
# format the area name to title case with spaces instead of underscores
area_name_title = area_name.replace("_", " ").title()
# Get building footprints from OSM for this area
if buildings_geojson is not None:
geoms, buildings = from_geojson(buildings_geojson)
# instantiate an empty list to hold the results from each building
results_list = []
for catid in tqdm.tqdm(catids):
# Get the image (with acomp, if possible)
try:
cimage = CatalogImage(catid,
band_type='MS',
bbox=bbox,
pansharpen=True,
acomp=True)
except AcompUnavailable as e:
cimage = CatalogImage(catid,
band_type='MS',
bbox=bbox,
pansharpen=True,
acomp=False)
# Turn off "Fetching Image..." statements
cimage._read = partial(cimage._read, quiet=True)
# find the blue buildings
blue_polys = find_blue_polys(cimage,
lower_blue_hue=.63,
upper_blue_hue=.67,
segment_blobs=True,
lower_blue_saturation=.3)
if buildings_geojson is not None:
blue_bldgs = filter_blue_polys(blue_polys, buildings)
else:
blue_bldgs = blue_polys
# Store the image acquisition date
image_acq_date = pd.to_datetime(
pd.to_datetime(
cimage.rda.metadata['image']["acquisitionDate"]).date())
# build a dictionary to hold the current results
results = {
'date': image_acq_date,
'n_blue_bldgs': len(blue_bldgs),
'catid': catid
}
# append current results to the full results list
results_list.append(results)
# compile the results into a data frame
new_results_df = pd.DataFrame(results_list)
# sort by data (ascending)
new_results_df.sort_values(by='date', inplace=True)
# add the area name
new_results_df['area'] = area_name_title
return new_results_df
def test_image_fetch(self):
wv2 = CatalogImage('1030010076B8F500')
aoi = wv2.aoi(bbox=[-104.98815365500539, 39.71459029774345, -104.98317715482573, 39.71956679792311])
aoi = apply_mock(aoi)
arr = aoi.read()
self.assertEquals(arr.shape, aoi.shape)
rgb = aoi.rgb()
self.assertEquals(rgb.shape, (256,256,3))
ndvi = aoi.ndvi()
self.assertEquals(ndvi.shape, (256,256))
def load_cat_image(catalog_id, bbox=None, pan=False):
"""Returns the CatalogImage.
:param catalog_id: catalog_id that is specified in geojson
:type catalog_id: String
:param bbox: can set if needed, but not necessary
:param pan: usually True for pansharpened image, but can set to False if loading image takes a while
:return: CatalogImage"""
if bbox == None:
return CatalogImage(catalog_id, band_type="MS", pansharpen=pan)
return CatalogImage(catalog_id, band_type="MS", pansharpen=pan, bbox=bbox)
def test_image_window_at(self):
wv2 = CatalogImage('1030010076B8F500')
c1 = shape(wv2).centroid
window = wv2.window_at(c1, (256, 256))
c2 = shape(window).centroid
bands, x, y = window.shape
# check the window is the correct shape
self.assertEquals(x, 256)
self.assertEquals(y, 256)
# make sure the center of the window is within 1 pixel
# of where it should be
self.assertTrue(c1.distance(c2) < wv2.metadata['georef']['scaleX'])
def test_image_pxbounds_overlapping(self):
wv2 = CatalogImage('1030010076B8F500')
_bands, ysize, xsize = wv2.shape
image_shape = shape(wv2)
image_bounds = image_shape.bounds
width = image_bounds[2] - image_bounds[0]
clip_area = translate(image_shape, xoff=-0.5 * width)
xmin, ymin, xmax, ymax = wv2.pxbounds(clip_area, clip=True)
self.assertEquals(xmin, 0)
self.assertEquals(ymin, 0)
self.assertEquals(xmax, xsize / 2)
self.assertEquals(ymax, ysize)
def test_image_window_at(self):
wv2 = CatalogImage('1030010076B8F500')
c1 = shape(wv2).centroid
window = wv2.window_at(c1, 256, 256)
c2 = shape(window).centroid
bands, x, y = window.shape
# check the window is the correct shape
self.assertEquals(x, 256)
self.assertEquals(y, 256)
# make sure the center of the window is within 1 pixel
# of where it should be
self.assertTrue(c1.distance(c2) < wv2.metadata['georef']['scaleX'])
def test_image_pxbounds_overlapping(self):
wv2 = CatalogImage('1030010076B8F500')
_bands, ysize, xsize = wv2.shape
image_shape = shape(wv2)
image_bounds = image_shape.bounds
width = image_bounds[2] - image_bounds[0]
clip_area = translate(image_shape, xoff=-0.5 * width)
xmin, ymin, xmax, ymax = wv2.pxbounds(clip_area, clip=True)
self.assertEquals(xmin, 0)
self.assertEquals(ymin, 0)
self.assertEquals(xmax, xsize/2)
self.assertEquals(ymax, ysize)
def test_image_fetch(self):
wv2 = CatalogImage('1030010076B8F500')
aoi = wv2.aoi(bbox=[
-104.98815365500539, 39.71459029774345, -104.98317715482573,
39.71956679792311
])
aoi = apply_mock(aoi)
arr = aoi.read()
self.assertEquals(arr.shape, aoi.shape)
rgb = aoi.rgb()
self.assertEquals(rgb.shape, (256, 256, 3))
ndvi = aoi.ndvi()
self.assertEquals(ndvi.shape, (256, 256))
def _load_image(self,
image: gbdxtools.CatalogImage,
path: str,
aoi: aoi.AreaOfInterest = None,
**kwargs):
try:
if aoi is not None:
image = image.aoi(bbox=aoi.bbox)
image.geotiff(path=path, **kwargs)
except Exception as e:
if errors.ImageExpiredError.catch(e):
raise errors.ImageExpiredError(str(e))
else:
raise e
def test_cat_image_with_proj(self):
_id = '104001002838EC00'
img = CatalogImage(_id,
bbox=[
-85.81455230712892, 10.416235163695223,
-85.77163696289064, 10.457089934231618
],
proj='EPSG:3857')
assert img._node_id == 'toa_reflectance'
assert img.shape == (8, 3088, 3190)
assert img._proj == 'EPSG:3857'
def test_class_instance_legacy_access(self):
# Test access raises warning
from gbdxtools import CatalogImage
wv2 = CatalogImage('1030010076B8F500')
with self.assertWarns(DeprecationWarning):
obj = wv2.ipe
# Test value is expected
self.assertEqual(wv2.ipe.__module__, "gbdxtools.rda.interface")
from gbdxtools.rda.interface import Op
self.assertTrue(isinstance(wv2.ipe, Op))
def get_idaho_tms_url(source_catid_or_image, gbdx):
if type(source_catid_or_image) == str:
image = CatalogImage(source_catid_or_image)
elif '_ipe_op' in list(source_catid_or_image.__dict__.keys()):
image = source_catid_or_image
else:
err = "Invalid type for source_catid_or_image. Must be either a Catalog ID (string) or CatalogImage object"
raise TypeError(err)
url_params = get_idaho_tms_ids(image)
url_params['token'] = str(gbdx.gbdx_connection.access_token)
url_params['z'] = '{z}'
url_params['x'] = '{x}'
url_params['y'] = '{y}'
url_template = 'https://idaho.geobigdata.io/v1/tile/idaho-images/{ms_id}/{z}/{x}/{y}?bands=4,2,1&token={token}&panId={pan_id}'
url = url_template.format(**url_params)
return url
def from_geojson(self, filename):
with open(filename, 'r') as f:
in_gj = json.loads(f.read())
in_features = in_gj['features']
catids = list(set([feat['properties']['catalog_id'] for feat in in_features]))
if len(catids) > 1:
raise ValueError("Input geojson references multiple catalog_ids")
else:
catid = catids[0]
bboxes = list(set([tuple(feat['properties']['bbox']) for feat in in_features]))
if len(bboxes) > 1:
raise ValueError("Input geojson references multiple bboxes")
else:
bbox = bboxes[0]
options_list = [feat['properties']['img_options'] for feat in in_features]
options_list_unique = list(set([tuple(sorted(zip(o.keys(), o.values()))) for o in options_list]))
if len(options_list_unique) > 1:
raise ValueError("Input geojson references multiple img_options")
else:
options = options_list[0]
self.image = CatalogImage(catid, bbox=bbox, dra=True, **options)
self.__validate_image__()
self.features = []
self.data = []
for feat in in_features:
geom = shape(feat['geometry'])
self.features.append(geom)
label_polygon = LabelPolygon(geom, feat['properties']['id'], self.image)
label_polygon.label_value = feat['properties']['label_value']
self.data.append(label_polygon)
self.n_features = len(self.data)
self.__validate_features__()
self.index = 0
def test_rda_materialize_payload(self):
img = CatalogImage(WV02_CATID)
pl = img.rda._create_materialize_payload('123',
'node',
None,
None,
'TILE_STREAM',
sample='yes')
assert pl['outputFormat'] == 'TILE_STREAM'
assert 'callbackUrl' not in pl
assert 'cropGeometryWKT' not in pl
assert pl['imageReference']['nodeId'] == 'node'
assert pl['imageReference']['templateId'] == '123'
assert pl['imageReference']['parameters'] == {'sample': 'yes'}
pl = img.rda._create_materialize_payload('123', 'node', [1, 2, 3, 4],
'sns://yes', 'TIF')
assert pl['outputFormat'] == 'TIF'
assert pl['callbackUrl'] == 'sns://yes'
assert pl['cropGeometryWKT'] == 'POLYGON ((3 2, 3 4, 1 4, 1 2, 3 2))'
assert pl['imageReference']['nodeId'] == 'node'
assert pl['imageReference']['templateId'] == '123'
assert pl['imageReference']['parameters'] == {}
plt.plot([bb[2], bb[2]], [bb[3], bb[1]])
plt.plot([bb[0], bb[2]], [bb[1], bb[1]])
plt.show()'''
from gbdxtools.task import env
from gbdxtools import CatalogImage
catalog_id = env.inputs.get('catalog_id', '1040010033CCDF00')
bbox = env.inputs.get(
'bbox',
'-72.65748023986818, 41.51410470031741, -72.64263153076173, 41.52779263120528'
)
image = CatalogImage(catalog_id,
band_type="MS",
bbox=map(float, bbox.split(",")))
print(image.shape)
image = image.rgb()
plt.imshow(image)
plt.show()
'''for i in (range(len(imstocat))):
imprel = CatalogImage(imstocat[i]) # READ IMAGE
aoi = imprel.aoi(bbox=list(bboxes[i])).rgb()
plt.imshow(aoi)
plt.show()
l = skimage.img_as_ubyte(imprel.aoi(bbox=list(bboxes[i])).read()) # CROP TO BBOX AND CONVERT TO RGB
img_name = os.path.join(r'/media/ei-edl01/user/bh163/figs/2018.03.15.gbdx/gbdx_geotiff',
imnames[i] + '.tif')
imprel.geotiff(path=img_name, proj='EPSG:4326', bands=[4,2,1])
#Image.fromarray(l).save(os.path.join(img_dir, imnames[i] + '.jpg'))'''
def test_sentinel2_image(self):
sentinel = CatalogImage(SENTINEL_ID)
self.assertTrue(isinstance(sentinel, Sentinel2))
def test_wv4_image(self):
wv4 = CatalogImage(WV04_CATID)
self.assertTrue(isinstance(wv4, WV04))
def test_is_available(self):
available = CatalogImage.is_available(WV02_CATID)
self.assertTrue(available)
def test_image_pxbounds_disjoint(self):
wv2 = CatalogImage('1030010076B8F500')
bounds = shape(wv2)
clip = translate(bounds, xoff=5, yoff=5)
with self.assertRaises(ValueError):
clipped = wv2.pxbounds(clip)
def test_cat_image_aoi(self):
_id = '104001002838EC00'
img = CatalogImage(_id)
assert img.cat_id == _id
aoi = img.aoi(bbox=[-85.81455230712892,10.416235163695223,-85.77163696289064,10.457089934231618])
assert aoi.shape == (8, 3037, 3190)
def get_SumWin(park_shape,UTM_EPSG_code):
# Getting the bounding boxes right
x_wgs,y_wgs = park_shape.exterior.xy
bbox_park_area = str([min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs)])
bbox_park_area_str = listToStringWithoutBrackets(bbox_park_area)
bbox_park_area = min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs)
bbox_wkt = box(*bbox_park_area).wkt
aoi = bbox_wkt
# Querying for WV03 or WV02 within the aoi
query = "(item_type:WV03_VNIR OR WV02)"
query += " AND NOT item_type:IDAHOImage AND item_type:DigitalGlobeProduct"
results = gbdx.vectors.query(aoi, query)
from collections import namedtuple
Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax')
ra = Rectangle(min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs))
# intersection here is (3, 3, 4, 3.5), or an area of 1*.5=.5
def area(a, b): # returns None if rectangles don't intersect
dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin)
dy = min(a.ymax, b.ymax) - max(a.ymin, b.ymin)
if (dx>=0) and (dy>=0):
return dx*dy
# set cloud check
images_df = pd.DataFrame(columns=['id','month','year','type','resolution','cloud cover','overlap','check'])
images_df2 = pd.DataFrame(columns=['id','month','year','type','resolution','cloud cover','overlap','check'])
for r in results:
props = r['properties']
geom = r['geometry']
coordinates_image = geom['coordinates'][0][0]
df_overlap = pd.DataFrame.from_records(coordinates_image,columns = ['x','y'])
rb = Rectangle(df_overlap.x.min(), df_overlap.y.min(), df_overlap.x.max(), df_overlap.y.max())
total_area = ((ra.xmax-ra.xmin)*(ra.ymax-ra.ymin))
total_overlap = area(ra, rb)
if total_overlap is None:
fraction_overlap = 0
else:
fraction_overlap = area(ra, rb)/total_area
images_df = images_df.append({'id': props['attributes']['catalogID'],'month':props['item_date'][5:7],'year':props['item_date'][0:4],'type':props['item_type'][1],'resolution':props['attributes']['resolution_dbl'],'cloud cover':props['attributes']['cloudCover_int'],'overlap': fraction_overlap,'check':props['attributes']['cloudCover_int'] < 100 and int(props['item_date'][5:7]) >= 5 and int(props['item_date'][5:7]) <= 8 and fraction_overlap >.9},ignore_index=True)
images_df2 = images_df2.append({'id': props['attributes']['catalogID'],'month':props['item_date'][5:7],'year':props['item_date'][0:4],'type':props['item_type'][1],'resolution':props['attributes']['resolution_dbl'],'cloud cover':props['attributes']['cloudCover_int'],'overlap': fraction_overlap,'check':props['attributes']['cloudCover_int'] < 100 and int(props['item_date'][5:7]) <= 2 or int(props['item_date'][5:7]) >= 11 and fraction_overlap >.9},ignore_index=True)
selections = images_df.loc[images_df.check == True].reset_index()
selectionw = images_df2.loc[images_df2.check == True].reset_index()
selections = selections.groupby(['id'],sort=['year','month','cloud cover'],as_index=False).first().sort_values(['year','month'], ascending=False).reset_index()
del selections['level_0'], selections['index']
selectionw = selectionw.groupby(['id'],sort=['year','month','cloud cover'],as_index=False).first().sort_values(['year','month'], ascending=False).reset_index()
del selectionw['level_0'], selectionw['index']
# First check if images are not defective and then fetch them
bbox = env.inputs.get('bbox', bbox_park_area_str)
# Select the first image
imagestatus ='unknown'
iS=0
iW=0
if (selections.empty | selectionw.empty):
error = 0
return error, error, error, error
else:
while imagestatus =='unknown':
# set catalog id from selection
catalog_id_s = env.inputs.get('catalog_id', selections.id[iS])
catalog_id_w = env.inputs.get('catalog_id', selectionw.id[iW])
# collect images
images = CatalogImage(catalog_id_s, band_type="MS", bbox=map(float, bbox.split(",")),proj=UTM_EPSG_code
,pansharpen= False)
imagew = CatalogImage(catalog_id_w, band_type="MS", bbox=map(float, bbox.split(",")),proj=UTM_EPSG_code
,pansharpen= False)
# calculate mean of coastal band to see if image is defective
mean_coastals = images[1,:,:].read().mean()
mean_coastalw = imagew[1,:,:].read().mean()
# use image if not defective, otherwise
if mean_coastals > 200:
pass
else:
iS=iS+1
if mean_coastalw > 200:
pass
else:
iW=iW+1
if mean_coastalw > 200 and mean_coastals > 200:
imagestatus = 'fine'
return selections,selectionw,images,imagew
def test_image_pxbounds_disjoint(self):
wv2 = CatalogImage('1030010076B8F500')
bounds = shape(wv2)
clip = translate(bounds, xoff=5, yoff=5)
with self.assertRaises(ValueError):
clipped = wv2.pxbounds(clip)
def fetch(self,
key,
aoi,
pansharpen=False,
acomp=False,
dra=False,
**kwargs):
if self.order():
# create tempfile for AOI
with tempfile.NamedTemporaryFile(suffix='.geojson',
mode='w',
delete=False) as f:
aoiname = f.name
aoistr = json.dumps(aoi)
f.write(aoistr)
geovec = gippy.GeoVector(aoiname)
ext = geovec.extent()
bbox = [ext.x0(), ext.y0(), ext.x1(), ext.y1()]
# defaults
spec = ''
pansharpen = False
acomp = False
dra = False
nodata = 0 if self['eo:platform'] in ['GEOEYE01', 'QUICKBIRD02'
] else -1e10
opts = COG
# set options
if key == 'rgb':
spec = 'rgb'
nodata = 0
#opts = JPEG_COG
elif key == 'rgb-pan':
pansharpen = True
spec = 'rgb'
nodata = 0
elif key == 'visual':
pansharpen = True
dra = True
nodata = 0
#opts = JPEG_COG
elif key == 'analytic':
acomp = True
fout = os.path.join(self.get_path(),
self.get_filename(suffix='_%s' % key)) + '.tif'
with TemporaryDirectory() as temp_dir:
try:
if not os.path.exists(fout):
logger.info('Fetching %s: %s' % (key, fout))
# TODO - allow for other projections
img = CatalogImage(
self['id'],
pansharpen=pansharpen,
acomp=acomp,
dra=dra,
bbox=bbox) #, proj=utm_epsg(scenes.center()))
tmp_fout1 = os.path.join(
temp_dir, '%s_%s_1.tif' % (self['id'], key))
tmp_fout2 = os.path.join(
temp_dir, '%s_%s_2.tif' % (self['id'], key))
tif = img.geotiff(path=tmp_fout1,
proj='EPSG:4326',
spec=spec)
# clip and save
geoimg = gippy.GeoImage(tif, True)
# workaround for gbdxtools scaling
if key in ['rgb', 'visual']:
geoimg = geoimg.autoscale(1, 255).save(tmp_fout2)
geoimg.set_nodata(0)
# this clips the image to the AOI
res = geoimg.resolution()
imgout = alg.cookie_cutter([geoimg],
fout,
geovec[0],
xres=res.x(),
yres=res.y(),
proj=geoimg.srs(),
options=opts)
imgout.add_overviews([2, 4, 8, 16],
resampler='average')
imgout = None
except Exception as e:
logger.warning('Error fetching: %s' % str(e))
#logger.warning('Traceback: %s', traceback.format_exc())
os.remove(aoiname)
return fout
def get_classifier(city,buffer_size, min_size, sample_size):
################ this takes some time
label_all = np.array([])
data_all = np.array([])
# labels of types to loop over searching for polygons
labels = ['forest', 'grass', 'water' , 'building']
# set minimal sizes for polygons of different types
dict_size = {'forest': 1 , 'grass': .6, 'water': 1, 'building': .8}
# create empty dataframes that will be filled
geom_list_selection_all = []
selection_all = pd.DataFrame()
# loop over OSM types and get polygons
for label in labels:
selection, geojson_select, geom_list_selection,UTM_EPSG_code,project_utm,project_wgs = getOSM.get_OSM_polygons(city = city, type_query = label,min_size = dict_size[label])
geom_list_selection_all.extend(geom_list_selection[1:sample_size])
selection_all = selection_all.append(selection[1:sample_size])
################ this takes some time
selection_all = selection_all.reset_index().drop('index',axis = 1)
# set classes for each polygon type
dict_type = {'Forest': 1,'Wood': 1,'Nature Reserve': 3,'Wetland': 1, 'Grass': 2, 'Farmland': 2, 'Water': 3, 'Building': 4, 'Theatre': 0}
# A list of "random" colors (for a nicer output)
COLORS = ["#000000", "#FFFF00", "#1CE6FF", "#FF34FF", "#FF4A46", "#008941"]
# load multipolygon type from pickle for check
multipolygon_type = pickle.load( open( "/home/gremlin/GGCW_tools_git/Pickle/multipolygon_type.p", "rb" ) )
################ this takes some time
for objects in selection_all.index:
print '-----------------------------------------\n'
### setting a buffer can remove the polygon or make it into a multipolygon, both are unusable so check if this is the case
park_utm = transform(project_utm, geom_list_selection_all[objects]) # apply projection
# perform check # get x y coordinates of polygon and set a buffer if polygon is large enough
if (type(park_utm.buffer(buffer_size)) == multipolygon_type) | (park_utm.buffer(buffer_size).area == 0):
message = "is multipolygon"
print message + ' object: ' + str(objects) +'\n'
else:
x,y = park_utm.buffer(buffer_size).exterior.xy
park_buffer_wgs = transform(project_wgs,park_utm.buffer(buffer_size)) # apply projection
# get wgs projected x,y coordinates and create bounding box for image aquisition
x_wgs,y_wgs = park_buffer_wgs.exterior.xy
bbox_park_area_float = min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs)
bbox_park_area = str([min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs)])
bbox_park_area_str = nf.listToStringWithoutBrackets(bbox_park_area)
# convert bounding box to well known format usable by GBDX tools
bbox_wkt = box(*bbox_park_area_float).wkt
selection_images = nf.image_query_check(bbox_wkt,park_utm,buffer_size,multipolygon_type,project_wgs,x_wgs,y_wgs)
if not selection_images.empty:
# set park bounding box
bbox = bbox_park_area_str
# set catalog id from selection
catalog_id = selection_images.id[0]
# collect image
image = CatalogImage(catalog_id, band_type="MS", bbox=map(float, bbox.split(",")),
proj=UTM_EPSG_code,pansharpen=False)
#create array from GBDX image
image_array = image[:,:,:].read()
# get second band to see if image is defective (some images show only black)
# use image if not defective, otherwise
if (image_array.size == 0):
print('no image')
elif (image_array[1,:,:].min() != 0) :
print 'image for ' + selection_all.OSM_id[objects] + ' is good object: ' + str(objects )
print image.ipe_metadata["image"]["acquisitionDate"]
print image.cat_id
print selection_all.item_type[objects]
# resize polygon and plot polygon over image
# subtract minimal values from utm polygon x and y to set 0,0 point as start
x1 = np.subtract(x, min(x))
y1 = np.subtract(y, min(y))
# devide the x and y coordinate of the polygon by the size of the image to match both sizes
x2 = np.divide(x1,max(x1)/image.shape[2])
y2 = np.divide(y1,max(y1)/image.shape[1])
n_bands, rows, cols = image.shape
# calculate total cells for each class by masking and setting pixel values to 1
# create sequence of edited x and y coordinates, widht and heigth for use in ImageDraw function
polygon = [(x2[i], y2[i]) for i in range(len(x2))]
width = image.shape[2]
height = image.shape[1]
# convert polygon coordinates to raster/array values using ImageDraw
img = Image.new('L', (width, height), 0)
ImageDraw.Draw(img).polygon(polygon, fill=dict_type[selection_all.item_type[objects]])
# convert image to array and set as mask
mask = np.array(img)
# flip the array for matching with the mask array
image_array_flipped = np.fliplr(image_array[:,:,:])
reshaped_data = image_array_flipped.reshape(8,(rows*cols))
reshaped_label = mask.reshape(1,(rows*cols))
# check if this is the first iteration, if so add the first data set otherwise:
# append the new image data to the other data
if data_all.size == 0:
data_all = reshaped_data
else:
data_all = np.concatenate((data_all,reshaped_data), axis = 1)
label_all = np.append(label_all,reshaped_label)
# Two subplots, the axes array is 1-d
f, axarr = plt.subplots(1,2)
axarr[0].imshow(mask)
axarr[1].imshow(image_array_flipped[1])
plt.show()
# plt.imshow(mask)
# plt.show()
print label_all.shape
print data_all.shape
else:
print 'image defective'
# move to next without doing analysis
else:
print 'no image'
# move to next without doing analysis
################ this takes some time
### Remove pixels without class
label_all_no0 = label_all[label_all != 0]
data_all_no0 = data_all[:,label_all != 0]
data = pd.DataFrame(data_all.T)
data['class'] = label_all
# Calculate NDVI
# 8-band (0:Coastal, 1:Blue, 2:Green, 3:Yellow, 4:Red, 5:Red-edge, 6:NIR1, 7:NIR2) Multispectral
# ndvi = (nir - red)/(nir + red)
# EVI = 2.5 * ( nir - red ) / ( nir + 6.5 * red - 7.5 * blue+ 1.0 )
data['ndvi'] = (data[6] - data[4])/(data[6] + data[4])
data['EVI'] = 2.5 * (data[6] - data[4]) / (data[6] + 6.5 * data[4] - 7.5 * data[1] + 1 )
data['water_index'] = (data[7] - data[0]) / (data[7] + data[0])
X = data.iloc[:,0:8]
y = data['class']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)
# initialize search space (as a library!)
param_grid = {
'max_depth':[20,10],'n_estimators':[40]
}
gs = make_pipeline( StandardScaler(),
GridSearchCV(RandomForestClassifier(min_samples_leaf=2),
param_grid = param_grid,
cv = 2,
refit = True,
n_jobs = 1,
verbose = 0))
# Instantiate random forest. You can specify default parameters here.
# These parameters are not being optimized.
# initialize grid search
#gs = GridSearchCV(rf, param_grid, verbose=2)#,scoring='roc_auc')
gs.fit(X_train,y_train)
return gs
Checking whether a workflow is complete (whether canceled, failed, or succeeded):
>>> workflow.complete
True
'''
while not workflow.succeeded:
pass
# save the data to S3
folder = 'test_folder'
workflow.savedata(aoptask.outputs.data, location=folder)
workflow.savedata(aoptask.outputs.data)
# convert the result into an image
img = CatalogImage(aoptask.outputs.data.value)
id_img = IdahoImage(aoptask.outputs.data.value)
# >>> img = CatalogImage(aoptask.outputs.data)
# Traceback (most recent call last):
# File "<stdin>", line 1, in <module>
# File "C:\Anaconda2\envs\joemcg_rootclone\lib\site-packages\gbdxtools\images\catalog_image.py", line 57, in __init__
# self._ipe_graphs = self._init_graphs()
# File "C:\Anaconda2\envs\joemcg_rootclone\lib\site-packages\gbdxtools\images\catalog_image.py", line 119, in _init_graphs
# for part in self.metadata['parts']:
# File "C:\Anaconda2\envs\joemcg_rootclone\lib\site-packages\gbdxtools\images\catalog_image.py", line 82, in metadata
# results = self._query_vectors(query)
# File "C:\Anaconda2\envs\joemcg_rootclone\lib\site-packages\gbdxtools\images\catalog_image.py", line 68, in _query_vectors
# raise Exception('Unable to query for image properties, the service may be currently down.', err)
# Exception: ('Unable to query for image properties, the service may be currently down.', HTTPError(u'504 Server Error: GATEWAY_TIMEOUT for url: https://vector.geobigdata.io/insight-vector/api/index/query/vector-gbdx-alpha-catalog-v2-*/paging?count=100&upper=90.0&lower=-90.0&right=180.0&q=item_type%3AIDAHOImage+AND+attributes.catalogID%3APort+data%3A%0A%09type%3A+directory%0A%09description%3A+The+output+data+directory%0A%09multiplex%3A+False&ttl=5m&left=-180.0',))
def test_image_window_cover_false(self):
""" Window cover should not padd the image and return only tiles of 100, 100 (9 windows) """
wv2 = CatalogImage('1030010076B8F500')
aoi = wv2.randwindow((325,300))
coverage = [dsk for dsk in aoi.window_cover((100,100), pad=False)]
self.assertEquals(len(coverage), 9)
def test_image_window_cover(self):
""" Window cover should padd the image by 25 pixels and return 9 windows """
wv2 = CatalogImage('1030010076B8F500')
aoi = wv2.randwindow((275, 300))
coverage = [dsk for dsk in aoi.window_cover((100, 100), pad=True)]
self.assertEquals(len(coverage), 9)
def test_image_acomp_avail(self):
can_acomp = CatalogImage.acomp_available('1050410000B1BD00')
self.assertFalse(can_acomp)
can_acomp = CatalogImage.acomp_available('104001002ABEA500')
self.assertTrue(can_acomp)
def test_image_ordered(self):
ordered = CatalogImage.is_ordered('1050410000B1BD00')
self.assertFalse(ordered)
ordered = CatalogImage.is_ordered('104001002ABEA500')
self.assertTrue(ordered)
def test_catalog_image_unsupported_type(self):
with self.assertRaises(Exception) as context:
img = CatalogImage('MONA_LISA')
def test_base_layer_match_dep(self):
# Test access raises warning
from gbdxtools import CatalogImage
wv2 = CatalogImage('1030010076B8F500')
with self.assertWarns(DeprecationWarning):
obj = wv2.base_layer_match()
def test_image_acomp_avail(self):
can_acomp = CatalogImage.acomp_available(WV02_CATID)
self.assertTrue(can_acomp)
cant_acomp = CatalogImage.acomp_available(WV01_CATID)
self.assertFalse(cant_acomp)
def test_image_window_cover(self):
""" Window cover should padd the image by 25 pixels and return 9 windows """
wv2 = CatalogImage('1030010076B8F500')
aoi = wv2.randwindow((275,300))
coverage = [dsk for dsk in aoi.window_cover((100,100), pad=True)]
self.assertEquals(len(coverage), 9)
def test_image_window_cover_false(self):
""" Window cover should not padd the image and return only tiles of 100, 100 (9 windows) """
wv2 = CatalogImage('1030010076B8F500')
aoi = wv2.randwindow((325, 300))
coverage = [dsk for dsk in aoi.window_cover((100, 100), pad=False)]
self.assertEquals(len(coverage), 9)