class DGImage: def __init__(self, image_id): self.image_id = image_id self.image = None self.image_path = os.path.join(GBDX_DIR, '{}.tif'.format(random_word(10))) self.bc = None def __getattr__(self, item): return getattr(self.image, item) def fetch(self, *args, **kwargs): self.image = CatalogImage(self.image_id, *args, **kwargs) def load(self, *args, **kwargs): if self.image is not None: self.image.geotiff(self.image_path, *args, **kwargs) else: raise AttributeError('Fetch image before loading') def transform(self, dst_path, channels): if os.path.exists(self.image_path): # split raster to separate bands split_raster(self.image, dst_path, channels) # extract meta information and save meta_fc = get_meta(self.image) meta_fc.save(os.path.join(dst_path, 'meta.geojson')) else: raise FileExistsError('Load GBDX Image before transform')
def test_cat_image_aoi(self): _id = '104001002838EC00' img = CatalogImage(_id) aoi = img.aoi(bbox=[ -85.81455230712892, 10.416235163695223, -85.77163696289064, 10.457089934231618 ]) assert aoi.shape == (8, 3037, 3189)
def test_rda_materialize(self): catid = '881e61e5-5c56-44bf-a0d0-c4f5260a8aff-inv' img = CatalogImage(catid) aoi = img.randwindow((1000, 1000)) assert aoi.shape == (4, 1000, 1000) job_id = img.materialize(bounds=aoi.bounds) status = img.materialize_status(job_id) assert status['jobStatus'] == 'processing'
def test_rda_materialize(self): # TODO: Fix this test catid = '8c3c4fc6-abcb-4f5f-bce6-d496c1a91676' img = CatalogImage(catid) aoi = img.randwindow((1000, 1000)) assert aoi.shape == (4, 1000, 1000) job_id = img.materialize(bounds=aoi.bounds) status = img.materialize_status(job_id) assert status['jobStatus'] == 'processing'
def analyze_area(area_name, bbox, catids, buildings_geojson): # format the area name to title case with spaces instead of underscores area_name_title = area_name.replace("_", " ").title() # Get building footprints from OSM for this area if buildings_geojson is not None: geoms, buildings = from_geojson(buildings_geojson) # instantiate an empty list to hold the results from each building results_list = [] for catid in tqdm.tqdm(catids): # Get the image (with acomp, if possible) try: cimage = CatalogImage(catid, band_type='MS', bbox=bbox, pansharpen=True, acomp=True) except AcompUnavailable as e: cimage = CatalogImage(catid, band_type='MS', bbox=bbox, pansharpen=True, acomp=False) # Turn off "Fetching Image..." statements cimage._read = partial(cimage._read, quiet=True) # find the blue buildings blue_polys = find_blue_polys(cimage, lower_blue_hue=.63, upper_blue_hue=.67, segment_blobs=True, lower_blue_saturation=.3) if buildings_geojson is not None: blue_bldgs = filter_blue_polys(blue_polys, buildings) else: blue_bldgs = blue_polys # Store the image acquisition date image_acq_date = pd.to_datetime( pd.to_datetime( cimage.rda.metadata['image']["acquisitionDate"]).date()) # build a dictionary to hold the current results results = { 'date': image_acq_date, 'n_blue_bldgs': len(blue_bldgs), 'catid': catid } # append current results to the full results list results_list.append(results) # compile the results into a data frame new_results_df = pd.DataFrame(results_list) # sort by data (ascending) new_results_df.sort_values(by='date', inplace=True) # add the area name new_results_df['area'] = area_name_title return new_results_df
def test_image_fetch(self): wv2 = CatalogImage('1030010076B8F500') aoi = wv2.aoi(bbox=[-104.98815365500539, 39.71459029774345, -104.98317715482573, 39.71956679792311]) aoi = apply_mock(aoi) arr = aoi.read() self.assertEquals(arr.shape, aoi.shape) rgb = aoi.rgb() self.assertEquals(rgb.shape, (256,256,3)) ndvi = aoi.ndvi() self.assertEquals(ndvi.shape, (256,256))
def load_cat_image(catalog_id, bbox=None, pan=False): """Returns the CatalogImage. :param catalog_id: catalog_id that is specified in geojson :type catalog_id: String :param bbox: can set if needed, but not necessary :param pan: usually True for pansharpened image, but can set to False if loading image takes a while :return: CatalogImage""" if bbox == None: return CatalogImage(catalog_id, band_type="MS", pansharpen=pan) return CatalogImage(catalog_id, band_type="MS", pansharpen=pan, bbox=bbox)
def test_image_window_at(self): wv2 = CatalogImage('1030010076B8F500') c1 = shape(wv2).centroid window = wv2.window_at(c1, (256, 256)) c2 = shape(window).centroid bands, x, y = window.shape # check the window is the correct shape self.assertEquals(x, 256) self.assertEquals(y, 256) # make sure the center of the window is within 1 pixel # of where it should be self.assertTrue(c1.distance(c2) < wv2.metadata['georef']['scaleX'])
def test_image_pxbounds_overlapping(self): wv2 = CatalogImage('1030010076B8F500') _bands, ysize, xsize = wv2.shape image_shape = shape(wv2) image_bounds = image_shape.bounds width = image_bounds[2] - image_bounds[0] clip_area = translate(image_shape, xoff=-0.5 * width) xmin, ymin, xmax, ymax = wv2.pxbounds(clip_area, clip=True) self.assertEquals(xmin, 0) self.assertEquals(ymin, 0) self.assertEquals(xmax, xsize / 2) self.assertEquals(ymax, ysize)
def test_image_window_at(self): wv2 = CatalogImage('1030010076B8F500') c1 = shape(wv2).centroid window = wv2.window_at(c1, 256, 256) c2 = shape(window).centroid bands, x, y = window.shape # check the window is the correct shape self.assertEquals(x, 256) self.assertEquals(y, 256) # make sure the center of the window is within 1 pixel # of where it should be self.assertTrue(c1.distance(c2) < wv2.metadata['georef']['scaleX'])
def test_image_pxbounds_overlapping(self): wv2 = CatalogImage('1030010076B8F500') _bands, ysize, xsize = wv2.shape image_shape = shape(wv2) image_bounds = image_shape.bounds width = image_bounds[2] - image_bounds[0] clip_area = translate(image_shape, xoff=-0.5 * width) xmin, ymin, xmax, ymax = wv2.pxbounds(clip_area, clip=True) self.assertEquals(xmin, 0) self.assertEquals(ymin, 0) self.assertEquals(xmax, xsize/2) self.assertEquals(ymax, ysize)
def test_image_fetch(self): wv2 = CatalogImage('1030010076B8F500') aoi = wv2.aoi(bbox=[ -104.98815365500539, 39.71459029774345, -104.98317715482573, 39.71956679792311 ]) aoi = apply_mock(aoi) arr = aoi.read() self.assertEquals(arr.shape, aoi.shape) rgb = aoi.rgb() self.assertEquals(rgb.shape, (256, 256, 3)) ndvi = aoi.ndvi() self.assertEquals(ndvi.shape, (256, 256))
def _load_image(self, image: gbdxtools.CatalogImage, path: str, aoi: aoi.AreaOfInterest = None, **kwargs): try: if aoi is not None: image = image.aoi(bbox=aoi.bbox) image.geotiff(path=path, **kwargs) except Exception as e: if errors.ImageExpiredError.catch(e): raise errors.ImageExpiredError(str(e)) else: raise e
def test_cat_image_with_proj(self): _id = '104001002838EC00' img = CatalogImage(_id, bbox=[ -85.81455230712892, 10.416235163695223, -85.77163696289064, 10.457089934231618 ], proj='EPSG:3857') assert img._node_id == 'toa_reflectance' assert img.shape == (8, 3088, 3190) assert img._proj == 'EPSG:3857'
def test_class_instance_legacy_access(self): # Test access raises warning from gbdxtools import CatalogImage wv2 = CatalogImage('1030010076B8F500') with self.assertWarns(DeprecationWarning): obj = wv2.ipe # Test value is expected self.assertEqual(wv2.ipe.__module__, "gbdxtools.rda.interface") from gbdxtools.rda.interface import Op self.assertTrue(isinstance(wv2.ipe, Op))
def get_idaho_tms_url(source_catid_or_image, gbdx): if type(source_catid_or_image) == str: image = CatalogImage(source_catid_or_image) elif '_ipe_op' in list(source_catid_or_image.__dict__.keys()): image = source_catid_or_image else: err = "Invalid type for source_catid_or_image. Must be either a Catalog ID (string) or CatalogImage object" raise TypeError(err) url_params = get_idaho_tms_ids(image) url_params['token'] = str(gbdx.gbdx_connection.access_token) url_params['z'] = '{z}' url_params['x'] = '{x}' url_params['y'] = '{y}' url_template = 'https://idaho.geobigdata.io/v1/tile/idaho-images/{ms_id}/{z}/{x}/{y}?bands=4,2,1&token={token}&panId={pan_id}' url = url_template.format(**url_params) return url
def from_geojson(self, filename): with open(filename, 'r') as f: in_gj = json.loads(f.read()) in_features = in_gj['features'] catids = list(set([feat['properties']['catalog_id'] for feat in in_features])) if len(catids) > 1: raise ValueError("Input geojson references multiple catalog_ids") else: catid = catids[0] bboxes = list(set([tuple(feat['properties']['bbox']) for feat in in_features])) if len(bboxes) > 1: raise ValueError("Input geojson references multiple bboxes") else: bbox = bboxes[0] options_list = [feat['properties']['img_options'] for feat in in_features] options_list_unique = list(set([tuple(sorted(zip(o.keys(), o.values()))) for o in options_list])) if len(options_list_unique) > 1: raise ValueError("Input geojson references multiple img_options") else: options = options_list[0] self.image = CatalogImage(catid, bbox=bbox, dra=True, **options) self.__validate_image__() self.features = [] self.data = [] for feat in in_features: geom = shape(feat['geometry']) self.features.append(geom) label_polygon = LabelPolygon(geom, feat['properties']['id'], self.image) label_polygon.label_value = feat['properties']['label_value'] self.data.append(label_polygon) self.n_features = len(self.data) self.__validate_features__() self.index = 0
def test_rda_materialize_payload(self): img = CatalogImage(WV02_CATID) pl = img.rda._create_materialize_payload('123', 'node', None, None, 'TILE_STREAM', sample='yes') assert pl['outputFormat'] == 'TILE_STREAM' assert 'callbackUrl' not in pl assert 'cropGeometryWKT' not in pl assert pl['imageReference']['nodeId'] == 'node' assert pl['imageReference']['templateId'] == '123' assert pl['imageReference']['parameters'] == {'sample': 'yes'} pl = img.rda._create_materialize_payload('123', 'node', [1, 2, 3, 4], 'sns://yes', 'TIF') assert pl['outputFormat'] == 'TIF' assert pl['callbackUrl'] == 'sns://yes' assert pl['cropGeometryWKT'] == 'POLYGON ((3 2, 3 4, 1 4, 1 2, 3 2))' assert pl['imageReference']['nodeId'] == 'node' assert pl['imageReference']['templateId'] == '123' assert pl['imageReference']['parameters'] == {}
plt.plot([bb[2], bb[2]], [bb[3], bb[1]]) plt.plot([bb[0], bb[2]], [bb[1], bb[1]]) plt.show()''' from gbdxtools.task import env from gbdxtools import CatalogImage catalog_id = env.inputs.get('catalog_id', '1040010033CCDF00') bbox = env.inputs.get( 'bbox', '-72.65748023986818, 41.51410470031741, -72.64263153076173, 41.52779263120528' ) image = CatalogImage(catalog_id, band_type="MS", bbox=map(float, bbox.split(","))) print(image.shape) image = image.rgb() plt.imshow(image) plt.show() '''for i in (range(len(imstocat))): imprel = CatalogImage(imstocat[i]) # READ IMAGE aoi = imprel.aoi(bbox=list(bboxes[i])).rgb() plt.imshow(aoi) plt.show() l = skimage.img_as_ubyte(imprel.aoi(bbox=list(bboxes[i])).read()) # CROP TO BBOX AND CONVERT TO RGB img_name = os.path.join(r'/media/ei-edl01/user/bh163/figs/2018.03.15.gbdx/gbdx_geotiff', imnames[i] + '.tif') imprel.geotiff(path=img_name, proj='EPSG:4326', bands=[4,2,1]) #Image.fromarray(l).save(os.path.join(img_dir, imnames[i] + '.jpg'))'''
def test_sentinel2_image(self): sentinel = CatalogImage(SENTINEL_ID) self.assertTrue(isinstance(sentinel, Sentinel2))
def test_wv4_image(self): wv4 = CatalogImage(WV04_CATID) self.assertTrue(isinstance(wv4, WV04))
def test_is_available(self): available = CatalogImage.is_available(WV02_CATID) self.assertTrue(available)
def test_image_pxbounds_disjoint(self): wv2 = CatalogImage('1030010076B8F500') bounds = shape(wv2) clip = translate(bounds, xoff=5, yoff=5) with self.assertRaises(ValueError): clipped = wv2.pxbounds(clip)
def test_cat_image_aoi(self): _id = '104001002838EC00' img = CatalogImage(_id) assert img.cat_id == _id aoi = img.aoi(bbox=[-85.81455230712892,10.416235163695223,-85.77163696289064,10.457089934231618]) assert aoi.shape == (8, 3037, 3190)
def get_SumWin(park_shape,UTM_EPSG_code): # Getting the bounding boxes right x_wgs,y_wgs = park_shape.exterior.xy bbox_park_area = str([min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs)]) bbox_park_area_str = listToStringWithoutBrackets(bbox_park_area) bbox_park_area = min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs) bbox_wkt = box(*bbox_park_area).wkt aoi = bbox_wkt # Querying for WV03 or WV02 within the aoi query = "(item_type:WV03_VNIR OR WV02)" query += " AND NOT item_type:IDAHOImage AND item_type:DigitalGlobeProduct" results = gbdx.vectors.query(aoi, query) from collections import namedtuple Rectangle = namedtuple('Rectangle', 'xmin ymin xmax ymax') ra = Rectangle(min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs)) # intersection here is (3, 3, 4, 3.5), or an area of 1*.5=.5 def area(a, b): # returns None if rectangles don't intersect dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin) dy = min(a.ymax, b.ymax) - max(a.ymin, b.ymin) if (dx>=0) and (dy>=0): return dx*dy # set cloud check images_df = pd.DataFrame(columns=['id','month','year','type','resolution','cloud cover','overlap','check']) images_df2 = pd.DataFrame(columns=['id','month','year','type','resolution','cloud cover','overlap','check']) for r in results: props = r['properties'] geom = r['geometry'] coordinates_image = geom['coordinates'][0][0] df_overlap = pd.DataFrame.from_records(coordinates_image,columns = ['x','y']) rb = Rectangle(df_overlap.x.min(), df_overlap.y.min(), df_overlap.x.max(), df_overlap.y.max()) total_area = ((ra.xmax-ra.xmin)*(ra.ymax-ra.ymin)) total_overlap = area(ra, rb) if total_overlap is None: fraction_overlap = 0 else: fraction_overlap = area(ra, rb)/total_area images_df = images_df.append({'id': props['attributes']['catalogID'],'month':props['item_date'][5:7],'year':props['item_date'][0:4],'type':props['item_type'][1],'resolution':props['attributes']['resolution_dbl'],'cloud cover':props['attributes']['cloudCover_int'],'overlap': fraction_overlap,'check':props['attributes']['cloudCover_int'] < 100 and int(props['item_date'][5:7]) >= 5 and int(props['item_date'][5:7]) <= 8 and fraction_overlap >.9},ignore_index=True) images_df2 = images_df2.append({'id': props['attributes']['catalogID'],'month':props['item_date'][5:7],'year':props['item_date'][0:4],'type':props['item_type'][1],'resolution':props['attributes']['resolution_dbl'],'cloud cover':props['attributes']['cloudCover_int'],'overlap': fraction_overlap,'check':props['attributes']['cloudCover_int'] < 100 and int(props['item_date'][5:7]) <= 2 or int(props['item_date'][5:7]) >= 11 and fraction_overlap >.9},ignore_index=True) selections = images_df.loc[images_df.check == True].reset_index() selectionw = images_df2.loc[images_df2.check == True].reset_index() selections = selections.groupby(['id'],sort=['year','month','cloud cover'],as_index=False).first().sort_values(['year','month'], ascending=False).reset_index() del selections['level_0'], selections['index'] selectionw = selectionw.groupby(['id'],sort=['year','month','cloud cover'],as_index=False).first().sort_values(['year','month'], ascending=False).reset_index() del selectionw['level_0'], selectionw['index'] # First check if images are not defective and then fetch them bbox = env.inputs.get('bbox', bbox_park_area_str) # Select the first image imagestatus ='unknown' iS=0 iW=0 if (selections.empty | selectionw.empty): error = 0 return error, error, error, error else: while imagestatus =='unknown': # set catalog id from selection catalog_id_s = env.inputs.get('catalog_id', selections.id[iS]) catalog_id_w = env.inputs.get('catalog_id', selectionw.id[iW]) # collect images images = CatalogImage(catalog_id_s, band_type="MS", bbox=map(float, bbox.split(",")),proj=UTM_EPSG_code ,pansharpen= False) imagew = CatalogImage(catalog_id_w, band_type="MS", bbox=map(float, bbox.split(",")),proj=UTM_EPSG_code ,pansharpen= False) # calculate mean of coastal band to see if image is defective mean_coastals = images[1,:,:].read().mean() mean_coastalw = imagew[1,:,:].read().mean() # use image if not defective, otherwise if mean_coastals > 200: pass else: iS=iS+1 if mean_coastalw > 200: pass else: iW=iW+1 if mean_coastalw > 200 and mean_coastals > 200: imagestatus = 'fine' return selections,selectionw,images,imagew
def fetch(self, key, aoi, pansharpen=False, acomp=False, dra=False, **kwargs): if self.order(): # create tempfile for AOI with tempfile.NamedTemporaryFile(suffix='.geojson', mode='w', delete=False) as f: aoiname = f.name aoistr = json.dumps(aoi) f.write(aoistr) geovec = gippy.GeoVector(aoiname) ext = geovec.extent() bbox = [ext.x0(), ext.y0(), ext.x1(), ext.y1()] # defaults spec = '' pansharpen = False acomp = False dra = False nodata = 0 if self['eo:platform'] in ['GEOEYE01', 'QUICKBIRD02' ] else -1e10 opts = COG # set options if key == 'rgb': spec = 'rgb' nodata = 0 #opts = JPEG_COG elif key == 'rgb-pan': pansharpen = True spec = 'rgb' nodata = 0 elif key == 'visual': pansharpen = True dra = True nodata = 0 #opts = JPEG_COG elif key == 'analytic': acomp = True fout = os.path.join(self.get_path(), self.get_filename(suffix='_%s' % key)) + '.tif' with TemporaryDirectory() as temp_dir: try: if not os.path.exists(fout): logger.info('Fetching %s: %s' % (key, fout)) # TODO - allow for other projections img = CatalogImage( self['id'], pansharpen=pansharpen, acomp=acomp, dra=dra, bbox=bbox) #, proj=utm_epsg(scenes.center())) tmp_fout1 = os.path.join( temp_dir, '%s_%s_1.tif' % (self['id'], key)) tmp_fout2 = os.path.join( temp_dir, '%s_%s_2.tif' % (self['id'], key)) tif = img.geotiff(path=tmp_fout1, proj='EPSG:4326', spec=spec) # clip and save geoimg = gippy.GeoImage(tif, True) # workaround for gbdxtools scaling if key in ['rgb', 'visual']: geoimg = geoimg.autoscale(1, 255).save(tmp_fout2) geoimg.set_nodata(0) # this clips the image to the AOI res = geoimg.resolution() imgout = alg.cookie_cutter([geoimg], fout, geovec[0], xres=res.x(), yres=res.y(), proj=geoimg.srs(), options=opts) imgout.add_overviews([2, 4, 8, 16], resampler='average') imgout = None except Exception as e: logger.warning('Error fetching: %s' % str(e)) #logger.warning('Traceback: %s', traceback.format_exc()) os.remove(aoiname) return fout
def get_classifier(city,buffer_size, min_size, sample_size): ################ this takes some time label_all = np.array([]) data_all = np.array([]) # labels of types to loop over searching for polygons labels = ['forest', 'grass', 'water' , 'building'] # set minimal sizes for polygons of different types dict_size = {'forest': 1 , 'grass': .6, 'water': 1, 'building': .8} # create empty dataframes that will be filled geom_list_selection_all = [] selection_all = pd.DataFrame() # loop over OSM types and get polygons for label in labels: selection, geojson_select, geom_list_selection,UTM_EPSG_code,project_utm,project_wgs = getOSM.get_OSM_polygons(city = city, type_query = label,min_size = dict_size[label]) geom_list_selection_all.extend(geom_list_selection[1:sample_size]) selection_all = selection_all.append(selection[1:sample_size]) ################ this takes some time selection_all = selection_all.reset_index().drop('index',axis = 1) # set classes for each polygon type dict_type = {'Forest': 1,'Wood': 1,'Nature Reserve': 3,'Wetland': 1, 'Grass': 2, 'Farmland': 2, 'Water': 3, 'Building': 4, 'Theatre': 0} # A list of "random" colors (for a nicer output) COLORS = ["#000000", "#FFFF00", "#1CE6FF", "#FF34FF", "#FF4A46", "#008941"] # load multipolygon type from pickle for check multipolygon_type = pickle.load( open( "/home/gremlin/GGCW_tools_git/Pickle/multipolygon_type.p", "rb" ) ) ################ this takes some time for objects in selection_all.index: print '-----------------------------------------\n' ### setting a buffer can remove the polygon or make it into a multipolygon, both are unusable so check if this is the case park_utm = transform(project_utm, geom_list_selection_all[objects]) # apply projection # perform check # get x y coordinates of polygon and set a buffer if polygon is large enough if (type(park_utm.buffer(buffer_size)) == multipolygon_type) | (park_utm.buffer(buffer_size).area == 0): message = "is multipolygon" print message + ' object: ' + str(objects) +'\n' else: x,y = park_utm.buffer(buffer_size).exterior.xy park_buffer_wgs = transform(project_wgs,park_utm.buffer(buffer_size)) # apply projection # get wgs projected x,y coordinates and create bounding box for image aquisition x_wgs,y_wgs = park_buffer_wgs.exterior.xy bbox_park_area_float = min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs) bbox_park_area = str([min(x_wgs), min(y_wgs), max(x_wgs), max(y_wgs)]) bbox_park_area_str = nf.listToStringWithoutBrackets(bbox_park_area) # convert bounding box to well known format usable by GBDX tools bbox_wkt = box(*bbox_park_area_float).wkt selection_images = nf.image_query_check(bbox_wkt,park_utm,buffer_size,multipolygon_type,project_wgs,x_wgs,y_wgs) if not selection_images.empty: # set park bounding box bbox = bbox_park_area_str # set catalog id from selection catalog_id = selection_images.id[0] # collect image image = CatalogImage(catalog_id, band_type="MS", bbox=map(float, bbox.split(",")), proj=UTM_EPSG_code,pansharpen=False) #create array from GBDX image image_array = image[:,:,:].read() # get second band to see if image is defective (some images show only black) # use image if not defective, otherwise if (image_array.size == 0): print('no image') elif (image_array[1,:,:].min() != 0) : print 'image for ' + selection_all.OSM_id[objects] + ' is good object: ' + str(objects ) print image.ipe_metadata["image"]["acquisitionDate"] print image.cat_id print selection_all.item_type[objects] # resize polygon and plot polygon over image # subtract minimal values from utm polygon x and y to set 0,0 point as start x1 = np.subtract(x, min(x)) y1 = np.subtract(y, min(y)) # devide the x and y coordinate of the polygon by the size of the image to match both sizes x2 = np.divide(x1,max(x1)/image.shape[2]) y2 = np.divide(y1,max(y1)/image.shape[1]) n_bands, rows, cols = image.shape # calculate total cells for each class by masking and setting pixel values to 1 # create sequence of edited x and y coordinates, widht and heigth for use in ImageDraw function polygon = [(x2[i], y2[i]) for i in range(len(x2))] width = image.shape[2] height = image.shape[1] # convert polygon coordinates to raster/array values using ImageDraw img = Image.new('L', (width, height), 0) ImageDraw.Draw(img).polygon(polygon, fill=dict_type[selection_all.item_type[objects]]) # convert image to array and set as mask mask = np.array(img) # flip the array for matching with the mask array image_array_flipped = np.fliplr(image_array[:,:,:]) reshaped_data = image_array_flipped.reshape(8,(rows*cols)) reshaped_label = mask.reshape(1,(rows*cols)) # check if this is the first iteration, if so add the first data set otherwise: # append the new image data to the other data if data_all.size == 0: data_all = reshaped_data else: data_all = np.concatenate((data_all,reshaped_data), axis = 1) label_all = np.append(label_all,reshaped_label) # Two subplots, the axes array is 1-d f, axarr = plt.subplots(1,2) axarr[0].imshow(mask) axarr[1].imshow(image_array_flipped[1]) plt.show() # plt.imshow(mask) # plt.show() print label_all.shape print data_all.shape else: print 'image defective' # move to next without doing analysis else: print 'no image' # move to next without doing analysis ################ this takes some time ### Remove pixels without class label_all_no0 = label_all[label_all != 0] data_all_no0 = data_all[:,label_all != 0] data = pd.DataFrame(data_all.T) data['class'] = label_all # Calculate NDVI # 8-band (0:Coastal, 1:Blue, 2:Green, 3:Yellow, 4:Red, 5:Red-edge, 6:NIR1, 7:NIR2) Multispectral # ndvi = (nir - red)/(nir + red) # EVI = 2.5 * ( nir - red ) / ( nir + 6.5 * red - 7.5 * blue+ 1.0 ) data['ndvi'] = (data[6] - data[4])/(data[6] + data[4]) data['EVI'] = 2.5 * (data[6] - data[4]) / (data[6] + 6.5 * data[4] - 7.5 * data[1] + 1 ) data['water_index'] = (data[7] - data[0]) / (data[7] + data[0]) X = data.iloc[:,0:8] y = data['class'] X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) # initialize search space (as a library!) param_grid = { 'max_depth':[20,10],'n_estimators':[40] } gs = make_pipeline( StandardScaler(), GridSearchCV(RandomForestClassifier(min_samples_leaf=2), param_grid = param_grid, cv = 2, refit = True, n_jobs = 1, verbose = 0)) # Instantiate random forest. You can specify default parameters here. # These parameters are not being optimized. # initialize grid search #gs = GridSearchCV(rf, param_grid, verbose=2)#,scoring='roc_auc') gs.fit(X_train,y_train) return gs
Checking whether a workflow is complete (whether canceled, failed, or succeeded): >>> workflow.complete True ''' while not workflow.succeeded: pass # save the data to S3 folder = 'test_folder' workflow.savedata(aoptask.outputs.data, location=folder) workflow.savedata(aoptask.outputs.data) # convert the result into an image img = CatalogImage(aoptask.outputs.data.value) id_img = IdahoImage(aoptask.outputs.data.value) # >>> img = CatalogImage(aoptask.outputs.data) # Traceback (most recent call last): # File "<stdin>", line 1, in <module> # File "C:\Anaconda2\envs\joemcg_rootclone\lib\site-packages\gbdxtools\images\catalog_image.py", line 57, in __init__ # self._ipe_graphs = self._init_graphs() # File "C:\Anaconda2\envs\joemcg_rootclone\lib\site-packages\gbdxtools\images\catalog_image.py", line 119, in _init_graphs # for part in self.metadata['parts']: # File "C:\Anaconda2\envs\joemcg_rootclone\lib\site-packages\gbdxtools\images\catalog_image.py", line 82, in metadata # results = self._query_vectors(query) # File "C:\Anaconda2\envs\joemcg_rootclone\lib\site-packages\gbdxtools\images\catalog_image.py", line 68, in _query_vectors # raise Exception('Unable to query for image properties, the service may be currently down.', err) # Exception: ('Unable to query for image properties, the service may be currently down.', HTTPError(u'504 Server Error: GATEWAY_TIMEOUT for url: https://vector.geobigdata.io/insight-vector/api/index/query/vector-gbdx-alpha-catalog-v2-*/paging?count=100&upper=90.0&lower=-90.0&right=180.0&q=item_type%3AIDAHOImage+AND+attributes.catalogID%3APort+data%3A%0A%09type%3A+directory%0A%09description%3A+The+output+data+directory%0A%09multiplex%3A+False&ttl=5m&left=-180.0',))
def test_image_window_cover_false(self): """ Window cover should not padd the image and return only tiles of 100, 100 (9 windows) """ wv2 = CatalogImage('1030010076B8F500') aoi = wv2.randwindow((325,300)) coverage = [dsk for dsk in aoi.window_cover((100,100), pad=False)] self.assertEquals(len(coverage), 9)
def test_image_window_cover(self): """ Window cover should padd the image by 25 pixels and return 9 windows """ wv2 = CatalogImage('1030010076B8F500') aoi = wv2.randwindow((275, 300)) coverage = [dsk for dsk in aoi.window_cover((100, 100), pad=True)] self.assertEquals(len(coverage), 9)
def test_image_acomp_avail(self): can_acomp = CatalogImage.acomp_available('1050410000B1BD00') self.assertFalse(can_acomp) can_acomp = CatalogImage.acomp_available('104001002ABEA500') self.assertTrue(can_acomp)
def test_image_ordered(self): ordered = CatalogImage.is_ordered('1050410000B1BD00') self.assertFalse(ordered) ordered = CatalogImage.is_ordered('104001002ABEA500') self.assertTrue(ordered)
def test_catalog_image_unsupported_type(self): with self.assertRaises(Exception) as context: img = CatalogImage('MONA_LISA')
def test_base_layer_match_dep(self): # Test access raises warning from gbdxtools import CatalogImage wv2 = CatalogImage('1030010076B8F500') with self.assertWarns(DeprecationWarning): obj = wv2.base_layer_match()
def test_image_acomp_avail(self): can_acomp = CatalogImage.acomp_available(WV02_CATID) self.assertTrue(can_acomp) cant_acomp = CatalogImage.acomp_available(WV01_CATID) self.assertFalse(cant_acomp)
def test_image_window_cover(self): """ Window cover should padd the image by 25 pixels and return 9 windows """ wv2 = CatalogImage('1030010076B8F500') aoi = wv2.randwindow((275,300)) coverage = [dsk for dsk in aoi.window_cover((100,100), pad=True)] self.assertEquals(len(coverage), 9)
def test_image_window_cover_false(self): """ Window cover should not padd the image and return only tiles of 100, 100 (9 windows) """ wv2 = CatalogImage('1030010076B8F500') aoi = wv2.randwindow((325, 300)) coverage = [dsk for dsk in aoi.window_cover((100, 100), pad=False)] self.assertEquals(len(coverage), 9)