def find_poly(user_analysis,Date_Ini, Date_Fin, shape_folder): if user_analysis == 'no': alldb = firebase.get('coordinatesUser/', None) pending = [] for item in alldb.items(): #itera por la bd usuario = item[0] for item_terreno in item[1].values(): try: if item_terreno['status'] == "Pendiente": pending.append(dict({"user" : usuario , "terrain": item_terreno['uid'], "timestamp" : item_terreno['timestamp'], "name":item_terreno['name']})) except: pass pending = sorted(pending, key = lambda i: i['timestamp'],reverse=True) user_analysis = pending[0]['user']+"/"+pending[0]['terrain'] #name = pending[0]['name'] ''' if (Date_Ini == 'no'): request_date = firebase.get('coordinatesUser/'+user_analysis+'/timestamp', None) #Conseguir fecha time_window = firebase.get('coordinatesUser/'+user_analysis+'/years', None) #Conseguir ventana de tiempo Date_Ini = time.strftime('%Y-%m-%d', time.gmtime((int(request_date)/1000) - (int(time_window))*31536000)) Date_Fin = time.strftime('%Y-%m-%d', time.gmtime(int(request_date)/1000)) ''' result = firebase.get('/coordinatesUser/'+user_analysis+'/Coordenadas', None) name = firebase.get('coordinatesUser/'+user_analysis+'/name', None) #Conseguir name lote lote_aoi = Polygon(result) polygons = [] polygons.append(Polygon(lote_aoi)) lote_aoi = gpd.GeoDataFrame(gpd.GeoSeries(poly for poly in polygons), columns=['geometry']) lote_aoi.crs = {'init':'epsg:4326', 'no_defs': True} #epsg:4326 is standard world coordinates #Conversión de coordenadas especificas locales lote_aoi_loc= lote_aoi.to_crs(32618) lote_aoi_loc["x"] = lote_aoi_loc.centroid.map(lambda p: p.x) lote_aoi_loc["y"] = lote_aoi_loc.centroid.map(lambda p: p.y) lote_aoi["x"] = float(lote_aoi.centroid.map(lambda p: p.x)) lote_aoi["y"] = float(lote_aoi.centroid.map(lambda p: p.y)) minx = float(lote_aoi_loc["x"])- (767.5*10) maxx = float(lote_aoi_loc["x"])+ (767.5*10) miny = float(lote_aoi_loc["y"])- (767.5*10) maxy = float(lote_aoi_loc["y"])+ (767.5*10) lote_aoi_loc["area"] = float(lote_aoi_loc.area) lote_aoi["area"] = lote_aoi_loc["area"] #copy area from local in meters #agregar nombre de lote lote_aoi["name"]=name lote_aoi_loc["name"]=name cols=lote_aoi.columns.tolist() cols=cols[-1:]+cols[:-1] #reorder column names lote_aoi = lote_aoi[cols] lote_aoi_loc = lote_aoi_loc[cols] #Creación del shape respecto a coordenas de vértices analysis_area = user_analysis.split("/")[1] w = shapefile.Writer(shape_folder+analysis_area+'/big_box') w.field('name', 'C') w.poly([ [[minx,miny], [minx,maxy], [maxx,maxy], [maxx,miny], [minx,miny]] ]) w.record('polygon') w.close() #creacion de bounding box respecot a vertices, para cloud_finder inProj = Proj(init='epsg:32618') outProj = Proj(init='epsg:4326') x1,y1 = transform(inProj,outProj,minx,miny) x2,y2 = transform(inProj,outProj,maxx,maxy) bbox_coords_wgs84 = [x1,y2,x2,y1] bounding_box = BBox(bbox_coords_wgs84, crs=CRS.WGS84) return lote_aoi, lote_aoi_loc, minx,maxx,miny,maxy, bounding_box, user_analysis,analysis_area, Date_Ini, Date_Fin
def download_cube(self, sensors, date_range, bounds, bands, crs=None, out_bounds=None, outdir='.', ref_res=None, l57_angles_path=None, l8_angles_path=None, **kwargs): """ Downloads a cube of Landsat and/or Sentinel 2 imagery Args: sensors (str or list): The sensors, or sensor, to download. date_range (list): The date range, given as [date1, date2], where the date format is yyyy-mm-dd. bounds (GeoDataFrame, list, or tuple): The geometry bounds (in WGS84 lat/lon) that define the cube extent to download. If given as a ``GeoDataFrame``, only the first ``DataFrame`` record will be used. If given as a ``tuple`` or a ``list``, the order should be (left, bottom, right, top). bands (str or list): The bands to download. crs (Optional[str or object]): The output CRS. If ``bounds`` is a ``GeoDataFrame``, the CRS is taken from the object. out_bounds (Optional[list or tuple]): The output bounds in ``crs``. If not given, the bounds are taken from ``bounds``. outdir (Optional[str]): The output directory. ref_res (Optional[tuple]): A reference cell resolution. l57_angles_path (str): The path to the Landsat 5 and 7 angles bin. l8_angles_path (str): The path to the Landsat 8 angles bin. kwargs (Optional[dict]): Keyword arguments passed to ``to_raster``. Examples: >>> from geowombat.util import GeoDownloads >>> gdl = GeoDownloads() >>> >>> # Download a Landsat 7 panchromatic cube >>> gdl.download_cube(['l7'], >>> ['2010-01-01', '2010-02-01'], >>> (-91.57, 40.37, -91.46, 40.42), >>> ['pan'], >>> crs="+proj=aea +lat_1=-5 +lat_2=-42 +lat_0=-32 +lon_0=-60 +x_0=0 +y_0=0 +ellps=aust_SA +units=m +no_defs") >>> >>> # Download a Landsat 7, 8 and Sentinel 2 cube of the visible spectrum >>> gdl.download_cube(['l7', 'l8', 's2'], >>> ['2017-01-01', '2018-01-01'], >>> (-91.57, 40.37, -91.46, 40.42), >>> ['blue', 'green', 'red'], >>> crs={'init': 'epsg:102033'}, >>> readxsize=1024, >>> readysize=1024, >>> n_workers=1, >>> n_threads=8) """ # TODO: parameterize # kwargs = dict(readxsize=1024, # readysize=1024, # verbose=1, # separate=False, # n_workers=1, # n_threads=8, # n_chunks=100, # overwrite=False) angle_infos = dict() rt = RadTransforms() br = BRDF() la = LinearAdjustments() main_path = Path(outdir) outdir_brdf = main_path.joinpath('brdf') if not main_path.is_dir(): main_path.mkdir() if not outdir_brdf.is_dir(): outdir_brdf.mkdir() if isinstance(sensors, str): sensors = [sensors] status = Path(outdir).joinpath('status.txt') if not status.is_file(): with open(status.as_posix(), mode='w') as tx: pass # Get bounds from geometry if isinstance(bounds, tuple) or isinstance(bounds, list): bounds = Polygon([ (bounds[0], bounds[3]), # upper left (bounds[2], bounds[3]), # upper right (bounds[2], bounds[1]), # lower right (bounds[0], bounds[1]), # lower left (bounds[0], bounds[3]) ]) # upper left bounds = gpd.GeoDataFrame([0], geometry=[bounds], crs={'init': 'epsg:4326'}) bounds_object = bounds.geometry.values[0] if not out_bounds: # Project the bounds out_bounds = bounds.to_crs(crs).bounds.values[0].tolist() # Get WRS file data_bin = os.path.realpath(os.path.dirname(__file__)) data_dir = Path(data_bin).joinpath('../data') shp_dict = dict() if ('l7' in sensors) or ('l8' in sensors): path_tar = Path(data_dir).joinpath('wrs2.tar.gz') path_shp = Path(data_dir).joinpath('wrs2_descending.shp') wrs = os.path.realpath(path_shp.as_posix()) if not path_shp.is_file(): with tarfile.open(os.path.realpath(path_tar.as_posix()), mode='r:gz') as tf: tf.extractall(data_dir.as_posix()) df_wrs = gpd.read_file(wrs) df_wrs = df_wrs[df_wrs.geometry.intersects(bounds_object)] if df_wrs.empty: logger.warning(' The geometry bounds is empty.') return shp_dict['wrs'] = df_wrs if 's2' in sensors: path_tar = Path(data_dir).joinpath('mgrs.tar.gz') path_shp = Path(data_dir).joinpath('sentinel2_grid.shp') mgrs = os.path.realpath(path_shp.as_posix()) if not path_shp.is_file(): with tarfile.open(os.path.realpath(path_tar.as_posix()), mode='r:gz') as tf: tf.extractall(data_dir.as_posix()) df_mgrs = gpd.read_file(mgrs) df_mgrs = df_mgrs[df_mgrs.geometry.intersects(bounds_object)] if df_mgrs.empty: logger.warning(' The geometry bounds is empty.') return shp_dict['mgrs'] = df_mgrs dt1 = datetime.strptime(date_range[0], '%Y-%m-%d') dt2 = datetime.strptime(date_range[1], '%Y-%m-%d') year = dt1.year month = dt1.month months = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] if month < dt2.month: month_range = months[months.index(month):months.index(dt2.month) + 1] else: month_range = months[ months.index(month):] + months[:months.index(dt2.month) + 1] while True: if year > dt2.year: break for m in month_range: yearmonth_query = '{:d}{:02d}'.format(year, m) for sensor in sensors: band_associations = self.associations[sensor] # TODO: get path/row and MGRS from geometry # location = '21/H/UD' # or '225/083' if sensor.lower() == 's2': locations = [ '{}/{}/{}'.format(dfrow.Name[:2], dfrow.Name[2], dfrow.Name[3:]) for dfi, dfrow in shp_dict['mgrs'].iterrows() ] else: locations = [ '{:03d}/{:03d}'.format(int(dfrow.PATH), int(dfrow.ROW)) for dfi, dfrow in shp_dict['wrs'].iterrows() ] for location in locations: if sensor.lower() == 's2': query = '{LOCATION}/*{YM}*.SAFE/GRANULE/*'.format( LOCATION=location, YM=yearmonth_query) else: query = '{LOCATION}/*{PATHROW}_{YM}*_T*'.format( LOCATION=location, PATHROW=location.replace('/', ''), YM=yearmonth_query) # Query and list available files on the GCP self.list_gcp(sensor, query) if not self.search_dict: logger.warning( ' No results found for {SENSOR} at location {LOC}, year {YEAR:d}, month {MONTH:d}.' .format(SENSOR=sensor, LOC=location, YEAR=year, MONTH=m)) continue # Download data if sensor.lower() == 's2': load_bands = sorted([ 'B{:02d}'.format(band_associations[bd]) if bd != 'rededge' else 'B{:02d}A'.format( band_associations[bd]) for bd in bands ]) search_wildcards = ['MTD_TL.xml'] + [ bd + '.jp2' for bd in load_bands ] file_info = self.download_gcp( sensor, outdir=outdir, search_wildcards=search_wildcards, check_file=status.as_posix(), verbose=1) # Reorganize the dictionary to combine bands and metadata new_dict_ = dict() for finfo_key, finfo_dict in file_info.items(): sub_dict_ = dict() if 'meta' in finfo_dict: key = finfo_dict['meta'].name sub_dict_['meta'] = finfo_dict['meta'] for finfo_key_, finfo_dict_ in file_info.items( ): if 'meta' not in finfo_dict_: for bdkey_, bdinfo_ in finfo_dict_.items( ): if '_'.join( bdinfo_.name.split('_') [:-1]) in key: sub_dict_[bdkey_] = bdinfo_ new_dict_[finfo_key] = sub_dict_ file_info = new_dict_ else: del_keys = [ k for k, v in self.search_dict.items() if 'gap_mask' in k ] for dk in del_keys: del self.search_dict[dk] load_bands = sorted([ 'B{:d}'.format(band_associations[bd]) for bd in bands ]) search_wildcards = [ 'ANG.txt', 'MTL.txt', 'BQA.TIF' ] + [bd + '.TIF' for bd in load_bands] file_info = self.download_gcp( sensor, outdir=outdir, search_wildcards=search_wildcards, check_file=status.as_posix(), verbose=1) logger.info(' Finished downloading files') # Create pixel angle files # TODO: this can be run in parallel for finfo_key, finfo_dict in file_info.items(): brdfp = '_'.join( Path(finfo_dict['meta'].name).name.split('_') [:-1]) logger.info(' Processing {} ...'.format(brdfp)) out_brdf = outdir_brdf.joinpath(brdfp + '.tif').as_posix() if os.path.isfile(out_brdf): logger.warning( ' The output BRDF file, {}, already exists.' .format(brdfp)) continue with open(status.as_posix(), mode='r') as tx: lines = tx.readlines() if sensor == 's2': outdir_angles = main_path.joinpath( 'angles_{}'.format( Path(finfo_dict['meta'].name).name. replace('_MTD_TL.xml', ''))) else: outdir_angles = main_path.joinpath( 'angles_{}'.format( Path(finfo_dict['meta'].name).name. replace('_MTL.txt', ''))) outdir_angles.mkdir(parents=True, exist_ok=True) ref_file = finfo_dict[load_bands[0]].name if sensor.lower() == 's2': angle_info = sentinel_pixel_angles( finfo_dict['meta'].name, ref_file, outdir_angles.as_posix(), nodata=-32768, overwrite=False, verbose=1) if ' '.join( bands ) == 'blue green red nir1 nir2 nir3 nir rededge swir1 swir2': rad_sensor = 's2' elif ' '.join( bands ) == 'blue green red nir swir1 swir2': rad_sensor = 's2l7' elif ' '.join(bands) == 'blue green red nir': rad_sensor = 's210' else: rad_sensor = 's2' bandpass_sensor = angle_info.sensor else: meta = rt.get_landsat_coefficients( finfo_dict['meta'].name) angle_info = landsat_pixel_angles( finfo_dict['angle'].name, ref_file, outdir_angles.as_posix(), meta.sensor, l57_angles_path=l57_angles_path, l8_angles_path=l8_angles_path, verbose=1) if (len(bands) == 1) and (bands[0] == 'pan'): rad_sensor = sensor + bands[0] else: if (len(bands) == 6) and (meta.sensor == 'l8'): rad_sensor = 'l8l7' elif (len(bands) == 7) and (meta.sensor == 'l8') and ('pan' in bands): rad_sensor = 'l8l7mspan' elif (len(bands) == 7) and (meta.sensor == 'l7') and ('pan' in bands): rad_sensor = 'l7mspan' else: rad_sensor = meta.sensor bandpass_sensor = sensor # Get band names from user load_bands_names = [ finfo_dict[bd].name for bd in load_bands ] with gw.config.update(sensor=rad_sensor, ref_bounds=out_bounds, ref_crs=crs, ref_res=ref_res if ref_res else load_bands_names[-1]): with gw.open(angle_info.sza, resampling='cubic') as sza, \ gw.open(angle_info.vza, resampling='cubic') as vza, \ gw.open(angle_info.saa, resampling='cubic') as saa, \ gw.open(angle_info.vaa, resampling='cubic') as vaa: with gw.open( load_bands_names, band_names=bands, stack_dim='band', resampling='cubic') as data: #, \ # gw.open(finfo_dict['qa'].name, # band_names=['qa']) as qa: # Setup the mask # if sensor.lower() != 's2': # # if sensor.lower() == 'l8': # qa_sensor = 'l8-c1' # else: # qa_sensor = 'l-c1' # mask = QAMasker(qa, # qa_sensor, # mask_items=['clear', # 'fill', # 'shadow', # 'cloudconf', # 'cirrusconf', # 'snowiceconf'], # confidence_level='maybe').to_mask() if sensor.lower() == 's2': # The S-2 data are in TOAR (0-10000) toar_scaled = (data * 0.0001).clip( 0, 1).astype('float64') toar_scaled.attrs = data.attrs.copy( ) # Convert TOAR to surface reflectance sr = rt.toar_to_sr( toar_scaled, sza, saa, vza, vaa, sensor) else: # Convert DN to surface reflectance sr = rt.dn_to_sr(data, sza, saa, vza, vaa, sensor=rad_sensor, meta=meta) # BRDF normalization sr_brdf = br.norm_brdf( sr, sza, saa, vza, vaa, sensor=rad_sensor, wavelengths=data.band.values. tolist(), out_range=10000.0, nodata=65535) if bandpass_sensor.lower() in [ 'l5', 'l7', 's2a', 's2b' ]: # Linear adjust to Landsat 8 sr_brdf = la.bandpass( sr_brdf, bandpass_sensor.lower(), to='l8', scale_factor=0.0001) # Mask non-clear pixels # attrs = sr_brdf.attrs # sr_brdf = xr.where(mask.sel(band='mask') < 2, sr_brdf, 65535) # sr_brdf = sr_brdf.transpose('band', 'y', 'x') # sr_brdf.attrs = attrs attrs = sr_brdf.attrs.copy() sr_brdf = sr_brdf.clip( 0, 10000).astype('uint16') sr_brdf.attrs = attrs.copy() sr_brdf.gw.to_raster( out_brdf, **kwargs) angle_infos[finfo_key] = angle_info shutil.rmtree(outdir_angles.as_posix()) for k, v in finfo_dict.items(): os.remove(v.name) lines.append(finfo_dict['meta'].name + '\n') with open(status.as_posix(), mode='r+') as tx: tx.writelines(lines) year += 1