def find_poly(user_analysis,Date_Ini, Date_Fin, shape_folder):
if user_analysis == 'no':
alldb = firebase.get('coordinatesUser/', None)
pending = []
for item in alldb.items(): #itera por la bd
usuario = item[0]
for item_terreno in item[1].values():
try:
if item_terreno['status'] == "Pendiente":
pending.append(dict({"user" : usuario , "terrain": item_terreno['uid'], "timestamp" : item_terreno['timestamp'], "name":item_terreno['name']}))
except:
pass
pending = sorted(pending, key = lambda i: i['timestamp'],reverse=True)
user_analysis = pending[0]['user']+"/"+pending[0]['terrain']
#name = pending[0]['name']
'''
if (Date_Ini == 'no'):
request_date = firebase.get('coordinatesUser/'+user_analysis+'/timestamp', None) #Conseguir fecha
time_window = firebase.get('coordinatesUser/'+user_analysis+'/years', None) #Conseguir ventana de tiempo
Date_Ini = time.strftime('%Y-%m-%d', time.gmtime((int(request_date)/1000) - (int(time_window))*31536000))
Date_Fin = time.strftime('%Y-%m-%d', time.gmtime(int(request_date)/1000))
'''
result = firebase.get('/coordinatesUser/'+user_analysis+'/Coordenadas', None)
name = firebase.get('coordinatesUser/'+user_analysis+'/name', None) #Conseguir name lote
lote_aoi = Polygon(result)
polygons = []
polygons.append(Polygon(lote_aoi))
lote_aoi = gpd.GeoDataFrame(gpd.GeoSeries(poly for poly in polygons), columns=['geometry'])
lote_aoi.crs = {'init':'epsg:4326', 'no_defs': True} #epsg:4326 is standard world coordinates
#Conversión de coordenadas especificas locales
lote_aoi_loc= lote_aoi.to_crs(32618)
lote_aoi_loc["x"] = lote_aoi_loc.centroid.map(lambda p: p.x)
lote_aoi_loc["y"] = lote_aoi_loc.centroid.map(lambda p: p.y)
lote_aoi["x"] = float(lote_aoi.centroid.map(lambda p: p.x))
lote_aoi["y"] = float(lote_aoi.centroid.map(lambda p: p.y))
minx = float(lote_aoi_loc["x"])- (767.5*10)
maxx = float(lote_aoi_loc["x"])+ (767.5*10)
miny = float(lote_aoi_loc["y"])- (767.5*10)
maxy = float(lote_aoi_loc["y"])+ (767.5*10)
lote_aoi_loc["area"] = float(lote_aoi_loc.area)
lote_aoi["area"] = lote_aoi_loc["area"] #copy area from local in meters
#agregar nombre de lote
lote_aoi["name"]=name
lote_aoi_loc["name"]=name
cols=lote_aoi.columns.tolist()
cols=cols[-1:]+cols[:-1] #reorder column names
lote_aoi = lote_aoi[cols]
lote_aoi_loc = lote_aoi_loc[cols]
#Creación del shape respecto a coordenas de vértices
analysis_area = user_analysis.split("/")[1]
w = shapefile.Writer(shape_folder+analysis_area+'/big_box')
w.field('name', 'C')
w.poly([
[[minx,miny], [minx,maxy], [maxx,maxy], [maxx,miny], [minx,miny]]
])
w.record('polygon')
w.close()
#creacion de bounding box respecot a vertices, para cloud_finder
inProj = Proj(init='epsg:32618')
outProj = Proj(init='epsg:4326')
x1,y1 = transform(inProj,outProj,minx,miny)
x2,y2 = transform(inProj,outProj,maxx,maxy)
bbox_coords_wgs84 = [x1,y2,x2,y1]
bounding_box = BBox(bbox_coords_wgs84, crs=CRS.WGS84)
return lote_aoi, lote_aoi_loc, minx,maxx,miny,maxy, bounding_box, user_analysis,analysis_area, Date_Ini, Date_Fin
def download_cube(self,
sensors,
date_range,
bounds,
bands,
crs=None,
out_bounds=None,
outdir='.',
ref_res=None,
l57_angles_path=None,
l8_angles_path=None,
**kwargs):
"""
Downloads a cube of Landsat and/or Sentinel 2 imagery
Args:
sensors (str or list): The sensors, or sensor, to download.
date_range (list): The date range, given as [date1, date2], where the date format is yyyy-mm-dd.
bounds (GeoDataFrame, list, or tuple): The geometry bounds (in WGS84 lat/lon) that define the cube extent
to download. If given as a ``GeoDataFrame``, only the first ``DataFrame`` record will be used.
If given as a ``tuple`` or a ``list``, the order should be (left, bottom, right, top).
bands (str or list): The bands to download.
crs (Optional[str or object]): The output CRS. If ``bounds`` is a ``GeoDataFrame``, the CRS is taken
from the object.
out_bounds (Optional[list or tuple]): The output bounds in ``crs``. If not given, the bounds are
taken from ``bounds``.
outdir (Optional[str]): The output directory.
ref_res (Optional[tuple]): A reference cell resolution.
l57_angles_path (str): The path to the Landsat 5 and 7 angles bin.
l8_angles_path (str): The path to the Landsat 8 angles bin.
kwargs (Optional[dict]): Keyword arguments passed to ``to_raster``.
Examples:
>>> from geowombat.util import GeoDownloads
>>> gdl = GeoDownloads()
>>>
>>> # Download a Landsat 7 panchromatic cube
>>> gdl.download_cube(['l7'],
>>> ['2010-01-01', '2010-02-01'],
>>> (-91.57, 40.37, -91.46, 40.42),
>>> ['pan'],
>>> crs="+proj=aea +lat_1=-5 +lat_2=-42 +lat_0=-32 +lon_0=-60 +x_0=0 +y_0=0 +ellps=aust_SA +units=m +no_defs")
>>>
>>> # Download a Landsat 7, 8 and Sentinel 2 cube of the visible spectrum
>>> gdl.download_cube(['l7', 'l8', 's2'],
>>> ['2017-01-01', '2018-01-01'],
>>> (-91.57, 40.37, -91.46, 40.42),
>>> ['blue', 'green', 'red'],
>>> crs={'init': 'epsg:102033'},
>>> readxsize=1024,
>>> readysize=1024,
>>> n_workers=1,
>>> n_threads=8)
"""
# TODO: parameterize
# kwargs = dict(readxsize=1024,
# readysize=1024,
# verbose=1,
# separate=False,
# n_workers=1,
# n_threads=8,
# n_chunks=100,
# overwrite=False)
angle_infos = dict()
rt = RadTransforms()
br = BRDF()
la = LinearAdjustments()
main_path = Path(outdir)
outdir_brdf = main_path.joinpath('brdf')
if not main_path.is_dir():
main_path.mkdir()
if not outdir_brdf.is_dir():
outdir_brdf.mkdir()
if isinstance(sensors, str):
sensors = [sensors]
status = Path(outdir).joinpath('status.txt')
if not status.is_file():
with open(status.as_posix(), mode='w') as tx:
pass
# Get bounds from geometry
if isinstance(bounds, tuple) or isinstance(bounds, list):
bounds = Polygon([
(bounds[0], bounds[3]), # upper left
(bounds[2], bounds[3]), # upper right
(bounds[2], bounds[1]), # lower right
(bounds[0], bounds[1]), # lower left
(bounds[0], bounds[3])
]) # upper left
bounds = gpd.GeoDataFrame([0],
geometry=[bounds],
crs={'init': 'epsg:4326'})
bounds_object = bounds.geometry.values[0]
if not out_bounds:
# Project the bounds
out_bounds = bounds.to_crs(crs).bounds.values[0].tolist()
# Get WRS file
data_bin = os.path.realpath(os.path.dirname(__file__))
data_dir = Path(data_bin).joinpath('../data')
shp_dict = dict()
if ('l7' in sensors) or ('l8' in sensors):
path_tar = Path(data_dir).joinpath('wrs2.tar.gz')
path_shp = Path(data_dir).joinpath('wrs2_descending.shp')
wrs = os.path.realpath(path_shp.as_posix())
if not path_shp.is_file():
with tarfile.open(os.path.realpath(path_tar.as_posix()),
mode='r:gz') as tf:
tf.extractall(data_dir.as_posix())
df_wrs = gpd.read_file(wrs)
df_wrs = df_wrs[df_wrs.geometry.intersects(bounds_object)]
if df_wrs.empty:
logger.warning(' The geometry bounds is empty.')
return
shp_dict['wrs'] = df_wrs
if 's2' in sensors:
path_tar = Path(data_dir).joinpath('mgrs.tar.gz')
path_shp = Path(data_dir).joinpath('sentinel2_grid.shp')
mgrs = os.path.realpath(path_shp.as_posix())
if not path_shp.is_file():
with tarfile.open(os.path.realpath(path_tar.as_posix()),
mode='r:gz') as tf:
tf.extractall(data_dir.as_posix())
df_mgrs = gpd.read_file(mgrs)
df_mgrs = df_mgrs[df_mgrs.geometry.intersects(bounds_object)]
if df_mgrs.empty:
logger.warning(' The geometry bounds is empty.')
return
shp_dict['mgrs'] = df_mgrs
dt1 = datetime.strptime(date_range[0], '%Y-%m-%d')
dt2 = datetime.strptime(date_range[1], '%Y-%m-%d')
year = dt1.year
month = dt1.month
months = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
if month < dt2.month:
month_range = months[months.index(month):months.index(dt2.month) +
1]
else:
month_range = months[
months.index(month):] + months[:months.index(dt2.month) + 1]
while True:
if year > dt2.year:
break
for m in month_range:
yearmonth_query = '{:d}{:02d}'.format(year, m)
for sensor in sensors:
band_associations = self.associations[sensor]
# TODO: get path/row and MGRS from geometry
# location = '21/H/UD' # or '225/083'
if sensor.lower() == 's2':
locations = [
'{}/{}/{}'.format(dfrow.Name[:2], dfrow.Name[2],
dfrow.Name[3:])
for dfi, dfrow in shp_dict['mgrs'].iterrows()
]
else:
locations = [
'{:03d}/{:03d}'.format(int(dfrow.PATH),
int(dfrow.ROW))
for dfi, dfrow in shp_dict['wrs'].iterrows()
]
for location in locations:
if sensor.lower() == 's2':
query = '{LOCATION}/*{YM}*.SAFE/GRANULE/*'.format(
LOCATION=location, YM=yearmonth_query)
else:
query = '{LOCATION}/*{PATHROW}_{YM}*_T*'.format(
LOCATION=location,
PATHROW=location.replace('/', ''),
YM=yearmonth_query)
# Query and list available files on the GCP
self.list_gcp(sensor, query)
if not self.search_dict:
logger.warning(
' No results found for {SENSOR} at location {LOC}, year {YEAR:d}, month {MONTH:d}.'
.format(SENSOR=sensor,
LOC=location,
YEAR=year,
MONTH=m))
continue
# Download data
if sensor.lower() == 's2':
load_bands = sorted([
'B{:02d}'.format(band_associations[bd])
if bd != 'rededge' else 'B{:02d}A'.format(
band_associations[bd]) for bd in bands
])
search_wildcards = ['MTD_TL.xml'] + [
bd + '.jp2' for bd in load_bands
]
file_info = self.download_gcp(
sensor,
outdir=outdir,
search_wildcards=search_wildcards,
check_file=status.as_posix(),
verbose=1)
# Reorganize the dictionary to combine bands and metadata
new_dict_ = dict()
for finfo_key, finfo_dict in file_info.items():
sub_dict_ = dict()
if 'meta' in finfo_dict:
key = finfo_dict['meta'].name
sub_dict_['meta'] = finfo_dict['meta']
for finfo_key_, finfo_dict_ in file_info.items(
):
if 'meta' not in finfo_dict_:
for bdkey_, bdinfo_ in finfo_dict_.items(
):
if '_'.join(
bdinfo_.name.split('_')
[:-1]) in key:
sub_dict_[bdkey_] = bdinfo_
new_dict_[finfo_key] = sub_dict_
file_info = new_dict_
else:
del_keys = [
k for k, v in self.search_dict.items()
if 'gap_mask' in k
]
for dk in del_keys:
del self.search_dict[dk]
load_bands = sorted([
'B{:d}'.format(band_associations[bd])
for bd in bands
])
search_wildcards = [
'ANG.txt', 'MTL.txt', 'BQA.TIF'
] + [bd + '.TIF' for bd in load_bands]
file_info = self.download_gcp(
sensor,
outdir=outdir,
search_wildcards=search_wildcards,
check_file=status.as_posix(),
verbose=1)
logger.info(' Finished downloading files')
# Create pixel angle files
# TODO: this can be run in parallel
for finfo_key, finfo_dict in file_info.items():
brdfp = '_'.join(
Path(finfo_dict['meta'].name).name.split('_')
[:-1])
logger.info(' Processing {} ...'.format(brdfp))
out_brdf = outdir_brdf.joinpath(brdfp +
'.tif').as_posix()
if os.path.isfile(out_brdf):
logger.warning(
' The output BRDF file, {}, already exists.'
.format(brdfp))
continue
with open(status.as_posix(), mode='r') as tx:
lines = tx.readlines()
if sensor == 's2':
outdir_angles = main_path.joinpath(
'angles_{}'.format(
Path(finfo_dict['meta'].name).name.
replace('_MTD_TL.xml', '')))
else:
outdir_angles = main_path.joinpath(
'angles_{}'.format(
Path(finfo_dict['meta'].name).name.
replace('_MTL.txt', '')))
outdir_angles.mkdir(parents=True, exist_ok=True)
ref_file = finfo_dict[load_bands[0]].name
if sensor.lower() == 's2':
angle_info = sentinel_pixel_angles(
finfo_dict['meta'].name,
ref_file,
outdir_angles.as_posix(),
nodata=-32768,
overwrite=False,
verbose=1)
if ' '.join(
bands
) == 'blue green red nir1 nir2 nir3 nir rededge swir1 swir2':
rad_sensor = 's2'
elif ' '.join(
bands
) == 'blue green red nir swir1 swir2':
rad_sensor = 's2l7'
elif ' '.join(bands) == 'blue green red nir':
rad_sensor = 's210'
else:
rad_sensor = 's2'
bandpass_sensor = angle_info.sensor
else:
meta = rt.get_landsat_coefficients(
finfo_dict['meta'].name)
angle_info = landsat_pixel_angles(
finfo_dict['angle'].name,
ref_file,
outdir_angles.as_posix(),
meta.sensor,
l57_angles_path=l57_angles_path,
l8_angles_path=l8_angles_path,
verbose=1)
if (len(bands) == 1) and (bands[0] == 'pan'):
rad_sensor = sensor + bands[0]
else:
if (len(bands) == 6) and (meta.sensor
== 'l8'):
rad_sensor = 'l8l7'
elif (len(bands)
== 7) and (meta.sensor
== 'l8') and ('pan'
in bands):
rad_sensor = 'l8l7mspan'
elif (len(bands)
== 7) and (meta.sensor
== 'l7') and ('pan'
in bands):
rad_sensor = 'l7mspan'
else:
rad_sensor = meta.sensor
bandpass_sensor = sensor
# Get band names from user
load_bands_names = [
finfo_dict[bd].name for bd in load_bands
]
with gw.config.update(sensor=rad_sensor,
ref_bounds=out_bounds,
ref_crs=crs,
ref_res=ref_res if ref_res
else load_bands_names[-1]):
with gw.open(angle_info.sza,
resampling='cubic') as sza, \
gw.open(angle_info.vza,
resampling='cubic') as vza, \
gw.open(angle_info.saa,
resampling='cubic') as saa, \
gw.open(angle_info.vaa,
resampling='cubic') as vaa:
with gw.open(
load_bands_names,
band_names=bands,
stack_dim='band',
resampling='cubic') as data: #, \
# gw.open(finfo_dict['qa'].name,
# band_names=['qa']) as qa:
# Setup the mask
# if sensor.lower() != 's2':
#
# if sensor.lower() == 'l8':
# qa_sensor = 'l8-c1'
# else:
# qa_sensor = 'l-c1'
# mask = QAMasker(qa,
# qa_sensor,
# mask_items=['clear',
# 'fill',
# 'shadow',
# 'cloudconf',
# 'cirrusconf',
# 'snowiceconf'],
# confidence_level='maybe').to_mask()
if sensor.lower() == 's2':
# The S-2 data are in TOAR (0-10000)
toar_scaled = (data * 0.0001).clip(
0, 1).astype('float64')
toar_scaled.attrs = data.attrs.copy(
)
# Convert TOAR to surface reflectance
sr = rt.toar_to_sr(
toar_scaled, sza, saa, vza,
vaa, sensor)
else:
# Convert DN to surface reflectance
sr = rt.dn_to_sr(data,
sza,
saa,
vza,
vaa,
sensor=rad_sensor,
meta=meta)
# BRDF normalization
sr_brdf = br.norm_brdf(
sr,
sza,
saa,
vza,
vaa,
sensor=rad_sensor,
wavelengths=data.band.values.
tolist(),
out_range=10000.0,
nodata=65535)
if bandpass_sensor.lower() in [
'l5', 'l7', 's2a', 's2b'
]:
# Linear adjust to Landsat 8
sr_brdf = la.bandpass(
sr_brdf,
bandpass_sensor.lower(),
to='l8',
scale_factor=0.0001)
# Mask non-clear pixels
# attrs = sr_brdf.attrs
# sr_brdf = xr.where(mask.sel(band='mask') < 2, sr_brdf, 65535)
# sr_brdf = sr_brdf.transpose('band', 'y', 'x')
# sr_brdf.attrs = attrs
attrs = sr_brdf.attrs.copy()
sr_brdf = sr_brdf.clip(
0, 10000).astype('uint16')
sr_brdf.attrs = attrs.copy()
sr_brdf.gw.to_raster(
out_brdf, **kwargs)
angle_infos[finfo_key] = angle_info
shutil.rmtree(outdir_angles.as_posix())
for k, v in finfo_dict.items():
os.remove(v.name)
lines.append(finfo_dict['meta'].name + '\n')
with open(status.as_posix(), mode='r+') as tx:
tx.writelines(lines)
year += 1
