def save_planet_images_to_shapefile(geojson_list, save_shp_path, wkt_string, extent_polygon=None, b_group_date=False):
'''
get the meta data and extent of download
:param geojson_list: geojson_list
:param save_shp_path:
:param extent_polygon: a extent polygon
:param b_group_date:
:return:
'''
# remove incomplete scenes
geojson_list = [ item for item in geojson_list if 'incomplete_scenes' not in item ]
if len(geojson_list) < 1:
raise ValueError('No geojson files (exclude incomplete_scenes) the given folder')
if extent_polygon is not None and len(extent_polygon) > 1:
raise ValueError('Only support one extent polygon')
extent = extent_polygon[0]
if b_group_date is False:
geojson_group = {'all': geojson_list}
else:
geojson_group = group_geojson_by_date(geojson_list)
for key in geojson_group.keys():
sub_geojsons = geojson_group[key]
if len(sub_geojsons) < 1:
continue
sel_geojson_list, sel_polygons = get_geojson_list_overlap_a_polygon(extent,sub_geojsons)
if len(sel_geojson_list) < 1:
continue
scene_table, scene_without_asset = get_meta_dict(sel_geojson_list)
if len(scene_table['scene_id']) != len(sel_polygons):
raise ValueError('The count of scence ID and polygon are different, could due to some duplicated scenes ')
# to strings, ESRI Shapefile does not support datetime fields
scene_table['acquisitionDate'] = [ timeTools.datetime2str(item) for item in scene_table['acquisitionDate']]
scene_table['downloadTime'] = [ timeTools.datetime2str(item) for item in scene_table['downloadTime']]
scene_table['Polygons'] = sel_polygons
df = pd.DataFrame(scene_table)
if key=="all":
save_path = save_shp_path
else:
date_str = timeTools.date2str(key)
save_path = io_function.get_name_by_adding_tail(save_shp_path,date_str)
vector_gpd.save_polygons_to_files(df,'Polygons', wkt_string, save_path)
return True
def main():
file_list = io_function.get_file_list_by_pattern(arcticDEM_reg_tif_dir,
'*_dem_reg.tif')
print('Get %d dem_reg.tif from %s' %
(len(file_list), arcticDEM_reg_tif_dir))
year_dates = [
timeTools.get_yeardate_yyyymmdd(os.path.basename(item),
pattern='[0-9]{8}_')
for item in file_list
]
month_list = [item.month for item in year_dates]
value_list = month_list
# save unique date to txt file
dates_unique = set(year_dates)
dates_unique = sorted(dates_unique)
dates_unique_str = [
timeTools.date2str(item, '%Y-%m-%d') for item in dates_unique
]
io_function.save_list_to_txt('dates_unique.txt', dates_unique_str)
# plot a histogram
# bin_count = 12
bins = np.arange(0, 12, 1)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8, 8))
n, bins, patches = ax.hist(value_list,
bins=bins,
alpha=0.75,
ec="black",
linewidth='1.5',
color='grey',
hatch='',
rwidth=1) # density = True, # label=labels,
# ax.legend(prop={'size': 12})
plt.xticks(bins)
ax.tick_params(axis='both',
which='both',
direction='out',
length=7,
labelsize=20) # ,width=50 #,
# if xlabelrotation is not None:
# ax.tick_params(axis='x', labelrotation=90)
# if ylim is not None:
# ax.set_ylim(ylim)
plt.gcf().subplots_adjust(bottom=0.15)
# plt.grid(True)
plt.savefig('ArcticDEM_strip_date_hist.jpg') #
def merge_multi_headwall_shp_to_one(shp_list, save_path):
'''
merge multiple shapefile of headwall on different dates to one file
:param shp_dir:
:param save_path:
:return:
'''
# shp_list = io_function.get_file_list_by_ext('.shp',shp_dir,bsub_folder=False)
if len(shp_list) < 1:
print('Warning, no input shapefile, skip merging multiple shapefiles')
return False
if os.path.isfile(save_path):
print('warning, %s already exists, skip' % save_path)
return True
# merge shapefile, one by one, and add the year and date from filename
line_list = []
id_list = []
year_list = []
date_list = []
length_m_list = [] # length in meters
for shp in shp_list:
# these are line vector, we still can use the following function to read them
lines, lengths = vector_gpd.read_polygons_attributes_list(
shp, 'length_m')
curr_count = len(id_list)
acuiqsition_date = timeTools.get_yeardate_yyyymmdd(
os.path.basename(shp))
year = acuiqsition_date.year
for idx, (line, length) in enumerate(zip(lines, lengths)):
id_list.append(idx + curr_count)
line_list.append(line)
length_m_list.append(length)
year_list.append(year)
date_list.append(timeTools.date2str(acuiqsition_date))
save_pd = pd.DataFrame({
'id': id_list,
'length_m': length_m_list,
'dem_year': year_list,
'dem_date': date_list,
'Line': line_list
})
ref_prj = map_projection.get_raster_or_vector_srs_info_proj4(shp_list[0])
return vector_gpd.save_polygons_to_files(save_pd, 'Line', ref_prj,
save_path)
def dem_diff_newest_oldest(dem_tif_list, out_dem_diff, out_date_diff):
'''
get DEM difference, for each pixel, newest vaild value - oldest valid value
:param dem_list:
:param output:
:return:
'''
if len(dem_tif_list) < 2:
basic.outputlogMessage('error, the count of DEM is smaller than 2')
return False
# groups DEM with original images acquired at the same year months
dem_groups_date = group_demTif_yearmonthDay(dem_tif_list, diff_days=0)
# sort based on yeardate in accending order : operator.itemgetter(0)
dem_groups_date = dict(
sorted(dem_groups_date.items(), key=operator.itemgetter(0)))
txt_save_path = os.path.splitext(out_date_diff)[0] + '.txt'
io_function.save_dict_to_txt_json(txt_save_path, dem_groups_date)
date_list = list(dem_groups_date.keys())
dem_tif_list = [dem_groups_date[key][0] for key in dem_groups_date.keys()
] # each date, only have one tif
tif_obj_list = [raster_io.open_raster_read(tif) for tif in dem_tif_list]
height, width, _ = raster_io.get_width_heigth_bandnum(tif_obj_list[0])
# check them have the width and height
for tif, obj in zip(dem_tif_list[1:], tif_obj_list[1:]):
h, w, _ = raster_io.get_width_heigth_bandnum(obj)
if h != height or w != width:
raise ValueError(
'the height and width of %s is different from others' % tif)
# divide the image the many small patches, then calcuate one by one, solving memory issues.
image_patches = split_image.sliding_window(width,
height,
1024,
1024,
adj_overlay_x=0,
adj_overlay_y=0)
patch_count = len(image_patches)
tif_obj_list = None
# read all and their date
date_pair_list = list(combinations(date_list, 2))
date_diff_list = [(item[1] - item[0]).days for item in date_pair_list]
# sort based on day difference (from max to min)
date_pair_list_sorted = [
x for _, x in sorted(zip(date_diff_list, date_pair_list), reverse=True)
] # descending
# get the difference
date_diff_np = np.zeros((height, width), dtype=np.uint16)
dem_diff_np = np.empty((height, width), dtype=np.float32)
dem_diff_np[:] = np.nan
for idx, patch in enumerate(image_patches):
print('tile: %d / %d' % (idx + 1, patch_count))
patch_w = patch[2]
patch_h = patch[3]
patch_date_diff = np.zeros((patch_h, patch_w), dtype=np.uint16)
patch_dem_diff = np.empty((patch_h, patch_w), dtype=np.float32)
patch_dem_diff[:] = np.nan
# use dict to read data from disk (only need)
dem_data_dict = {}
for p_idx, pair in enumerate(date_pair_list_sorted):
diff_days = (pair[1] - pair[0]).days
basic.outputlogMessage(
'Getting DEM difference using the one on %s and %s, total day diff: %d'
% (timeTools.date2str(pair[1]), timeTools.date2str(
pair[0]), diff_days))
# print(pair,':',(pair[1] - pair[0]).days)
data_old, data_new = read_date_dem_to_memory(p_idx,
pair,
date_pair_list_sorted,
dem_data_dict,
dem_groups_date,
boundary=patch)
# print('data_old shape:',data_old.shape)
# print('data_new shape:',data_new.shape)
diff_two = data_new - data_old
# print(diff_two)
# fill the element
new_ele = np.where(
np.logical_and(np.isnan(patch_dem_diff), ~np.isnan(diff_two)))
patch_dem_diff[new_ele] = diff_two[new_ele]
patch_date_diff[new_ele] = diff_days
# check if all have been filled ( nan pixels)
diff_remain_hole = np.where(np.isnan(patch_dem_diff))
# basic.outputlogMessage(' remain %.4f percent pixels need to be filled'% (100.0*diff_remain_hole[0].size/patch_dem_diff.size) )
if diff_remain_hole[0].size < 1:
break
# copy to the entire image
row_s = patch[1]
row_e = patch[1] + patch[3]
col_s = patch[0]
col_e = patch[0] + patch[2]
dem_diff_np[row_s:row_e, col_s:col_e] = patch_dem_diff
date_diff_np[row_s:row_e, col_s:col_e] = patch_date_diff
# save date diff to tif (16 bit)
raster_io.save_numpy_array_to_rasterfile(date_diff_np,
out_date_diff,
dem_tif_list[0],
nodata=0,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
# # stretch the DEM difference, save to 8 bit.
# dem_diff_np_8bit = raster_io.image_numpy_to_8bit(dem_diff_np,10,-10,dst_nodata=0)
# out_dem_diff_8bit = io_function.get_name_by_adding_tail(out_dem_diff, '8bit')
# raster_io.save_numpy_array_to_rasterfile(dem_diff_np_8bit, out_dem_diff_8bit, dem_tif_list[0], nodata=0)
# if possible, save to 16 bit, to save the disk storage.
# dem_diff_np[0:5,0] = -500
# dem_diff_np[0,0:5] = 500
# print(np.nanmin(dem_diff_np))
# print(np.nanmax(dem_diff_np))
range = np.iinfo(np.int16)
# dem_diff_np_cm = dem_diff_np*100
# if np.nanmin(dem_diff_np_cm) < range.min or np.nanmax(dem_diff_np_cm) > range.max:
# save dem diff to files (float), meter
raster_io.save_numpy_array_to_rasterfile(dem_diff_np,
out_dem_diff,
dem_tif_list[0],
nodata=-9999,
compress='lzw',
tiled='yes',
bigtiff='if_safer')
# else:
# # save dem diff to 16bit, centimeter, only handle diff from -327.67 to 327.67 meters
# dem_diff_np_cm = dem_diff_np_cm.astype(np.int16) # save to int16
# raster_io.save_numpy_array_to_rasterfile(dem_diff_np_cm, out_dem_diff_cm, dem_tif_list[0],nodata=32767,compress='lzw',tiled='yes',bigtiff='if_safer')
return True
def main(options, args):
# https://www.earthdatascience.org/tutorials/intro-google-earth-engine-python-api/
# for each computer, need to run "earthengine authenticate" first.
ee.Initialize()
# all images will save to Google Drive first
# currently, manually download them from Google Drive
save_folder = args[0]
extent_shp = options.extent_shp
if extent_shp is None:
raise ValueError('must provide a exent shp')
region_name = options.region_name
if region_name is None:
region_name = os.path.splitext(os.path.basename(extent_shp))[0]
# checking input shapefile
# if extent_shp is not None:
assert io_function.is_file_exist(extent_shp)
shp_polygon_projection = get_projection_proj4(extent_shp).strip()
if shp_polygon_projection != '+proj=longlat +datum=WGS84 +no_defs':
raise ValueError('Only accept %s, please check projection of %s' %
(shp_polygon_projection, extent_shp))
resolution = options.resolution
global out_res
out_res = resolution
projection = options.projection_epsg
global out_projection
out_projection = projection
extent_4points = get_extent_xy_from_shp(extent_shp)
if os.path.isdir(save_folder) is False:
io_function.mkdir(save_folder)
# Sentinel-1 SAR GRD
# https://developers.google.com/earth-engine/datasets/catalog/COPERNICUS_S1_GRD
# product_list = ['COPERNICUS/S1_GRD']
# https://developers.google.com/earth-engine/tutorials/community/sar-basics
# The Level-1 GRD processing assumes an ellipsoid Earth surface model, thus Level-1 GRD needs to be processed to create the Level-2 geocoded,
# calibrated backscattering coefficients which end up in the GEE COPERNICUS/S1_GRD_FLOAT collection.
# The S1_GRD_FLOAT collection, and its log-scaled COPERNICUS/S1_GRD computed equivalent,
# contains "analysis-ready" images, as they have square pixel spacing, with pixel values as FLOAT32,
# and a UTM projection in the auto-determined zone. Thus, they can be combined with other images, used in feature extractions and reductions, etc.
product_list = ['COPERNICUS/S1_GRD_FLOAT']
Polarisation_list = ['VV', 'HH', 'HV', 'VH']
instrumentMode_list = ['IW']
orbit_pass_list = ['DESCENDING', 'ASCENDING']
b_crop = False # crop images to input extent
dates_list = options.dates_list_txt
if dates_list is None:
start_date, end_date = options.start_date, options.end_date
for product in product_list:
for polarization in Polarisation_list:
for mode in instrumentMode_list:
for orbit_pass in orbit_pass_list:
gee_download_sentinel1_sar_grd(region_name,
product,
polarization,
mode,
orbit_pass,
start_date,
end_date,
extent_4points,
save_folder,
resolution,
projection,
crop=b_crop,
wait_all_finished=True)
else:
all_task_list = []
dates_list = io_function.read_list_from_txt(dates_list)
for idx, date_str in enumerate(dates_list):
active_tasks = active_task_count(all_task_list)
while active_tasks > maximum_submit_tasks:
print(
'%s : %d (>%d) tasks already in the queue, wait 60 seconds'
% (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
active_tasks, maximum_submit_tasks))
time.sleep(60)
active_tasks = active_task_count(all_task_list)
print('%d/%d Get snow cover for %s' %
(idx, len(dates_list), date_str))
start_date = date_str
end_date = timeTools.str2date(
start_date, format='%Y-%m-%d') + timedelta(days=1)
end_date = timeTools.date2str(end_date, format='%Y-%m-%d')
for product in product_list:
for polarization in Polarisation_list:
for mode in instrumentMode_list:
for orbit_pass in orbit_pass_list:
tasks = gee_download_sentinel1_sar_grd(
region_name,
product,
polarization,
mode,
orbit_pass,
start_date,
end_date,
extent_4points,
save_folder,
resolution,
projection,
crop=b_crop,
wait_all_finished=False)
if tasks is False:
continue
all_task_list.extend(tasks)
# wait until all finished
wait_all_task_finished(all_task_list)
def main(options, args):
# https://www.earthdatascience.org/tutorials/intro-google-earth-engine-python-api/
# for each computer, need to run "earthengine authenticate" first.
ee.Initialize()
# all images will save to Google Drive first, then move them to the below folder
# not yet implemented, currently, manually download them from Google Drive
save_folder = args[0]
extent_shp = options.extent_shp
if extent_shp is None:
raise ValueError('must provide a exent shp')
region_name = options.region_name
if region_name is None:
region_name = os.path.splitext(os.path.basename(extent_shp))[0]
# checking input shapefile
# if extent_shp is not None:
assert io_function.is_file_exist(extent_shp)
shp_polygon_projection = get_projection_proj4(extent_shp).strip()
if shp_polygon_projection != '+proj=longlat +datum=WGS84 +no_defs':
raise ValueError('Only accept %s, please check projection of %s' %
(shp_polygon_projection, extent_shp))
extent_4points = get_extent_xy_from_shp(extent_shp)
if os.path.isdir(save_folder) is False:
io_function.mkdir(save_folder)
snow_products = ['MODIS/006/MYD10A1', 'MODIS/006/MOD10A1']
# snow_products = ['MODIS/006/MYD10A1'] # , 'MODIS/006/MOD10A1'
# band_names = ['NDSI']
band_names = ['NDSI_Snow_Cover', 'NDSI_Snow_Cover_Class']
# there are nine bands.
# "id": "NDSI_Snow_Cover",
# "id": "NDSI_Snow_Cover_Basic_QA",
# "id": "NDSI_Snow_Cover_Algorithm_Flags_QA",
# "id": "NDSI",
# "id": "Snow_Albedo_Daily_Tile",
# "id": "orbit_pnt",
# "id": "granule_pnt",
# "id": "NDSI_Snow_Cover_Class",
# "id": "Snow_Albedo_Daily_Tile_Class",
dates_list = options.dates_list_txt
if dates_list is None:
start_date, end_date = options.start_date, options.end_date
for product in snow_products:
for band_name in band_names:
gee_download_modis_snow(region_name, product, band_name,
start_date, end_date, extent_4points,
save_folder)
else:
all_task_list = []
dates_list = io_function.read_list_from_txt(dates_list)
for idx, date_str in enumerate(dates_list):
active_tasks = active_task_count(all_task_list)
while active_tasks > maximum_submit_tasks:
print(
'%s : %d (>%d) tasks already in the queue, wait 60 seconds'
% (datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
active_tasks, maximum_submit_tasks))
time.sleep(60)
active_tasks = active_task_count(all_task_list)
print('%d/%d Get snow cover for %s' %
(idx, len(dates_list), date_str))
start_date = date_str
end_date = timeTools.str2date(
start_date, format='%Y-%m-%d') + timedelta(days=1)
end_date = timeTools.date2str(end_date, format='%Y-%m-%d')
for product in snow_products:
for band_name in band_names:
tasks = gee_download_modis_snow(region_name,
product,
band_name,
start_date,
end_date,
extent_4points,
save_folder,
wait_all_finished=False)
if tasks is False:
continue
all_task_list.extend(tasks)
# wait until all finished
wait_all_task_finished(all_task_list)
pass