def test_icor_timeseries():
cloudmask = connection.load_collection("SENTINEL2_L2A_SENTINELHUB",temporal_extent=["2019-02-01", "2019-11-01"],bands=["CLM"])
l1c = connection.load_collection("SENTINEL2_L1C_SENTINELHUB",
temporal_extent=["2019-02-01", "2019-11-01"],
bands=['B08','B04', 'B8A','B09', 'B11', 'sunAzimuthAngles',
'sunZenithAngles', 'viewAzimuthMean', 'viewZenithMean'])
start_time = time.time()
json = l1c.mask(cloudmask).atmospheric_correction().ndvi(nir='B08',red='B04').polygonal_mean_timeseries(polygon).execute()
elapsed_time = time.time() - start_time
print("seconds: " + str(elapsed_time))
from openeo.rest.conversions import timeseries_json_to_pandas
df = timeseries_json_to_pandas(json)
l2a = connection.load_collection("SENTINEL2_L2A_SENTINELHUB",
temporal_extent=["2019-02-01", "2019-11-01"], bands=['B08','B04'])
start_time = time.time()
l2a_json = l2a.mask(cloudmask).ndvi(nir='B08',red='B04').polygonal_mean_timeseries(polygon).execute()
elapsed_time = time.time() - start_time
print("seconds: " + str(elapsed_time))
df_l2a = timeseries_json_to_pandas(l2a_json)
import pandas as pd
df.index = pd.to_datetime(df.index)
df_l2a.index = pd.to_datetime(df_l2a.index)
df.name = "iCor"
df_l2a.name = "Sen2Cor"
joined = pd.concat([df, df_l2a], axis=1)
print(joined)
joined.to_csv('timeseries_icor_masked.csv')
def get_angle(geo, start, end):
scale = 0.0005
offset = 29
orbit_passes = [r'ASCENDING', r'DESCENDING']
dict_df_angles_fields = dict()
for orbit_pass in orbit_passes:
angle = self._eoconn.load_collection('S1_GRD_SIGMA0_{}'.format(orbit_pass), bands = ['angle']).band('angle')
try:
angle_fields = angle.filter_temporal(start,end).polygonal_mean_timeseries(geo).send_job().start_and_wait().get_result().load_json()
df_angle_fields = timeseries_json_to_pandas(angle_fields)
except Exception:
print('RUNNING IN EXECUTE MODE WAS NOT POSSIBLE ... TRY BATCH MODE')
angle.polygonal_mean_timeseries(geo).filter_temporal(start, end).execute_batch('angle_{}.json'.format(orbit_pass))
with open('angle_{}.json'.format(orbit_pass), 'r') as angle_file:
angle_fields_ts = json.load(angle_file)
df_angle_fields = timeseries_json_to_pandas(angle_fields_ts)
df_angle_fields.index = pd.to_datetime(df_angle_fields.index).date
angle_file.close()
os.remove(os.path.join(os.getcwd(),'angle_{}.json'.format(orbit_pass)))
new_columns = [str(item) + '_angle' for item in list(df_angle_fields.columns.values)]
df_angle_fields.rename(columns = dict(zip(list(df_angle_fields.columns.values), new_columns)), inplace= True)
df_angle_fields = df_angle_fields*scale + offset
dict_df_angles_fields.update({'{}'.format(orbit_pass): df_angle_fields})
return dict_df_angles_fields
def test_timeseries_json_to_pandas_auto_collapse(timeseries, index,
with_auto_collapse,
without_auto_collapse):
df = timeseries_json_to_pandas(timeseries, index=index, auto_collapse=True)
if isinstance(with_auto_collapse, pd.Series):
assert_series_equal(df.sort_index(), with_auto_collapse)
else:
assert_frame_equal(df, with_auto_collapse)
df = timeseries_json_to_pandas(timeseries,
index=index,
auto_collapse=False)
assert_frame_equal(df, without_auto_collapse)
def test_timeseries_json_to_pandas_basic():
# 2 dates, 2 polygons, 3 bands
timeseries = {
DATE1: [[1, 2, 3], [4, 5, 6]],
DATE2: [[7, 8, 9], [10, 11, 12]],
}
df = timeseries_json_to_pandas(timeseries)
expected = pd.DataFrame(data=[[1, 2, 3, 4, 5, 6], [7, 8, 9, 10, 11, 12]],
index=pd.Index([DATE1, DATE2], name="date"),
columns=pd.MultiIndex.from_tuples([(0, 0), (0, 1),
(0, 2), (1, 0),
(1, 1), (1, 2)],
names=("polygon",
"band")))
assert_frame_equal(df, expected)
def main():
url = "https://openeo.vito.be"
conn = openeo.connect(url)
polygon = shapely.geometry.Polygon([(3.71, 51.01), (3.72, 51.02),
(3.73, 51.01)])
bbox = polygon.bounds
result = (conn.load_collection(
"TERRASCOPE_S2_TOC_V2",
temporal_extent=["2020-01-01", "2020-03-10"],
spatial_extent=dict(zip(["west", "south", "east", "north"], bbox)),
bands=["B04",
"B08"]).ndvi().polygonal_mean_timeseries(polygon).execute())
pprint(timeseries_json_to_pandas(result))
def test_timeseries_json_to_pandas_index_polygon():
# 2 dates, 2 polygons, 3 bands
timeseries = {
DATE1: [[1, 2, 3], [4, 5, 6]],
DATE2: [[7, 8, 9], [10, 11, 12]],
}
df = timeseries_json_to_pandas(timeseries, index="polygon")
expected = pd.DataFrame(data=[[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]],
index=pd.Index([0, 1], name="polygon"),
columns=pd.MultiIndex.from_tuples([(DATE1, 0),
(DATE1, 1),
(DATE1, 2),
(DATE2, 0),
(DATE2, 1),
(DATE2, 2)],
names=("date",
"band")))
assert_frame_equal(df, expected)
def test_timeseries_json_to_pandas_none_nan_empty_handling():
timeseries = {
DATE1: [[1, 2], [3, 4]],
DATE2: [[5, 6], [None, None]],
DATE3: [[], []],
DATE4: [[], [7, 8]],
}
df = timeseries_json_to_pandas(timeseries)
expected = pd.DataFrame(data=[
[1, 2, 3, 4],
[5, 6, np.nan, np.nan],
[np.nan, np.nan, np.nan, np.nan],
[np.nan, np.nan, 7, 8],
],
dtype=float,
index=pd.Index([DATE1, DATE2, DATE3, DATE4],
name="date"),
columns=pd.MultiIndex.from_tuples([(0, 0), (0, 1),
(1, 0), (1, 1)],
names=("polygon",
"band")))
assert_frame_equal(df, expected)
datacube = connection.load_collection("TERRASCOPE_S2_NDVI_V2")
polygons_file = r'../data/BRP_Gewaspercelen_2017_subset.geojson'
import geopandas as gpd
#apply a
local_polygons = gpd.read_file(polygons_file)
def load_udf(relative_path):
dir = Path(os.path.dirname(os.path.realpath(__file__)))
with open(dir / relative_path, 'r+') as f:
return f.read()
smoothing_udf = load_udf('./smooth_savitzky_golay.py')
#for zonal stats on large shp files: the file can be stored on terrascope, and batch jobs should be used
ndvi_zonal_statistics = datacube\
.mask(scl_mask)\
.apply_dimension(smoothing_udf,dimension='t',runtime='Python')\
.filter_temporal('2017-01-01', '2018-01-01')\
.polygonal_mean_timeseries(GeometryCollection(list(local_polygons.geometry)))
print(json.dumps(ndvi_zonal_statistics.graph, indent=2))
#Backend can derive bbox from polygons provided to aggregate_spatial?
#.filter_bbox(west=5.251809,east=5.462144,north=51.838069,south=51.705835)\
result = ndvi_zonal_statistics.execute()
df = timeseries_json_to_pandas(result)
df.index = pd.to_datetime(df.index)
df.interpolate().plot()
def udf_cropcalendars(udf_data: UdfData):
context_param_var = udf_data.user_context
print(context_param_var)
ts_dict = udf_data.get_structured_data_list()[0].data
if not ts_dict: #workaround of ts_dict is empty
return
ts_df = timeseries_json_to_pandas(ts_dict)
ts_df.index = pd.to_datetime(ts_df.index).date
# function to calculate the cropsar curve
ts_df_cropsar = get_cropsar_TS(
ts_df, context_param_var.get('unique_ids_fields'),
context_param_var.get('metrics_order'),
context_param_var.get('fAPAR_rescale_Openeo'))
# rescale cropsar values
ts_df_cropsar = rescale_cropSAR(
ts_df_cropsar, context_param_var.get('fAPAR_range_normalization'),
context_param_var.get('unique_ids_fields'), 'cropSAR')
# function to rescale the metrics based
# on the rescaling factor of the metric
def rescale_metrics(df, rescale_factor, fAPAR_range, unique_ids_fields,
metric_suffix):
df[[
item + '_{}'.format(str(metric_suffix))
for item in unique_ids_fields
]] = df.loc[:,
ts_df.columns.isin([
item + '_{}'.format(str(metric_suffix))
for item in unique_ids_fields
])] * rescale_factor
df[[
item + '_{}'.format(str(metric_suffix))
for item in unique_ids_fields
]] = 2 * (df[[
item + '_{}'.format(str(metric_suffix))
for item in unique_ids_fields
]] - fAPAR_range[0]) / (fAPAR_range[1] - fAPAR_range[0]) - 1
return df
#### USE THE FUNCTIONS TO DETERMINE THE CROP CALENDAR DATES
### EVENT 1: HARVEST DETECTION
NN_model_dir = context_param_var.get('path_harvest_model')
amount_metrics_model = len(context_param_var.get(
'metrics_crop_event')) * context_param_var.get('window_values')
#### PREPARE THE DATAFRAMES (REFORMATTING AND RESCALING) IN THE
# RIGHT FORMAT TO ALLOW THE USE OF THE TRAINED NN
ts_df_prepro = rename_df_columns(
ts_df, context_param_var.get('unique_ids_fields'),
context_param_var.get('metrics_order'))
ts_df_prepro = VHVV_calc_rescale(
ts_df_prepro, context_param_var.get('unique_ids_fields'),
context_param_var.get('VH_VV_range_normalization'))
#### rescale the fAPAR to 0 and 1 and convert
# it to values between -1 and 1
ts_df_prepro = rescale_metrics(
ts_df_prepro, context_param_var.get('fAPAR_rescale_Openeo'),
context_param_var.get('fAPAR_range_normalization'),
context_param_var.get('unique_ids_fields'), 'fAPAR')
ro_s = {
'ascending': context_param_var.get('RO_ascending_selection_per_field'),
'descending':
context_param_var.get('RO_descending_selection_per_field')
}
#### now merge the cropsar ts file with the other
# df containing the S1 metrics
date_range = pd.date_range(ts_df_cropsar.index[0],
ts_df_cropsar.index[-1]).date
ts_df_prepro = ts_df_prepro.reindex(
date_range) # need to set the index axis on the same frequency
ts_df_prepro = pd.concat(
[ts_df_cropsar, ts_df_prepro], axis=1
) # the columns of the cropsar df need to be the first ones in the new df to ensure the correct position for applying the NN model
### create windows in the time series to extract the metrics
# and store each window in a seperate row in the dataframe
ts_df_input_NN = prepare_df_NN_model(
ts_df_prepro, context_param_var.get('window_values'),
context_param_var.get('unique_ids_fields'), ro_s,
context_param_var.get('metrics_crop_event'))
### apply the trained NN model on the window extracts
df_NN_prediction = apply_NN_model_crop_calendars(
ts_df_input_NN, amount_metrics_model,
context_param_var.get('thr_detection'),
context_param_var.get('crop_calendar_event'), NN_model_dir)
df_crop_calendars_result = create_crop_calendars_fields(
df_NN_prediction, context_param_var.get('unique_ids_fields'),
context_param_var.get('index_window_above_thr'))
print(df_crop_calendars_result)
# return the predicted crop calendar events as a dict (json format)
udf_data.set_structured_data_list([
StructuredData(description="crop calendar json",
data=df_crop_calendars_result.to_dict(),
type="dict")
])
return udf_data
def test_timeseries_json_to_pandas_invalid_polygon_and_band_counts(error, ts):
with pytest.raises(InvalidTimeSeriesException, match=error):
timeseries_json_to_pandas(ts)