def pre_ins_func_on_click(b):
progress.clear_output()
try:
functions = glob.glob(f"{path_plug}*.func")
db = config.get_value(['set', 'db_conn'])
sche = config.get_value(['db', db, 'conn', 'sche'])
user = config.get_value(['db', db, 'conn', 'user'])
for f in functions:
database.insert_function(open(f).read().format(
schema=sche, owner=user))
finc_list = [f"ipycbm_{f.split('/')[-1].split('.')[0]}, " for f in functions]
outlog(f"The functions: {('').join(finc_list)}where added to the database")
except Exception as err:
outlog("Could not add functions to dattabase.", err)
def data_source():
source = int(config.get_value(['set', 'data_source']))
if source == 0:
from src.ipycbm.sources import rest_api
return rest_api
elif source == 1:
from src.ipycbm.sources import direct
return direct
def get_data(parcel):
values = config.read()
get_requests = data_source()
pid = parcel['ogc_fid'][0]
source = int(config.get_value(['set', 'data_source']))
if source == 0:
datapath = f'{paths.value}{aois.value}{year.value}/parcel_{pid}/'
elif source == 1:
ds_conf = config.get_value(['set', 'ds_conf'])
datapath = f'{paths.value}{ds_conf}/parcel_{pid}/'
file_pinf = f"{datapath}{pid}_information"
outlog(data_handler.export(parcel, 10, file_pinf))
if pts_bt.value is True:
outlog(f"Getting time series for parcel: '{pid}',",
f"({pts_tstype.value} {pts_band.value}).")
for pts in pts_tstype.value:
ts = json.loads(
get_requests.pts(aois.value, year.value, pid, pts,
pts_band.value))
band = ''
if pts_band.value != '':
band = f"_{pts_band.value}"
file_ts = f"{datapath}{pid}_time_series_{pts}{band}"
outlog(data_handler.export(ts, 11, file_ts))
if pci_bt.value is True:
files_pci = f"{datapath}{pid}_chip_images/"
outlog(f"Getting '{pci_band.value}' chip images for parcel: {pid}")
with progress:
get_requests.rcbl(parcel, pci_start_date.value,
pci_end_date.value, pci_band.value,
pci_satellite.value, pci_chipsize.value,
files_pci)
filet = f'{datapath}/{pid}_chip_images/{pid}_images_list.{pci_band.value[0]}.csv'
if file_len(filet) > 1:
outlog(
f"Completed, all GeoTIFFs for bands '{pci_band.value}' are ",
f"downloaded in the folder: '{datapath}/{pid}_chip_images'"
)
else:
outlog(
"No files where downloaded, please check your configurations"
)
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 1:
display(wbox_lat_lot)
elif obj['new'] == 2:
display(wbox_pids)
elif obj['new'] == 3:
display(
get_maps.base_map(
aois.value,
int(config.get_value(['set', 'data_source']))))
elif obj['new'] == 4:
display(
VBox([
get_maps.polygon(
aois.value,
int(config.get_value(['set', 'data_source']))),
get_ids_box, ppoly_out
]))
def get_value(dict_keys, var_name=None):
"""Get value for tables.
Example:
database.get_value(['database', 'table'], variable_table_name)
Arguments:
dict_keys, list of keys to get value from.
var_name, the name ofthe variable
"""
if dict_keys[0][-1] == '2':
config_value = config.get_value(dict_keys)
value = config.autoselect(config_value, tables(2), True)
else:
config_value = config.get_value(dict_keys)
value = config.autoselect(config_value, tables(1), True)
if var_name is not None:
if value == None:
print(f"!WARNING! The value for table '{var_name}' is: '{value}'.")
else:
print(f"The value for table '{var_name}' is: '{value}'.")
return value
def bt_get_on_click(b):
progress.clear_output()
if method.value == 1:
try:
with progress:
get_requests = data_source()
lon, lat = plon.value, plat.value
get_from_location(lon, lat)
except Exception as err:
outlog(
f"Could not get parcel information for location '{lon}', '{lat}': {err}"
)
elif method.value == 2:
try:
with progress:
pids = pid.value.replace(" ", "").split(",")
get_from_id(pids)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 3:
try:
marker = get_maps.base_map.map_marker
lon = str(round(marker.location[1], 2))
lat = str(round(marker.location[0], 2))
get_from_location(lon, lat)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 4:
try:
file = config.get_value(['files', 'pids_poly'])
with open(file, "r") as text_file:
pids = text_file.read().split('\n')
outlog("Geting data form the parcels:")
outlog(pids)
if len(pids) <= plimit:
get_from_id(pids)
else:
outlog(
"You exceeded the maximum amount of selected parcels ",
f"({plimit}) to get data. Please select smaller area.")
except Exception as err:
outlog("No pids file found.", err)
else:
outlog(f"Please select method to get parcel information.")
def widget_box():
source = int(config.get_value(['set', 'data_source']))
sources = RadioButtons(
options=[
("JRC RESTful API.", 0),
("Direct access to database and object storage.", 1)
],
value=source,
layout={'width': 'max-content'}
)
sources_box = Box([
Label(value="Data sources:"),
sources]
)
info_api = Label("RESTful API Settings.")
info_direct = Label("Direct access settings")
view_options = VBox([info_direct])
if source == 0:
view_options.children = [info_api, rest_api()]
elif source == 1:
view_options.children = [info_direct, direct()]
def on_source_change(change):
view_options.children = []
if sources.value == 0:
view_options.children = [info_api, rest_api()]
elif sources.value == 1:
view_options.children = [info_direct, direct()]
config.update(['set', 'data_source'], str(sources.value))
sources.observe(on_source_change, 'value')
wbox_sources = VBox([sources_box, view_options],
layout=Layout(border='1px solid black'))
info_general = Label(value="General settings:")
wbox = VBox([wbox_sources, info_general, settings.widget_box()])
return wbox
def widget_box():
source = config.get_value(['set', 'data_source'])
sources = RadioButtons(
options=[
("DIAS API.", 'dias_api'),
("Direct access to database and object storage.", 'direct')
],
layout={'width': 'max-content'}
)
sources_box = Box([
Label(value="Available sources:"),
sources]
)
info_api = Label("DIAS API Settings.")
info_direct = Label("Direct access settings")
view_options = VBox([info_direct])
if source == 'direct':
view_options.children = [info_direct]
else:
view_options.children = [info_api, dias_api()]
def on_source_change(change):
view_options.children = []
if sources.value == 'dias_api':
view_options.children = [info_api, dias_api()]
elif sources.value == 'direct':
view_options.children = [info_direct]
config.update(['preferences', 'data_source'], str(sources.value))
sources.observe(on_source_change, 'value')
wbox_sources = VBox([sources_box, view_options],
layout=Layout(border='1px solid black'))
info_general = Label(value="General settings:")
wbox = VBox([wbox_sources, info_general, settings.widget_box()])
return wbox
def startup():
import sys
if sys.version_info < (3, 6):
print("Not supoted python version, ipycbm needs python version > 3.6")
return
paths = config.get_value(['paths'])
for p in paths:
os.makedirs(paths[p], exist_ok=True)
try:
config.update_keys()
except Exception as err:
folder_repo = os.path.dirname(dirname(dirname(abspath(__file__))))
print("The repossitory's folder is: ", folder_repo)
print("!Warning! Could not update config file:", err)
update.check()
display(config.clean_temp(True))
def proc():
l_connec = Label("1. Connect to database and object storage.")
l_create = Label("2. Create the essential CbM tables. The parcels table name will be added as prefix.")
l_upload = Label("3. Upload .shp, with all the required files (.shp, .cpg, .dbf, .prj, .shx).")
l_carddb = Label("4. Add CARD metadata to databse tabel 'xx_dias_catalogue'.")
l_extrac = Label("5. Run parcel extraction routines.")
path_data = f"{config.get_value(['paths', 'temp'])}proc/"
# Connect
db_select = Dropdown(
options=[db for db in config.get_value(['db'])],
description='Configure:',
disabled=True,
layout=Layout(width='140px')
)
db_config = Button(
value=False,
disabled=False,
button_style='info',
tooltip='Configure db connection.',
icon='cogs',
layout=Layout(width='40px')
)
db_box = HBox([db_select, db_config])
db_conf_box = HBox([])
@db_config.on_click
def db_config_on_click(b):
if db_conf_box.children == ():
db_conf_box.children = [settings.direct_conn()]
else:
db_conf_box.children = ()
wbox = VBox([l_connec, VBox([db_box, db_conf_box]),
l_create, proc_func.create_tables(),
l_upload, proc_func.upload_shp(path_data),
l_carddb, proc_card2db.dias_cat(),
l_extrac, proc_func.extraction()])
return wbox
def bt_get_ids_on_click(b):
with ppoly_out:
try:
get_requests = data_source()
ppoly_out.clear_output()
polygon = get_maps.polygon_map.feature_collection['features'][
-1]['geometry']['coordinates'][0]
polygon_str = '-'.join(
['_'.join(map(str, c)) for c in polygon])
outlog_poly(f"Geting parcel ids within the polygon...")
polyids = json.loads(
get_requests.ppoly(aois.value, year.value, polygon_str,
False, True))
outlog_poly(
f"'{len(polyids['ogc_fid'])}' parcels where found:")
outlog_poly(polyids['ogc_fid'])
file = config.get_value(['files', 'pids_poly'])
with open(file, "w") as text_file:
text_file.write('\n'.join(map(str, polyids['ogc_fid'])))
except Exception as err:
outlog("No parcel ids found:", err)
def extractS2(startdate, enddate):
start = time.time()
values = config.read()
dsc = values['set']['ds_conf']
dsy = values['set']['ds_year']
dias_catalogue = values['ds_conf'][dsc]['years'][dsy]['tables']['dias_catalog']
parcels_table = values['ds_conf'][dsc]['years'][dsy]['tables']['parcels']
results_table = values['ds_conf'][dsc]['years'][dsy]['tables']['s2']
inconn = database.connection()
if not inconn:
print("No in connection established")
sys.exit(1)
outconn = database.connection()
if not outconn:
print("No out connection established")
sys.exit(1)
incurs = inconn.cursor()
srid = -1
sridSql = "SELECT srid FROM geometry_columns WHERE f_table_name = '{}';"
try:
incurs.execute(sridSql.format(parcels_table))
result = incurs.fetchone()
if not result:
print("{} does not exist or is not a spatial table")
else:
srid = result[0]
except (Exception, psycopg2.DatabaseError) as error:
print(error)
inconn.close()
sys.exit(1)
# print("Parcel srid = ", srid)
# Get the first image record that is not yet processed
imagesql = f"""
SELECT id, reference, obstime FROM {dias_catalogue}
WHERE obstime between '{startdate}' And '{enddate}'
And status ='ingested' And card = 's2'
ORDER by obstime asc LIMIT 1
"""
updateSql = """
UPDATE {} SET status='{}'
WHERE id = {} And status = '{}'
"""
with inconn:
with inconn.cursor() as trans_cur:
trans_cur.execute(imagesql)
result = trans_cur.fetchone()
if not result:
print("All signatures for the given dates have been extracted.")
inconn.close()
sys.exit(1)
else:
oid = result[0]
reference = result[1]
obstime = result[2]
# Fails if this record is changed in the meantime
trans_cur.execute(updateSql.format(
dias_catalogue, 'inprogress', oid, 'ingested'))
obstime = reference.split('_')[2][0:8]
# print(obstime)
obs_path = "{}/{}/{}".format(obstime[0:4], obstime[4:6], obstime[6:8])
mgrs_tile = reference.split('_')[5]
full_tstamp = reference.split('_')[2]
# Due to some ESA issues with the manifest.safe sometime during 2018, the GRANULE
# directory need to be checked to understand where image data is located.
dias = values['obst']['osdias']
if dias in ['EOSC', 'CREODIAS']:
rootpath = 'Sentinel-2/MSI/L2A'
s3path = "{}/{}/{}/GRANULE/".format(rootpath, obs_path, reference)
elif dias == 'SOBLOO':
rootpath = '{}/L1C'.format(reference.split('_')[0])
s3path = "{}/{}/{}.SAFE/GRANULE/".format(rootpath, reference,
reference.replace('MSIL1C', 'MSIL2A'))
elif dias == 'MUNDI':
from .utils.mundi import get_mundi_s3path
s3path = get_mundi_s3path(reference, obs_path)
flist = object_storage.list_files(s3path)
if not flist:
print("Resource {} not available in S3 storage (FATAL)".format(s3path))
incurs.execute(updateSql.format(
dias_catalogue, 'S2_nopath', oid, 'inprogress'))
inconn.commit()
incurs.close()
inconn.close()
sys.exit(1)
# We want 3 image files only, e.g. to create NDVI
# SOBLOO does not produce 10 m L2A bands and only B8A (not B08)
s3subdir = flist[1]['Key'].replace(s3path, '').split('/')[0]
# print(s3path)
# print(flist[1])
# print(s3subdir)
selection = {'B4': '{}/{}_{}_{}_{}.jp2'.format('R10m', mgrs_tile, full_tstamp, 'B04', '10m'),
'B8': '{}/{}_{}_{}_{}.jp2'.format('R10m', mgrs_tile, full_tstamp, 'B08', '10m'),
'SC': '{}/{}_{}_{}_{}.jp2'.format('R20m', mgrs_tile, full_tstamp, 'SCL', '20m')
}
file_set = {}
# Copy input data from S3 to local disk
for k in selection.keys():
s = selection.get(k)
fpath = f"tmp/{s.split('/')[-1]}"
alt_s = s.replace('0m/', '0m/L2A_')
if object_storage.get_file('{}{}/IMG_DATA/{}'.format(
s3path, s3subdir, s), fpath) == 1:
# print("Image {} found in bucket".format(s))
file_set[k] = fpath
elif object_storage.get_file('{}{}/IMG_DATA/{}'.format(
s3path, s3subdir, alt_s), fpath) == 1:
# LEVEL2AP has another naming convention.
# print("Image {} found in bucket".format(alt_s))
file_set[k] = fpath
else:
print("Neither Image {} nor {} found in bucket".format(s, alt_s))
incurs.execute(updateSql.format(
dias_catalogue, '{} notfound'.format(k), oid, 'inprogress'))
inconn.commit()
incurs.close()
inconn.close()
sys.exit(1)
# Get the parcel polygon in this image' footprint
print(f"Downloaded '*{file_set['B4'][4:-12]}*' images ...")
outsrid = int('326{}'.format(mgrs_tile[1:3]))
incurs.close()
outconn = database.connection()
if not outconn:
print("No out connection established")
sys.exit(1)
# Open a named cursor
incurs = inconn.cursor(name='fetch_image_coverage',
cursor_factory=psycopg2.extras.DictCursor)
ds_conf = config.get_value(['set', 'ds_conf'])
ds_year = config.get_value(['set', 'ds_year'])
pid_column = config.get_value(['ds_conf', ds_conf, 'years', ds_year, 'columns', 'parcels_id'])
parcelsql = f"""
SELECT p.{pid_column}, ST_AsGeoJSON(st_transform(p.wkb_geometry,
{outsrid}))::json
FROM {parcels_table} p, {dias_catalogue} dc
WHERE p.wkb_geometry && st_transform(dc.footprint, {srid})
And st_area(p.wkb_geometry) > 3000.0
And dc.id = {oid}
-- And p.{pid_column} not in (SELECT distinct pid
-- FROM {results_table} where obsid = {oid})
"""
incurs.execute(parcelsql)
sqlload = time.time() - start
# print(f"Images loaded and nrecs[0] features selected from database in {sqlload} seconds")
nrows = {}
for k in file_set.keys():
nrows[k] = 0
affine = {}
array = {}
bands = file_set.keys()
for b in bands:
with rasterio.open(file_set.get(b)) as src:
affine[b] = src.transform
array[b] = src.read(1)
print(f"Extracting signatures for '*{file_set['B4'][4:-12]}* images ...'")
while True:
rowset = incurs.fetchmany(size=2000)
if not rowset:
break
features = {"type": "FeatureCollection",
"features": [{"type": "feature", "geometry": f[1],
"properties": {"pid": int(f[0])}} for f in rowset]}
for b in bands:
zs = zonal_stats(features, array[b], affine=affine[b], stats=[
"count", "mean", "std", "min", "max",
"percentile_25", "percentile_50",
"percentile_75"],
prefix="", nodata=0, geojson_out=True)
df = pd.DataFrame(zs)
df = pd.DataFrame.from_dict(
df.properties.to_dict(), orient='index')
df['obsid'] = oid
df['band'] = b
df.rename(index=str, columns={
"percentile_25": "p25", "percentile_50": "p50",
"percentile_75": "p75"}, inplace=True)
nrows[b] = nrows[b] + len(df)
# df is the dataframe
if len(df) > 0:
df.dropna(inplace=True)
if len(df.values) > 0:
df_columns = list(df)
s_buf = io.StringIO()
df.to_csv(s_buf, header=False, index=False, sep=',')
s_buf.seek(0)
outcurs = outconn.cursor()
# print(tuple(df_columns))
try:
#psycopg2.extras.execute_batch(outcurs, insert_stmt, df.values)
outcurs.copy_from(s_buf, results_table,
columns=tuple(df_columns), sep=',')
outconn.commit()
except psycopg2.IntegrityError as e:
print(f"insert statement {insert_stmt} contains duplicate index")
# except Error as e:
# print(e)
finally:
outcurs.close()
else:
print(f"No valid data in block {nrows[b]}")
outconn.close()
incurs.close()
incurs = inconn.cursor()
try:
incurs.execute(updateSql.format(
dias_catalogue, 'extracted', oid, 'inprogress'))
inconn.commit()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
inconn.close()
if outconn:
outconn.close()
incurs.close()
inconn.close()
print(f"Removing '*{file_set['B4'][4:-12]}*' images.")
for f in file_set.keys():
if os.path.exists(file_set.get(f)):
# print("Removing {}".format(file_set.get(f)))
os.remove(file_set.get(f))
print("Total time required for {} features and {} bands: {} seconds".format(
nrows.get('B8'), len(bands), time.time() - start))
def get():
"""Get the parcel's dataset for the given location or ids"""
info = Label(
"1. Select the region and the year to get parcel information.")
values = config.read()
# Set the max number of parcels that can be downloaded at once.
plimit = int(values['set']['plimit'])
def aois_options():
values = config.read()
options = {}
if values['set']['data_source'] == '0':
for desc in values['api']['options']['aois']:
aoi = f"{values['api']['options']['aois'][desc]}"
options[(desc, aoi)] = values['api']['options']['years'][aoi]
elif values['set']['data_source'] == '1':
for aoi in values['ds_conf']:
desc = f"{values['ds_conf'][aoi]['desc']}"
confgs = values['ds_conf'][aoi]['years']
options[(f'{desc} ({aoi})', aoi)] = [y for y in confgs]
return options
def aois_years():
values = config.read()
years = {}
if values['set']['data_source'] == '0':
for desc in values['api']['options']['aois']:
aoi = values['api']['options']['aois'][desc]
years[aoi] = values['api']['options']['years'][aoi]
elif values['set']['data_source'] == '1':
for aoi in values['ds_conf']:
desc = f"{values['ds_conf'][aoi]['desc']}"
years[aoi] = [y for y in values['ds_conf'][aoi]['years']]
return years
try:
aois = Dropdown(
options=tuple(aois_options()),
value=values['set']['ds_conf'],
description='AOI:',
disabled=False,
)
except:
aois = Dropdown(
options=tuple(aois_options()),
description='AOI:',
disabled=False,
)
year = Dropdown(
options=next(iter(aois_options().values())),
description='Year:',
disabled=False,
)
button_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
@button_refresh.on_click
def button_refresh_on_click(b):
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
def table_options_change(change):
try:
year.options = aois_years()[change.new]
except:
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
aois.observe(table_options_change, 'value')
info_method = Label("2. Select a method to get the data.")
method = ToggleButtons(
options=[('Parcel ID', 2), ('Coordinates', 1), ('Map marker', 3),
('Polygon', 4)],
value=None,
description='',
disabled=False,
button_style='info',
tooltips=[
'Enter lat lon', 'Enter parcel ID', 'Select a point on a map',
'Get parcels id in a polygon'
],
)
plon = Text(value='5.664',
placeholder='Add lon',
description='Lon:',
disabled=False)
plat = Text(value='52.694',
placeholder='Add lat',
description='Lat:',
disabled=False)
wbox_lat_lot = VBox(children=[plat, plon])
info_pid = Label(
"Multiple parcel id codes can be added (comma ',' separated, e.g.: 11111, 22222)."
)
pid = Textarea(value='34296',
placeholder='12345, 67890',
description='Parcel(s) ID:',
disabled=False)
wbox_pids = VBox(children=[info_pid, pid])
bt_get_ids = Button(description="Find parcels",
disabled=False,
button_style='info',
tooltip='Find parcels within the polygon.',
icon='')
get_ids_box = HBox(
[bt_get_ids,
Label("Find the parcels that are in the polygon.")])
ppoly_out = Output()
progress = Output()
def outlog(*text):
with progress:
print(*text)
def outlog_poly(*text):
with ppoly_out:
print(*text)
@bt_get_ids.on_click
def bt_get_ids_on_click(b):
with ppoly_out:
try:
get_requests = data_source()
ppoly_out.clear_output()
polygon = get_maps.polygon_map.feature_collection['features'][
-1]['geometry']['coordinates'][0]
polygon_str = '-'.join(
['_'.join(map(str, c)) for c in polygon])
outlog_poly(f"Geting parcel ids within the polygon...")
polyids = json.loads(
get_requests.ppoly(aois.value, year.value, polygon_str,
False, True))
outlog_poly(
f"'{len(polyids['ogc_fid'])}' parcels where found:")
outlog_poly(polyids['ogc_fid'])
file = config.get_value(['files', 'pids_poly'])
with open(file, "w") as text_file:
text_file.write('\n'.join(map(str, polyids['ogc_fid'])))
except Exception as err:
outlog("No parcel ids found:", err)
method_out = Output(layout=Layout(border='1px solid black'))
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 1:
display(wbox_lat_lot)
elif obj['new'] == 2:
display(wbox_pids)
elif obj['new'] == 3:
display(
get_maps.base_map(
aois.value,
int(config.get_value(['set', 'data_source']))))
elif obj['new'] == 4:
display(
VBox([
get_maps.polygon(
aois.value,
int(config.get_value(['set', 'data_source']))),
get_ids_box, ppoly_out
]))
method.observe(method_options, 'value')
info_type = Label("3. Select datasets to download.")
table_options = HBox([aois, button_refresh, year])
# ########### Time series options #########################################
pts_bt = ToggleButton(
value=False,
description='Time series',
disabled=False,
button_style='success', # success
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pts_bands = data_options.pts_bands()
pts_tstype = SelectMultiple(
options=data_options.pts_tstype(),
value=['s2'],
rows=3,
description='TS type:',
disabled=False,
)
pts_band = Dropdown(
options=list(pts_bands['s2']),
value='',
description='Band:',
disabled=False,
)
def pts_tstype_change(change):
if len(pts_tstype.value) <= 1:
pts_band.disabled = False
try:
pts_b = change.new[0]
pts_band.options = pts_bands[pts_b]
except:
pass
else:
pts_band.value = ''
pts_band.disabled = True
pts_tstype.observe(pts_tstype_change, 'value')
pts_options = VBox(children=[pts_tstype, pts_band])
# ########### Chip images options #########################################
pci_bt = ToggleButton(value=False,
description='Chip images',
disabled=False,
button_style='success',
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pci_start_date = DatePicker(value=datetime.date(2019, 6, 1),
description='Start Date',
disabled=False)
pci_end_date = DatePicker(value=datetime.date(2019, 6, 30),
description='End Date',
disabled=False)
pci_plevel = RadioButtons(
options=['LEVEL2A', 'LEVEL1C'],
value='LEVEL2A',
description='Proces. level:', # Processing level
disabled=False,
layout=Layout(width='50%'))
pci_chipsize = IntSlider(value=640,
min=100,
max=5120,
step=10,
description='Chip size:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='d')
pci_bands = data_options.pci_bands()
pci_satellite = RadioButtons(options=list(pci_bands),
value='Sentinel 2',
disabled=True,
layout=Layout(width='100px'))
pci_band = SelectMultiple(options=list(pci_bands['Sentinel 2']),
value=['B04'],
rows=11,
description='Band:',
disabled=False)
sats_plevel = HBox([pci_satellite, pci_plevel])
def on_sat_change(change):
sat = change.new
pci_band.options = pci_bands[sat]
pci_satellite.observe(on_sat_change, 'value')
pci_options = VBox(children=[
pci_start_date, pci_end_date, sats_plevel, pci_chipsize, pci_band
])
# ########### General options #############################################
pts_wbox = VBox(children=[])
pci_wbox = VBox(children=[])
def pts_observe(button):
if button['new']:
pts_bt.icon = 'toggle-on'
pts_wbox.children = [pts_options]
else:
pts_bt.icon = 'toggle-off'
pts_wbox.children = []
def pci_observe(button):
if button['new']:
pci_bt.icon = 'toggle-on'
pci_wbox.children = [pci_options]
else:
pci_bt.icon = 'toggle-off'
pci_wbox.children = []
pts_bt.observe(pts_observe, names='value')
pci_bt.observe(pci_observe, names='value')
pts = VBox(children=[pts_bt, pts_wbox], layout=Layout(width='40%'))
pci = VBox(children=[pci_bt, pci_wbox], layout=Layout(width='40%'))
data_types = HBox(children=[pts, pci])
info_get = Label("4. Download the selected data.")
bt_get = Button(description='Download',
disabled=False,
button_style='warning',
tooltip='Send the request',
icon='download')
path_temp = config.get_value(['paths', 'temp'])
path_data = config.get_value(['paths', 'data'])
info_paths = HTML("".join([
"<style>div.c {line-height: 1.1;}</style>",
"<div class='c';>By default data will be stored in the temp folder ",
f"({path_temp}), you will be asked to empty the temp folder each time ",
"you start the notebook.<br>In your personal data folder ",
f"({path_data}) you can permanently store the data.</div>"
]))
paths = RadioButtons(options=[
(f"Temporary folder: '{path_temp}'.", path_temp),
(f"Personal data folder: '{path_data}'.", path_data)
],
layout={'width': 'max-content'},
value=path_temp)
paths_box = Box([Label(value="Select folder:"), paths])
def file_len(fname):
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
def get_data(parcel):
values = config.read()
get_requests = data_source()
pid = parcel['ogc_fid'][0]
source = int(config.get_value(['set', 'data_source']))
if source == 0:
datapath = f'{paths.value}{aois.value}{year.value}/parcel_{pid}/'
elif source == 1:
ds_conf = config.get_value(['set', 'ds_conf'])
datapath = f'{paths.value}{ds_conf}/parcel_{pid}/'
file_pinf = f"{datapath}{pid}_information"
outlog(data_handler.export(parcel, 10, file_pinf))
if pts_bt.value is True:
outlog(f"Getting time series for parcel: '{pid}',",
f"({pts_tstype.value} {pts_band.value}).")
for pts in pts_tstype.value:
ts = json.loads(
get_requests.pts(aois.value, year.value, pid, pts,
pts_band.value))
band = ''
if pts_band.value != '':
band = f"_{pts_band.value}"
file_ts = f"{datapath}{pid}_time_series_{pts}{band}"
outlog(data_handler.export(ts, 11, file_ts))
if pci_bt.value is True:
files_pci = f"{datapath}{pid}_chip_images/"
outlog(f"Getting '{pci_band.value}' chip images for parcel: {pid}")
with progress:
get_requests.rcbl(parcel, pci_start_date.value,
pci_end_date.value, pci_band.value,
pci_satellite.value, pci_chipsize.value,
files_pci)
filet = f'{datapath}/{pid}_chip_images/{pid}_images_list.{pci_band.value[0]}.csv'
if file_len(filet) > 1:
outlog(
f"Completed, all GeoTIFFs for bands '{pci_band.value}' are ",
f"downloaded in the folder: '{datapath}/{pid}_chip_images'"
)
else:
outlog(
"No files where downloaded, please check your configurations"
)
def get_from_location(lon, lat):
get_requests = data_source()
outlog(f"Finding parcel information for coordinates: {lon}, {lat}")
parcel = json.loads(
get_requests.ploc(aois.value, year.value, lon, lat, True))
pid = parcel['ogc_fid'][0]
outlog(f"The parcel '{pid}' was found at this location.")
try:
get_data(parcel)
except Exception as err:
print(err)
def get_from_id(pids):
get_requests = data_source()
outlog(f"Getting parcels information for: '{pids}'")
for pid in pids:
try:
parcel = json.loads(
get_requests.pid(aois.value, year.value, pid, True))
get_data(parcel)
except Exception as err:
print(err)
@bt_get.on_click
def bt_get_on_click(b):
progress.clear_output()
if method.value == 1:
try:
with progress:
get_requests = data_source()
lon, lat = plon.value, plat.value
get_from_location(lon, lat)
except Exception as err:
outlog(
f"Could not get parcel information for location '{lon}', '{lat}': {err}"
)
elif method.value == 2:
try:
with progress:
pids = pid.value.replace(" ", "").split(",")
get_from_id(pids)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 3:
try:
marker = get_maps.base_map.map_marker
lon = str(round(marker.location[1], 2))
lat = str(round(marker.location[0], 2))
get_from_location(lon, lat)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 4:
try:
file = config.get_value(['files', 'pids_poly'])
with open(file, "r") as text_file:
pids = text_file.read().split('\n')
outlog("Geting data form the parcels:")
outlog(pids)
if len(pids) <= plimit:
get_from_id(pids)
else:
outlog(
"You exceeded the maximum amount of selected parcels ",
f"({plimit}) to get data. Please select smaller area.")
except Exception as err:
outlog("No pids file found.", err)
else:
outlog(f"Please select method to get parcel information.")
return VBox([
info, table_options, info_method, method, method_out, info_type,
data_types, info_get, info_paths, paths_box, bt_get, progress
])
def foi():
path_plug = "ipycbm/plugins/foi/foi_db_func/"
path_data = f"{config.get_value(['paths', 'temp'])}foi/"
progress = Output()
def outlog(*text):
with progress:
print(*text)
foi_info = HTML(
value="""FOI procedures need direct access to the database. <br>
In case there no image is provided, access to object storage will be needed to generate the base image from sentinel images.
""",
placeholder='FOI Information',
)
# Connect to database
db_info = Label(
f"1. Connect to database and object storage.")
db_select = Dropdown(
options=[db for db in config.get_value(['db'])],
description='Configure:',
disabled=True,
layout=Layout(width='140px')
)
db_config = Button(
value=False,
disabled=False,
button_style='info',
tooltip='Configure db connection.',
icon='cogs',
layout=Layout(width='40px')
)
db_box = HBox([db_select, db_config])
parcels_table = RadioButtons(
options=[('Upload .shp', 0), ('From database', 1)],
# value='pineapple', # Defaults to 'pineapple'
# layout={'width': 'max-content'}, # If the items' names are long
# description='Pizza topping:',
disabled=False
)
par_box = HBox([])
def on_parcels_table(method):
if method.new == 0:
par_box.children = []
elif method.new == 1:
par_box.children = []
parcels_table.observe(on_parcels_table, 'value')
# Generate or upload image.
img_info = Label(
f"3. Upload or generate raster base image. (Only upload is currently available)")
img_option = ToggleButtons(
options=['Upload', 'Generate'],
value=None,
disabled=True,
button_style='info', # 'success', 'info', 'warning', 'danger' or ''
tooltips=['Upnload your base image', 'Get from object storage']
)
img_dist_folder = Text(
value=f"{path_data}raster/",
placeholder='tmp/',
description='Folder:',
disabled=False
)
img_select = FileUpload(
description='Select file:',
icon='plus',
accept='.tif, .tiff',
multiple=True # True to accept multiple files upload else False
)
img_clear = Button(
value=False,
disabled=False,
button_style='info',
tooltip='Clear selections.',
icon='broom',
layout=Layout(width='40px')
)
img_upload = Button(
value=False,
disabled=False,
button_style='info',
tooltip='Upload foi base image (.tif)',
icon='fa-upload',
layout=Layout(width='40px')
)
img_box = HBox([HBox([img_dist_folder, img_select,
img_clear, img_upload])])
# YAML File upload
yml_select = FileUpload(
description='Select file:',
icon='plus',
accept='.yml, .yaml, .txt',
multiple=False
)
yml_clear = Button(
value=False,
disabled=False,
button_style='info',
tooltip='Clear selection.',
icon='broom',
layout=Layout(width='40px')
)
yml_upload = Button(
value=False,
disabled=False,
button_style='info',
tooltip='Upload yaml file.',
icon='fa-upload',
layout=Layout(width='40px')
)
yml_box = HBox([yml_select, yml_clear, yml_upload])
# Prepare procedures
pre_info = Label("4. Prepare FOI procedure.")
pre_ins_func = Button(
value=False,
disabled=False,
button_style='info',
tooltip='Insert functions to database.',
icon='fa-share-square',
layout=Layout(width='40px')
)
pre_ins_func_box = HBox([Label("Add functions to database:"), pre_ins_func])
vector_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}vector/*'
) if '.shp' in s],
description='Vector file:',
disabled=False,
)
raster_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}raster/*'
) if '.tif' in s],
description='Raster file:',
disabled=False,
)
yaml_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}/*'
) if '.yml' in s],
description='yaml file:',
disabled=False,
)
# heterogeneity_threshold
pre_heto_chec = HTML("""
Minimum and maximum thresholds for heterogeneity checks. In the example,
any parcel with percentage of pixels for one class between 30 and 70 from
the total, will be considered heterogenous.
""")
pre_min_het = IntText(
value=30,
description='MIN:',
tooltip="Minimum threshold for heterogeneity checks",
disabled=False,
layout=Layout(width='150px')
)
pre_max_het = IntText(
value=70,
description='MAX:',
tooltip="Maximum threshold for heterogeneity checks",
disabled=False,
layout=Layout(width='150px')
)
pre_heto_chec_box = HBox([pre_min_het, pre_max_het])
pre_min_area = IntText(
value=2000,
description='area:',
tooltip="Minimum area for clusters selection.",
disabled=False,
layout=Layout(width='200px')
)
refresh_selections = Button(
layout=Layout(width='35px'),
icon='fa-refresh'
)
pre_box = VBox([
pre_info, pre_ins_func_box,
HBox([Label("Select the requared files:"), refresh_selections]),
HTML("""a. Spatial data to be tested - parcels that will
be checked for heterogeneity and cardinality."""),
HBox([vector_file]),
HTML("""b. Thematic raster - classification raster, or raster
from other source that will be used for testing heterogeneity and cardinality."""),
HBox([raster_file]),
HTML("""c. YAML file that holds the classes form the thematic raster file -
can be also a simple list of values in the notebook
corespondence between pixel values and names for the classes"""),
HBox([yaml_file, yml_box]),
pre_heto_chec, pre_heto_chec_box,
HBox([pre_min_area, HTML(
"Minimum area for clusters selection - only clusters bigger from this threshold will be counted.")])
])
# Run procedures
run_info = Label("5. Run FOI procedure.")
run_proc = Button(
description='Run FOI procedure',
value=False,
disabled=False,
button_style='info',
tooltip='Run',
icon='play',
)
run_box = HBox([run_proc])
def on_img_option_change(change):
if img_option.value == 'Upload':
img_box.children = [HBox([img_dist_folder, img_select,
img_clear, img_upload])]
else:
img_box.children = ()
img_option.observe(on_img_option_change, 'value')
@refresh_selections.on_click
def refresh_selections_on_click(b):
vector_file.options = [s for s in glob.glob(f'{path_data}vector/*'
) if '.shp' in s]
raster_file.options = [s for s in glob.glob(f'{path_data}raster/*'
) if '.tif' in s]
yaml_file.options = [s for s in glob.glob(f'{path_data}/*'
) if '.yml' in s]
@img_clear.on_click
def img_clear_on_click(b):
img_select.value.clear()
img_select._counter = 0
@img_upload.on_click
def img_upload_on_click(b):
progress.clear_output()
os.makedirs(img_dist_folder.value, exist_ok=True)
for key in img_select.value:
content = img_select.value[key]['content']
with open(f'{img_dist_folder.value}{key}', 'wb') as f:
f.write(content)
outlog("All files are uploaded.")
@yml_clear.on_click
def yml_clear_on_click(b):
yml_select.value.clear()
yml_select._counter = 0
@yml_upload.on_click
def yml_upload_on_click(b):
progress.clear_output()
yml_dist_folder = f'{path_data}'
os.makedirs(yml_dist_folder, exist_ok=True)
for key in yml_select.value:
content = yml_select.value[key]['content']
with open(f'{yml_dist_folder}{key}', 'wb') as f:
f.write(content)
outlog("The yaml file is uploaded.")
db_conf_box = HBox([])
@db_config.on_click
def db_config_on_click(b):
if db_conf_box.children == ():
db_conf_box.children = [settings.direct_conn()]
else:
db_conf_box.children = ()
@pre_ins_func.on_click
def pre_ins_func_on_click(b):
progress.clear_output()
try:
functions = glob.glob(f"{path_plug}*.func")
db = config.get_value(['set', 'db_conn'])
sche = config.get_value(['db', db, 'conn', 'sche'])
user = config.get_value(['db', db, 'conn', 'user'])
for f in functions:
database.insert_function(open(f).read().format(
schema=sche, owner=user))
finc_list = [f"ipycbm_{f.split('/')[-1].split('.')[0]}, " for f in functions]
outlog(f"The functions: {('').join(finc_list)}where added to the database")
except Exception as err:
outlog("Could not add functions to dattabase.", err)
@run_proc.on_click
def run_proc_on_click(b):
with progress:
foi_proc.proc(vector_file.value, raster_file.value, yaml_file.value,
pre_min_het.value, pre_max_het.value, pre_min_area.value)
wbox = VBox([foi_info,
db_info, db_box, db_conf_box,
proc_func.upload_shp(path_data),
img_info, img_option, img_box,
pre_box,
run_info,
run_box,
progress])
return wbox
def qa():
# path_plug = "ipycbm/plugins/foi/foi_db_func/"
path_data = f"{config.get_value(['paths', 'temp'])}qa/"
progress = Output()
def outlog(*text):
with progress:
print(*text)
qa_info = HTML(
value=
"""The CbM Quality Assurance (QA) is a framework intended to enable
the Member State (MS) to report to the Commission about the state of one of
the components inside the control and management system.<br>
To run the Quality Assessment (QA) procedures direct access to the
database and the object storage with CARD data is required.<br>
""",
placeholder='QA Information',
)
# Connect
db_select = Dropdown(options=[db for db in config.get_value(['db'])],
description='Configure:',
disabled=True,
layout=Layout(width='140px'))
db_config = Button(value=False,
disabled=False,
button_style='info',
tooltip='Configure db connection.',
icon='cogs',
layout=Layout(width='40px'))
db_box = HBox([db_select, db_config])
db_conf_box = HBox([])
@db_config.on_click
def db_config_on_click(b):
if db_conf_box.children == ():
db_conf_box.children = [settings.direct_conn()]
else:
db_conf_box.children = ()
accor = Accordion(children=[
VBox([db_box, db_conf_box]),
proc_func.upload_shp(path_data),
proc_func.create_tables(),
proc_card2db.dias_cat(),
get_settings.direct_settings(),
proc_func.extraction(),
VBox([get_panel.get()], layout=Layout(border='1px solid black')),
view_panel.view()
])
def on_tb_name(change):
# Get to the widget that is inside the accordion widget
# Any change of the order of the widgets within the widgets
# box will break this connection.
new_value = accor.children[1].children[3].children[0].value
accor.children[2].children[1].value = new_value
accor.children[1].children[3].children[0].observe(on_tb_name, 'value')
def on_start(change):
new_value = accor.children[3].children[2].children[0].value
accor.children[5].children[0].children[0].value = new_value
accor.children[3].children[2].children[0].observe(on_start, 'value')
def on_end(change):
new_value = accor.children[3].children[2].children[1].value
accor.children[5].children[0].children[1].value = new_value
accor.children[3].children[2].children[1].observe(on_end, 'value')
accor.set_title(0, "1. Connect to database and object storage.")
accor.set_title(
1, "2. Upload .shp, with all the required files (cpg, dbf, prj, shx).")
accor.set_title(2, "3. Create the essential CbM tables.")
accor.set_title(
3, "4. Add CARD metadata to database table 'xx_dias_catalogue'.")
accor.set_title(4, "5. Configure database tables.")
accor.set_title(5, "6. Run parcel extraction routines.")
accor.set_title(6, "7. Get parcel information, time series and images.")
accor.set_title(7, "8. View parcel data.")
wbox = VBox([qa_info, accor])
return wbox
def error(err):
if config.get_value(['api', 'pass']) == '':
print("Please provide a password for you selected data source.")
elif 'line 1 column 1 (char 0)' in str(err):
print("Cloud not conect to the selected data source.",
"Please check your user name and password.")
def extractS1bs(startdate, enddate):
start = time.time()
frootpath = 'tmp'
values = config.read()
dsc = values['set']['ds_conf']
dsy = values['set']['ds_year']
dias_catalogue = values['ds_conf'][dsc]['years'][dsy]['tables'][
'dias_catalog']
parcels_table = values['ds_conf'][dsc]['years'][dsy]['tables']['parcels']
results_table = values['ds_conf'][dsc]['years'][dsy]['tables']['s2']
inconn = database.connection()
if not inconn:
print("No in connection established")
sys.exit(1)
incurs = inconn.cursor()
srid = -1
sridSql = "select srid from geometry_columns where f_table_name = '{}';"
try:
incurs.execute(sridSql.format(parcels_table))
result = incurs.fetchone()
if not result:
print("{} does not exist or is not a spatial table")
else:
srid = result[0]
except (Exception, psycopg2.DatabaseError) as error:
print(error)
inconn.close()
sys.exit(1)
print("Parcel srid = ", srid)
# Get the first image record that is not yet processed
imagesql = f"""
SELECT id, reference, obstime FROM {dias_catalogue}
WHERE obstime between '{startdate}' And '{enddate}'
And status ='ingested' And card = 's2'
ORDER by obstime asc LIMIT 1
"""
updateSql = """
UPDATE {} SET status='{}'
WHERE id = {} And status = '{}'
"""
with inconn:
with inconn.cursor() as trans_cur:
trans_cur.execute(imagesql)
result = trans_cur.fetchone()
if not result:
print(
"All signatures for the given dates have been extracted.")
inconn.close()
sys.exit(1)
else:
oid = result[0]
reference = result[1]
obstime = result[2]
# Fails if this record is changed in the meantime
trans_cur.execute(
updateSql.format(dias_catalogue, 'inprogress', oid,
'ingested'))
inconn.commit()
print(reference)
obstime = reference.split('_')[2][0:8]
print(obstime)
obs_path = "{}/{}/{}".format(obstime[0:4], obstime[4:6], obstime[6:8])
print(obs_path)
mgrs_tile = reference.split('_')[5]
full_tstamp = reference.split('_')[2]
# Copy input data from S3 to local disk
dias = values['obst']['osdias']
if dias in ['EOSC', 'CREODIAS']:
rootpath = 'Sentinel-1/SAR/CARD-BS'
print(datetime.strptime(obstime, '%Y/%m/%d'), reference)
s3path = '{}/{}/{}/{}.data/Gamma0_VV.img'.format(
rootpath, datetime.strftime(obstime, '%Y/%m/%d'), reference,
reference)
elif dias == 'SOBLOO':
s3path = '{}/GRD/{}/{}.data/Gamma0_VV.img'.format(
reference.split('_')[0], reference, reference)
fpath = f'{frootpath}/{reference}_VV.img'
outsrid = -1
if object_storage.get_file(s3path, fpath) == 0:
print("Resource {} not available in S3 storage (FATAL)".format(s3path))
incurs.execute(updateSql.format('No S3 VV img', oid, 'inprogress'))
inconn.commit()
incurs.close()
inconn.close()
sys.exit(1)
s3path = s3path.replace('.img', '.hdr')
fpath = f'{frootpath}/{reference}_VV.hdr'
if object_storage.get_file(s3path, fpath) == 0:
print("Resource {} not available in S3 storage (FATAL)".format(s3path))
incurs.execute(updateSql.format('No S3 VV hdr', oid, 'inprogress'))
inconn.commit()
incurs.close()
inconn.close()
sys.exit(1)
else:
# Only if the header file is present can we open the image to check its projection
with rasterio.open(fpath.replace('hdr', 'img')) as src:
outsrid = src.crs.to_epsg()
print('Out SRID: ', outsrid)
if dias in ['EOSC', 'CREODIAS']:
rootpath = 'Sentinel-1/SAR/CARD-BS'
s3path = '{}/{}/{}/{}.data/Gamma0_VH.img'.format(
rootpath, datetime.strftime(obstime, '%Y/%m/%d'), reference,
reference)
elif dias == 'SOBLOO':
s3path = '{}/GRD/{}/{}.data/Gamma0_VH.img'.format(
reference.split('_')[0], reference, reference)
fpath = f'{frootpath}/{reference}_VH.img'
if object_storage.get_file(s3path, fpath) == 0:
print("Resource {} not available in S3 storage (FATAL)".format(s3path))
incurs.execute(updateSql.format('No S3 VH img', oid, 'inprogress'))
inconn.commit()
incurs.close()
inconn.close()
sys.exit(1)
s3path = s3path.replace('.img', '.hdr')
fpath = f'{frootpath}/{reference}_VH.hdr'
if object_storage.get_file(s3path, fpath) == 0:
print("Resource {} not available in S3 storage (FATAL)".format(s3path))
incurs.execute(updateSql.format('No S3 VH hdr', oid, 'inprogress'))
inconn.commit()
incurs.close()
inconn.close()
sys.exit(1)
# Open a connection to save results
outconn = psycopg2.connect(connString)
if not outconn:
print("No out connection established")
incurs.execute(updateSql.format('no_out_conn', oid, 'inprogress'))
inconn.commit()
incurs.close()
inconn.close()
sys.exit(1)
# Get the parcel polygon in this image' footprint
incurs.close()
# Open a named cursor
incurs = inconn.cursor(name='fetch_image_coverage',
cursor_factory=psycopg2.extras.DictCursor)
ds_conf = config.get_value(['set', 'ds_conf'])
ds_year = config.get_value(['set', 'ds_year'])
pid_column = config.get_value(
['ds_conf', ds_conf, 'years', ds_year, 'columns', 'parcels_id'])
parcelsql = f"""
SELECT p.{pid_column}, ST_AsGeoJSON(st_transform(p.wkb_geometry,
{outsrid}))::json
FROM {parcels_table} p, {dias_catalogue} dc
WHERE p.wkb_geometry && st_transform(dc.footprint, {srid})
And st_area(p.wkb_geometry) > 3000.0
And dc.id = {oid}
-- And p.{pid_column} not in (SELECT distinct pid
-- FROM {results_table} where obsid = {oid})
"""
incurs.execute(parcelsql)
sqlload = time.time() - start
print(
"Images loaded and nrecs[0] features selected from database in {} seconds"
.format(sqlload))
nrows = {}
nrows['VV'] = 0
nrows['VH'] = 0
affine = {}
array = {}
bands = ['VV', 'VH']
for b in bands:
with rasterio.open(f'{frootpath}/{reference}_{b}.img') as src:
affine[b] = src.transform
array[b] = src.read(1)
while True:
rowset = incurs.fetchmany(size=2000)
if not rowset:
break
features = {
"type":
"FeatureCollection",
"features": [{
"type": "feature",
"geometry": f[1],
"properties": {
"pid": int(f[0])
}
} for f in rowset]
}
for b in bands:
zs = zonal_stats(features,
array[b],
affine=affine[b],
stats=[
"count", "mean", "std", "min", "max",
"percentile_25", "percentile_50",
"percentile_75"
],
prefix="",
nodata=0,
geojson_out=True)
df = pd.DataFrame(zs)
df = pd.DataFrame.from_dict(df.properties.to_dict(),
orient='index')
df['obsid'] = oid
df['band'] = b
df.rename(index=str,
columns={
"percentile_25": "p25",
"percentile_50": "p50",
"percentile_75": "p75"
},
inplace=True)
nrows[b] = nrows[b] + len(df)
# df is the dataframe
if len(df) > 0:
df.dropna(inplace=True)
if len(df.values) > 0:
df_columns = list(df)
s_buf = io.StringIO()
df.to_csv(s_buf, header=False, index=False, sep=',')
s_buf.seek(0)
outcurs = outconn.cursor()
#print(tuple(df_columns))
try:
#psycopg2.extras.execute_batch(outcurs, insert_stmt, df.values)
outcurs.copy_from(s_buf,
dbconfig['tables']['results_table'],
columns=tuple(df_columns),
sep=',')
outconn.commit()
except psycopg2.IntegrityError as e:
print("insert statement {} contains duplicate index".
format(insert_stmt))
#except Error as e:
# print(e)
finally:
outcurs.close()
else:
print("No valid data in block {}".format(nrows[b]))
outconn.close()
incurs.close()
incurs = inconn.cursor()
try:
incurs.execute(updateSql.format('extracted', oid, 'inprogress'))
inconn.commit()
except (Exception, psycopg2.DatabaseError) as error:
print(error)
inconn.close()
if outconn:
outconn.close()
incurs.close()
inconn.close()
fpath = f'{frootpath}/{reference}_VV.img'
if os.path.exists(fpath):
os.remove(fpath)
os.remove(fpath.replace('.img', '.hdr'))
fpath = f'{frootpath}/{reference}_VH.img'
if os.path.exists(fpath):
os.remove(fpath)
os.remove(fpath.replace('.img', '.hdr'))
print(
"Total time required for {} features and {} bands: {} seconds".format(
nrows['VV'], len(bands),
time.time() - start))
def view():
info = Label("Select a parcel to display.")
temppath = config.get_value(['paths', 'temp'])
datapath = config.get_value(['paths', 'data'])
method = ToggleButtons(options=[('From local storage', 0),
('Remote to memory', 1)],
value=0,
description='',
disabled=True,
button_style='info',
tooltips=[
'View data that are stored on the local drive.',
'View data from memory.'
])
paths = RadioButtons(options=[
(f"Temporary folder: '{temppath}'.", temppath),
(f"Personal data folder: '{datapath}'.", datapath)
],
layout={'width': 'max-content'},
value=temppath)
paths_box = Box([Label(value="Select folder:"), paths])
tables_first = [
f for f in os.listdir(paths.value)
if os.path.isdir(os.path.join(paths.value, f))
]
select_table = Dropdown(
options=[f for f in tables_first if not f.startswith('.')],
value=None,
description='Select tabe:',
disabled=False,
)
select_option = RadioButtons(options=[(f"Single parcel selection.", 1),
(f"Multiple parcels selection.", 2)],
disabled=True,
layout={'width': 'max-content'})
button_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
select_option_box = HBox([
select_table, button_refresh,
Label(value="Selection method:"), select_option
])
selection_single = Dropdown(
options=[],
value=None,
description='Select parcel:',
disabled=False,
)
selection_multi = SelectMultiple(
options=[],
value=[],
description='Select parcels:',
disabled=False,
)
view_method = ToggleButtons(
options=[],
value=None,
description='',
disabled=False,
button_style='info',
tooltips=[],
)
rm_parcel = Button(value=False,
disabled=False,
button_style='danger',
tooltip='Delete parcel data.',
icon='trash',
layout=Layout(width='35px'))
code_info = Label()
single_box = HBox([selection_single, rm_parcel])
select_box = Box([single_box])
method_0 = VBox([info, paths_box, select_option_box, select_box])
method_1 = VBox([])
view_box = Output(layout=Layout(border='1px solid black'))
method_out = Output()
with method_out:
display(method_0)
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 0:
display(method_0)
elif obj['new'] == 1:
display(method_1)
method.observe(method_options, 'value')
@button_refresh.on_click
def button_refresh_on_click(b):
view_box.clear_output()
tables_first = [
f for f in os.listdir(paths.value)
if os.path.isdir(os.path.join(paths.value, f))
]
select_table.options = [
f for f in tables_first if not f.startswith('.')
]
if select_table.value is not None:
parcels = f"{paths.value}{select_table.value}"
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
selection_multi.options = parcels_list
else:
selection_single.options = []
selection_single.value = None
selection_multi.options = []
selection_multi.value = []
@rm_parcel.on_click
def rm_parcel_on_click(b):
try:
parcel_to_rm = f"{paths.value}{select_table.value}/{selection_single.value}"
try:
shutil.rmtree(f'{parcel_to_rm}')
except Exception:
pass
try:
os.remove(f'{parcel_to_rm}')
except Exception:
pass
# print(f"The parce: '{selection_single.value}' is deleted.")
parcels = f"{paths.value}{select_table.value}"
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
view_box.clear_output()
except Exception:
pass
def on_select_option_change(change):
if select_option.value == 1:
select_box.children = [single_box]
else:
select_box.children = [selection_multi]
select_option.observe(on_select_option_change, 'value')
def on_datapath_change(change):
tables = [
f for f in os.listdir(paths.value)
if os.path.isdir(os.path.join(paths.value, f))
]
tables = [f for f in tables if not f.startswith('.')]
select_table.options = tables
paths.observe(on_datapath_change, 'value')
def on_table_change(change):
if select_table.value is not None:
parcels = f"{paths.value}{select_table.value}"
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
selection_multi.options = parcels_list
else:
selection_single.options = []
selection_single.value = None
selection_multi.options = []
selection_multi.value = []
view_method.options = []
select_table.observe(on_table_change, 'value')
def on_selection_change(obj):
code_info.value = "Select how to view the dataset."
options_list = [('Get example code', 1)]
if obj['new'] is not None:
parceldata = f"{paths.value}{select_table.value}/{selection_single.value}"
data_list = [
f for f in os.listdir(parceldata) if not f.startswith('.')
]
if any("time_series" in s for s in data_list):
options_list.append(('Plot time series', 2))
if any("chip_images" in s for s in data_list):
options_list.append(('View images', 3))
options_list.append(("Show on map", 4))
if select_option.value == 2:
options_list.append(('Comparison', 5))
view_method.options = options_list
view_method.value = None
selection_single.observe(on_selection_change, 'value')
selection_multi.observe(on_selection_change, 'value')
def method_options(obj):
view_box.clear_output()
with view_box:
if selection_single.value is None:
with view_box:
print("Please select a parcel")
elif select_option.value == 1:
data_path = f'{paths.value}{select_table.value}/{selection_single.value}/'
if obj['new'] == 1:
from src.ipycbm.ui_view import view_code
display(view_code.code(data_path))
elif obj['new'] == 2:
from src.ipycbm.ui_view import view_time_series
display(view_time_series.time_series(data_path))
elif obj['new'] == 3:
from src.ipycbm.ui_view import view_calendar
display(view_calendar.calendar(data_path))
elif obj['new'] == 4:
from src.ipycbm.ui_view import view_map
display(view_map.widget_box(data_path))
elif select_option.value == 2 and len(selection_multi.value) > 0:
data_path = f'{paths.value}{select_table.value}/'
data_paths = [
f'{data_path}{s}/' for s in selection_multi.value
]
if obj['new'] == 1:
from src.ipycbm.ui_view import view_code
display(view_code.code(data_paths[0]))
pass
elif obj['new'] == 2:
from src.ipycbm.ui_view import view_time_series
# display(view_time_series.time_series(data_list[0]))
pass
elif obj['new'] == 3:
from src.ipycbm.ui_view import view_calendar
# display(view_chip_images.calendar(data_path))
pass
elif obj['new'] == 4:
from src.ipycbm.ui_view import view_maps
display(view_maps.with_polygons(data_paths))
selection_single.observe(method_options, 'value')
selection_multi.observe(method_options, 'value')
view_method.observe(method_options, 'value')
notes_info = Label("Add a note for the parcel")
notes_bt = Button(value=False,
description='Add note',
disabled=False,
button_style='info',
tooltip='Add a note.',
icon='sticky-note')
notes_box = VBox([])
@notes_bt.on_click
def notes_bt_on_click(b):
if notes_box.children == ():
notes_box.children = [
view_notes.notes(f"{paths.value}{select_table.value}/",
select_table.value,
selection_single.value.replace('parcel_', ''))
]
else:
notes_box.children = []
wbox = VBox([
method_out, code_info, view_method, view_box,
HBox([notes_info, notes_bt]), notes_box
])
return wbox
def cbm():
# path_plug = "ipycbm/plugins/foi/foi_db_func/"
path_data = f"{config.get_value(['paths', 'temp'])}cbm/"
progress = Output()
def outlog(*text):
with progress:
print(*text)
cbm_info = HTML(
value="""Main src.ipycbm Jupyter Notebooks graphical panel to run all the required procedures <br>
for Checks by Monitoring (CbM) with a sample dataset.<br>
""",
placeholder='CbM Information',
)
# Connect
db_select = Dropdown(
options=[db for db in config.get_value(['db'])],
description='Configure:',
disabled=True,
layout=Layout(width='140px')
)
db_config = Button(
value=False,
disabled=False,
button_style='info',
tooltip='Configure db connection.',
icon='cogs',
layout=Layout(width='40px')
)
db_box = HBox([db_select, db_config])
db_conf_box = HBox([])
@db_config.on_click
def db_config_on_click(b):
if db_conf_box.children == ():
db_conf_box.children = [settings.direct_conn()]
else:
db_conf_box.children = ()
accor = Accordion(children=[
VBox([db_box, db_conf_box]),
proc_func.upload_shp(path_data),
proc_func.create_tables(),
proc_card2db.dias_cat(),
get_settings.direct_settings(),
proc_func.extraction(),
VBox([get_panel.get()],
layout=Layout(border='1px solid black')),
view_panel.view()])
def on_tb_name(change):
# Get to the widget that is inside the accordion widget
# Any change of the order of the widgets within the widgets
# box will break this connection.
new_value = accor.children[1].children[3].children[0].value
accor.children[2].children[1].value = new_value
accor.children[1].children[3].children[0].observe(on_tb_name, 'value')
def on_start(change):
new_value = accor.children[3].children[2].children[0].value
accor.children[5].children[0].children[0].value = new_value
accor.children[3].children[2].children[0].observe(on_start, 'value')
def on_end(change):
new_value = accor.children[3].children[2].children[1].value
accor.children[5].children[0].children[1].value = new_value
accor.children[3].children[2].children[1].observe(on_end, 'value')
accor.set_title(0, "1. Connect to database and object storage.")
accor.set_title(
1, "2. Upload .shp, with all the required files (cpg, dbf, prj, shx).")
accor.set_title(2, "3. Create the essential CbM tables.")
accor.set_title(
3, "4. Add CARD metadata to database table 'xx_dias_catalogue'.")
accor.set_title(4, "5. Configure database tables.")
accor.set_title(5, "6. Run parcel extraction routines.")
accor.set_title(6, "7. Get parcel information, time series and images.")
accor.set_title(7, "8. View parcel data.")
wbox = VBox([cbm_info, accor])
return wbox
