def get_color_switch_widget(colors_by_labels, colors_rgb, scatter_widget):
w = None
options = []
if colors_by_labels is not None:
options = ['labels']
is_multilabels = isinstance(colors_by_labels, (list, tuple))
if is_multilabels:
options = [
'labels_' + str(i) for i in range(len(colors_by_labels))
]
if colors_rgb is not None:
options += ['rgb']
if len(options) > 1:
value = options[-1]
w = RadioButtons(options=options, description='colors', value=value)
def on_switch_colors(e):
value = e['new']
if value.startswith('labels'):
if is_multilabels:
ind = int(value.split('_')[1])
current_colors = colors_by_labels[ind]
else:
current_colors = colors_by_labels
else:
current_colors = colors_rgb
with scatter_widget.hold_trait_notifications():
scatter_widget.color = current_colors
w.observe(on_switch_colors, 'value')
return w
def _dock_add_radio_buttons(self, value, rng, callback, vertical=True,
layout=None):
# XXX: vertical=False is not supported yet
layout = self._dock_layout if layout is None else layout
widget = RadioButtons(
options=rng,
value=value,
disabled=False,
)
widget.observe(_generate_callback(callback), names='value')
self._layout_add_widget(layout, widget)
return _IpyWidgetList(widget)
def widget_box():
source = int(config.get_value(['set', 'data_source']))
sources = RadioButtons(
options=[
("JRC RESTful API.", 0),
("Direct access to database and object storage.", 1)
],
value=source,
layout={'width': 'max-content'}
)
sources_box = Box([
Label(value="Data sources:"),
sources]
)
info_api = Label("RESTful API Settings.")
info_direct = Label("Direct access settings")
view_options = VBox([info_direct])
if source == 0:
view_options.children = [info_api, rest_api()]
elif source == 1:
view_options.children = [info_direct, direct()]
def on_source_change(change):
view_options.children = []
if sources.value == 0:
view_options.children = [info_api, rest_api()]
elif sources.value == 1:
view_options.children = [info_direct, direct()]
config.update(['set', 'data_source'], str(sources.value))
sources.observe(on_source_change, 'value')
wbox_sources = VBox([sources_box, view_options],
layout=Layout(border='1px solid black'))
info_general = Label(value="General settings:")
wbox = VBox([wbox_sources, info_general, settings.widget_box()])
return wbox
def widget_box():
source = config.get_value(['set', 'data_source'])
sources = RadioButtons(
options=[
("DIAS API.", 'dias_api'),
("Direct access to database and object storage.", 'direct')
],
layout={'width': 'max-content'}
)
sources_box = Box([
Label(value="Available sources:"),
sources]
)
info_api = Label("DIAS API Settings.")
info_direct = Label("Direct access settings")
view_options = VBox([info_direct])
if source == 'direct':
view_options.children = [info_direct]
else:
view_options.children = [info_api, dias_api()]
def on_source_change(change):
view_options.children = []
if sources.value == 'dias_api':
view_options.children = [info_api, dias_api()]
elif sources.value == 'direct':
view_options.children = [info_direct]
config.update(['preferences', 'data_source'], str(sources.value))
sources.observe(on_source_change, 'value')
wbox_sources = VBox([sources_box, view_options],
layout=Layout(border='1px solid black'))
info_general = Label(value="General settings:")
wbox = VBox([wbox_sources, info_general, settings.widget_box()])
return wbox
def data_source():
source = config.get_value(['set', 'data_source'])
sources = RadioButtons(options=[
("RESTful API for CbM.", 'api'),
("Direct access to database and object storage.", 'direct')
],
value=source,
layout={'width': 'max-content'},
disabled=True)
sources_box = Box([Label(value="Data sources:"), sources])
info_api = Label("RESTful API Settings.")
info_direct = Label("Direct access settings")
view_options = VBox([info_direct])
if source == 'api':
view_options.children = [info_api, settings_ds.api()]
elif source == 'direct':
view_options.children = [info_direct, settings_ds.direct()]
def on_source_change(change):
view_options.children = []
if sources.value == 'api':
view_options.children = [info_api, settings_ds.api()]
elif sources.value == 'direct':
view_options.children = [info_direct, settings_ds.direct()]
config.set_value(['set', 'data_source'], sources.value)
sources.observe(on_source_change, 'value')
wbox_sources = VBox([sources_box, view_options],
layout=Layout(border='1px solid black'))
return wbox_sources
def view():
info = Label("Select a parcel to display.")
temppath = config.get_value(['paths', 'temp'])
datapath = config.get_value(['paths', 'data'])
method = ToggleButtons(options=[('From local storage', 0),
('Remote to memory', 1)],
value=0,
description='',
disabled=True,
button_style='info',
tooltips=[
'View data that are stored on the local drive.',
'View data from memory.'
])
paths = RadioButtons(options=[
(f"Temporary folder: '{temppath}'.", temppath),
(f"Personal data folder: '{datapath}'.", datapath)
],
layout={'width': 'max-content'},
value=temppath)
paths_box = Box([Label(value="Select folder:"), paths])
tables_first = [
f for f in os.listdir(paths.value)
if os.path.isdir(os.path.join(paths.value, f))
]
select_table = Dropdown(
options=[f for f in tables_first if not f.startswith('.')],
value=None,
description='Select tabe:',
disabled=False,
)
select_option = RadioButtons(options=[(f"Single parcel selection.", 1),
(f"Multiple parcels selection.", 2)],
disabled=True,
layout={'width': 'max-content'})
button_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
select_option_box = HBox([
select_table, button_refresh,
Label(value="Selection method:"), select_option
])
selection_single = Dropdown(
options=[],
value=None,
description='Select parcel:',
disabled=False,
)
selection_multi = SelectMultiple(
options=[],
value=[],
description='Select parcels:',
disabled=False,
)
view_method = ToggleButtons(
options=[],
value=None,
description='',
disabled=False,
button_style='info',
tooltips=[],
)
rm_parcel = Button(value=False,
disabled=False,
button_style='danger',
tooltip='Delete parcel data.',
icon='trash',
layout=Layout(width='35px'))
code_info = Label()
single_box = HBox([selection_single, rm_parcel])
select_box = Box([single_box])
method_0 = VBox([info, paths_box, select_option_box, select_box])
method_1 = VBox([])
view_box = Output(layout=Layout(border='1px solid black'))
method_out = Output()
with method_out:
display(method_0)
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 0:
display(method_0)
elif obj['new'] == 1:
display(method_1)
method.observe(method_options, 'value')
@button_refresh.on_click
def button_refresh_on_click(b):
view_box.clear_output()
tables_first = [
f for f in os.listdir(paths.value)
if os.path.isdir(os.path.join(paths.value, f))
]
select_table.options = [
f for f in tables_first if not f.startswith('.')
]
if select_table.value is not None:
parcels = f"{paths.value}{select_table.value}"
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
selection_multi.options = parcels_list
else:
selection_single.options = []
selection_single.value = None
selection_multi.options = []
selection_multi.value = []
@rm_parcel.on_click
def rm_parcel_on_click(b):
try:
parcel_to_rm = f"{paths.value}{select_table.value}/{selection_single.value}"
try:
shutil.rmtree(f'{parcel_to_rm}')
except Exception:
pass
try:
os.remove(f'{parcel_to_rm}')
except Exception:
pass
# print(f"The parce: '{selection_single.value}' is deleted.")
parcels = f"{paths.value}{select_table.value}"
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
view_box.clear_output()
except Exception:
pass
def on_select_option_change(change):
if select_option.value == 1:
select_box.children = [single_box]
else:
select_box.children = [selection_multi]
select_option.observe(on_select_option_change, 'value')
def on_datapath_change(change):
tables = [
f for f in os.listdir(paths.value)
if os.path.isdir(os.path.join(paths.value, f))
]
tables = [f for f in tables if not f.startswith('.')]
select_table.options = tables
paths.observe(on_datapath_change, 'value')
def on_table_change(change):
if select_table.value is not None:
parcels = f"{paths.value}{select_table.value}"
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
selection_multi.options = parcels_list
else:
selection_single.options = []
selection_single.value = None
selection_multi.options = []
selection_multi.value = []
view_method.options = []
select_table.observe(on_table_change, 'value')
def on_selection_change(obj):
code_info.value = "Select how to view the dataset."
options_list = [('Get example code', 1)]
if obj['new'] is not None:
parceldata = f"{paths.value}{select_table.value}/{selection_single.value}"
data_list = [
f for f in os.listdir(parceldata) if not f.startswith('.')
]
if any("time_series" in s for s in data_list):
options_list.append(('Plot time series', 2))
if any("chip_images" in s for s in data_list):
options_list.append(('View images', 3))
options_list.append(("Show on map", 4))
if select_option.value == 2:
options_list.append(('Comparison', 5))
view_method.options = options_list
view_method.value = None
selection_single.observe(on_selection_change, 'value')
selection_multi.observe(on_selection_change, 'value')
def method_options(obj):
view_box.clear_output()
with view_box:
if selection_single.value is None:
with view_box:
print("Please select a parcel")
elif select_option.value == 1:
data_path = f'{paths.value}{select_table.value}/{selection_single.value}/'
if obj['new'] == 1:
from src.ipycbm.ui_view import view_code
display(view_code.code(data_path))
elif obj['new'] == 2:
from src.ipycbm.ui_view import view_time_series
display(view_time_series.time_series(data_path))
elif obj['new'] == 3:
from src.ipycbm.ui_view import view_calendar
display(view_calendar.calendar(data_path))
elif obj['new'] == 4:
from src.ipycbm.ui_view import view_map
display(view_map.widget_box(data_path))
elif select_option.value == 2 and len(selection_multi.value) > 0:
data_path = f'{paths.value}{select_table.value}/'
data_paths = [
f'{data_path}{s}/' for s in selection_multi.value
]
if obj['new'] == 1:
from src.ipycbm.ui_view import view_code
display(view_code.code(data_paths[0]))
pass
elif obj['new'] == 2:
from src.ipycbm.ui_view import view_time_series
# display(view_time_series.time_series(data_list[0]))
pass
elif obj['new'] == 3:
from src.ipycbm.ui_view import view_calendar
# display(view_chip_images.calendar(data_path))
pass
elif obj['new'] == 4:
from src.ipycbm.ui_view import view_maps
display(view_maps.with_polygons(data_paths))
selection_single.observe(method_options, 'value')
selection_multi.observe(method_options, 'value')
view_method.observe(method_options, 'value')
notes_info = Label("Add a note for the parcel")
notes_bt = Button(value=False,
description='Add note',
disabled=False,
button_style='info',
tooltip='Add a note.',
icon='sticky-note')
notes_box = VBox([])
@notes_bt.on_click
def notes_bt_on_click(b):
if notes_box.children == ():
notes_box.children = [
view_notes.notes(f"{paths.value}{select_table.value}/",
select_table.value,
selection_single.value.replace('parcel_', ''))
]
else:
notes_box.children = []
wbox = VBox([
method_out, code_info, view_method, view_box,
HBox([notes_info, notes_bt]), notes_box
])
return wbox
def foi():
path_plug = "cbm/foi/foi_db_func/"
path_data = f"{config.get_value(['paths', 'temp'])}foi/"
progress = Output()
def outlog(*text):
with progress:
print(*text)
foi_info = HTML(
value="""FOI procedures need direct access to the database. <br>
In case there no image is provided, access to object storage will be needed to generate the base image from sentinel images.
""",
placeholder='FOI Information',
)
# Connect to database
db_info = Label(f"1. Connect to database and object storage.")
db_select = Dropdown(options=[db for db in config.get_value(['db'])],
description='Configure:',
disabled=True,
layout=Layout(width='140px'))
db_config = Button(value=False,
disabled=False,
button_style='info',
tooltip='Configure db connection.',
icon='cogs',
layout=Layout(width='40px'))
db_box = HBox([db_select, db_config])
parcels_table = RadioButtons(
options=[('Upload .shp', 0), ('From database', 1)],
# value='pineapple', # Defaults to 'pineapple'
# layout={'width': 'max-content'}, # If the items' names are long
# description='Pizza topping:',
disabled=False)
par_box = HBox([])
def on_parcels_table(method):
if method.new == 0:
par_box.children = []
elif method.new == 1:
par_box.children = []
parcels_table.observe(on_parcels_table, 'value')
# Generate or upload image.
img_info = Label(
f"3. Upload or generate raster base image. (Only upload is currently available)"
)
img_option = ToggleButtons(
options=['Upload', 'Generate'],
value=None,
disabled=True,
button_style='info', # 'success', 'info', 'warning', 'danger' or ''
tooltips=['Upnload your base image', 'Get from object storage'])
img_dist_folder = Text(value=f"{path_data}raster/",
placeholder='tmp/',
description='Folder:',
disabled=False)
img_select = FileUpload(
description='Select file:',
icon='plus',
accept='.tif, .tiff',
multiple=True # True to accept multiple files upload else False
)
img_clear = Button(value=False,
disabled=False,
button_style='info',
tooltip='Clear selections.',
icon='broom',
layout=Layout(width='40px'))
img_upload = Button(value=False,
disabled=False,
button_style='info',
tooltip='Upload foi base image (.tif)',
icon='fa-upload',
layout=Layout(width='40px'))
img_box = HBox(
[HBox([img_dist_folder, img_select, img_clear, img_upload])])
# YAML File upload
yml_select = FileUpload(description='Select file:',
icon='plus',
accept='.yml, .yaml, .txt',
multiple=False)
yml_clear = Button(value=False,
disabled=False,
button_style='info',
tooltip='Clear selection.',
icon='broom',
layout=Layout(width='40px'))
yml_upload = Button(value=False,
disabled=False,
button_style='info',
tooltip='Upload yaml file.',
icon='fa-upload',
layout=Layout(width='40px'))
yml_box = HBox([yml_select, yml_clear, yml_upload])
# Prepare procedures
pre_info = Label("4. Prepare FOI procedure.")
pre_ins_func = Button(value=False,
disabled=False,
button_style='info',
tooltip='Insert functions to database.',
icon='fa-share-square',
layout=Layout(width='40px'))
pre_ins_func_box = HBox(
[Label("Add functions to database:"), pre_ins_func])
vector_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}vector/*') if '.shp' in s],
description='Vector file:',
disabled=False,
)
raster_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}raster/*') if '.tif' in s],
description='Raster file:',
disabled=False,
)
yaml_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}/*') if '.yml' in s],
description='yaml file:',
disabled=False,
)
# heterogeneity_threshold
pre_heto_chec = HTML("""
Minimum and maximum thresholds for heterogeneity checks. In the example,
any parcel with percentage of pixels for one class between 30 and 70 from
the total, will be considered heterogenous.
""")
pre_min_het = IntText(value=30,
description='MIN:',
tooltip="Minimum threshold for heterogeneity checks",
disabled=False,
layout=Layout(width='150px'))
pre_max_het = IntText(value=70,
description='MAX:',
tooltip="Maximum threshold for heterogeneity checks",
disabled=False,
layout=Layout(width='150px'))
pre_heto_chec_box = HBox([pre_min_het, pre_max_het])
pre_min_area = IntText(value=2000,
description='area:',
tooltip="Minimum area for clusters selection.",
disabled=False,
layout=Layout(width='200px'))
refresh_selections = Button(layout=Layout(width='35px'), icon='fa-refresh')
pre_box = VBox([
pre_info, pre_ins_func_box,
HBox([Label("Select the requared files:"), refresh_selections]),
HTML("""a. Spatial data to be tested - parcels that will
be checked for heterogeneity and cardinality."""),
HBox([vector_file]),
HTML("""b. Thematic raster - classification raster, or raster
from other source that will be used for testing heterogeneity and cardinality."""
),
HBox([raster_file]),
HTML(
"""c. YAML file that holds the classes form the thematic raster file -
can be also a simple list of values in the notebook
corespondence between pixel values and names for the classes"""),
HBox([yaml_file, yml_box]), pre_heto_chec, pre_heto_chec_box,
HBox([
pre_min_area,
HTML(
"Minimum area for clusters selection - only clusters bigger from this threshold will be counted."
)
])
])
# Run procedures
run_info = Label("5. Run FOI procedure.")
run_proc = Button(
description='Run FOI procedure',
value=False,
disabled=False,
button_style='info',
tooltip='Run',
icon='play',
)
run_box = HBox([run_proc])
def on_img_option_change(change):
if img_option.value == 'Upload':
img_box.children = [
HBox([img_dist_folder, img_select, img_clear, img_upload])
]
else:
img_box.children = ()
img_option.observe(on_img_option_change, 'value')
@refresh_selections.on_click
def refresh_selections_on_click(b):
vector_file.options = [
s for s in glob.glob(f'{path_data}vector/*') if '.shp' in s
]
raster_file.options = [
s for s in glob.glob(f'{path_data}raster/*') if '.tif' in s
]
yaml_file.options = [
s for s in glob.glob(f'{path_data}/*') if '.yml' in s
]
@img_clear.on_click
def img_clear_on_click(b):
img_select.value.clear()
img_select._counter = 0
@img_upload.on_click
def img_upload_on_click(b):
progress.clear_output()
os.makedirs(img_dist_folder.value, exist_ok=True)
for key in img_select.value:
content = img_select.value[key]['content']
with open(f'{img_dist_folder.value}{key}', 'wb') as f:
f.write(content)
outlog("All files are uploaded.")
@yml_clear.on_click
def yml_clear_on_click(b):
yml_select.value.clear()
yml_select._counter = 0
@yml_upload.on_click
def yml_upload_on_click(b):
progress.clear_output()
yml_dist_folder = f'{path_data}'
os.makedirs(yml_dist_folder, exist_ok=True)
for key in yml_select.value:
content = yml_select.value[key]['content']
with open(f'{yml_dist_folder}{key}', 'wb') as f:
f.write(content)
outlog("The yaml file is uploaded.")
db_conf_box = HBox([])
@db_config.on_click
def db_config_on_click(b):
if db_conf_box.children == ():
db_conf_box.children = [settings.direct_conn()]
else:
db_conf_box.children = ()
@pre_ins_func.on_click
def pre_ins_func_on_click(b):
progress.clear_output()
try:
functions = glob.glob(f"{path_plug}*.func")
db = config.get_value(['set', 'db_conn'])
sche = config.get_value(['db', db, 'conn', 'sche'])
user = config.get_value(['db', db, 'conn', 'user'])
for f in functions:
database.insert_function(
open(f).read().format(schema=sche, owner=user))
finc_list = [
f"ipycbm_{f.split('/')[-1].split('.')[0]}, " for f in functions
]
outlog(
f"The functions: {('').join(finc_list)}where added to the database"
)
except Exception as err:
outlog("Could not add functions to dattabase.", err)
@run_proc.on_click
def run_proc_on_click(b):
with progress:
foi_proc.proc(vector_file.value, raster_file.value,
yaml_file.value, pre_min_het.value,
pre_max_het.value, pre_min_area.value)
wbox = VBox([
foi_info, db_info, db_box, db_conf_box,
proc_func.upload_shp(path_data), img_info, img_option, img_box,
pre_box, run_info, run_box, progress
])
return wbox
def dias_cat():
aoi_method = RadioButtons(
# options=[('Predefined MS polygons', 1), ('Get polygon from dataset extent', 2), ('Draw polygon on a map', 3)],
options=[('Predefined MS polygons', 1),
('Get polygon from dataset extent', 2)],
value=2,
description='AOI:',
disabled=False,
)
mss = data_options.ms_polygons()
ms = Dropdown(
options=[(m, mss[m]) for m in mss],
tooltip="AOI",
description='AOI:',
)
tb = Dropdown(
options=database.tables(),
tooltip="Select table",
description='Table:',
)
tb_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
aoi_box = VBox([HBox([tb, tb_refresh])])
dias = Dropdown(
options=data_options.dias_providers(),
value='EOSC',
description='DIAS Provider:',
disabled=True,
)
start = DatePicker(description='Start date', disabled=False)
end = DatePicker(description='End date', disabled=False)
plevel = RadioButtons(
options=['LEVEL2A', 'LEVEL2AP'],
value='LEVEL2A',
description='pLevel:',
disabled=False,
)
ptype = RadioButtons(
options=['CARD-COH6', 'CARD-BS'],
description='pType:',
disabled=False,
)
card_options = VBox([plevel])
card = RadioButtons(
options=[('Sentinel 1', 's1'), ('Sentinel 2', 's2')],
value='s2',
description='card:',
disabled=False,
)
bt_card2db = Button(description='Add CARD to db',
value=False,
disabled=False,
button_style='info',
tooltip='Add CARD catalogue to database',
icon='database')
progress = Output()
def outlog(*text):
with progress:
print(*text)
def on_aoi_method_change(change):
if aoi_method.value == 1:
aoi_box.children = [ms]
elif aoi_method.value == 2:
aoi_box.children = [HBox([tb, tb_refresh])]
elif aoi_method.value == 3:
aoi_box.children = []
aoi_method.observe(on_aoi_method_change, 'value')
def on_card_change(change):
if card.value == 's2':
card_options.children = [plevel]
else:
card_options.children = [ptype]
card.observe(on_card_change, 'value')
@tb_refresh.on_click
def tb_refresh_on_click(b):
tb.options = database.tables()
@bt_card2db.on_click
def bt_card2db_on_click(b):
progress.clear_output()
try:
with open(f"{config.get_value(['paths','temp'])}tb_prefix",
'r') as f:
tb_prefix = f.read()
except Exception:
tb_prefix = ''
dc_table = f'{tb_prefix}_dias_catalogue'
if database.tb_exist(dc_table) is True:
if aoi_method.value == 1:
polygon = ms.value.replace(' ', '+')
elif aoi_method.value == 2:
polygon = database.tb_extent(tb.value)
elif aoi_method.value == 3:
polygon = ms.value.replace(' ', '+')
# print(polygon)
if card.value == 's2':
option = plevel.value
else:
option = ptype.value
outlog("Inserting CARD catalogue to database ...")
with progress:
meta2DB.dias_cat(tb_prefix, f"POLYGON(({polygon}))",
start.value, end.value, card.value, option)
outlog("Completed.")
else:
outlog(f"Table {dc_table} does not exist.")
wbox = VBox([
dias,
HBox([aoi_method, aoi_box]),
HBox([start, end]),
HBox([card, card_options]), bt_card2db, progress
])
return wbox
class parcel_data_source_widget(VBox):
def __init__(self):
# Radio buttons allowing one to select the type of parcel data source
# to use
self.sources = RadioButtons(options=[("Text File", 'txt'),
("Text File and RESTful API",
'txt_rest'),
("From SHAPE File", 'shape'),
("From GeoJSON File", 'json')],
value='txt',
layout={'width': 'max-content'})
# Cosmetic element, it will be place close to the radio buttons
self.sources_box = Box(
[Label(value="Parcel Data sources:"), self.sources])
info_txt = Label("From Text File only")
info_txt_rest = Label("From Text File and RESTFul")
info_shape = Label("From Shape File")
info_json = Label("From GeoJSON File")
# default option
view_options = VBox([info_txt, source_from_txt()])
def on_source_change(change):
view_options.children = []
if self.sources.value == 'txt':
view_options.children = [info_txt, source_from_txt()]
elif self.sources.value == 'txt_rest':
view_options.children = [info_txt_rest, source_from_txt_rest()]
elif self.sources.value == 'shape':
view_options.children = [info_shape, source_from_shape()]
elif self.sources.value == 'json':
view_options.children = [info_json, source_from_shape('json')]
self.sources.observe(on_source_change, 'value')
self.wb_save = Button(description='Save', disabled=False, icon='save')
@self.wb_save.on_click
def wb_save_on_click(b):
# Create the output element in the JSON file
_dict = self.dump()
key = list(_dict.keys())[0]
value = _dict[key]
config.set_value(key, value)
self.wb_load = Button(description='Load', disabled=False, icon='load')
wfc_options = FileChooser(placeholder='Option file',
description='Option file:',
disabled=False)
self.whb_load = HBox([self.wb_load, wfc_options])
@self.wb_load.on_click
def wb_load_on_click(b):
if wfc_options.selected is not None:
optionFile = open(wfc_options.selected)
options = json.load(optionFile)
self.initialize(options)
super().__init__(
[self.sources_box, view_options, self.wb_save, self.whb_load],
layout=Layout(border='1px solid black'))
def dump(self) -> dict:
_dict = self.children[1].children[1].dump()
out_dict = {"parcelSource": _dict}
return out_dict
def initialize(self, options: dict):
if "parcelSource" not in options:
return
# keep only the relevant information
ps_opt = options["parcelSource"]
if "type" not in ps_opt:
return
source_opt = [a[1] for a in self.sources.options]
if ps_opt["type"] in source_opt:
self.sources.value = ps_opt["type"]
view_options = VBox()
info_txt = Label("From Text File only")
info_txt_rest = Label("From Text File and RESTFul")
info_shape = Label("From Shape File")
info_json = Label("From GeoJSON File")
if self.sources.value == 'txt':
source = source_from_txt()
source.initialize(ps_opt)
view_options.children = [info_txt, source]
elif self.sources.value == 'txt_rest':
source = source_from_txt_rest()
source.initialize(ps_opt)
view_options.children = [info_txt_rest, source]
elif self.sources.value == 'shape':
source = source_from_shape()
source.initialize(ps_opt)
view_options.children = [info_shape, source]
elif self.sources.value == 'json':
source = source_from_shape('json')
source.initialize(ps_opt)
view_options.children = [info_json, source]
self.children = [
self.sources_box, view_options, self.wb_save, self.whb_load
]
def on_source_change(change):
view_options.children = []
if self.sources.value == 'txt':
view_options.children = [info_txt, source_from_txt()]
elif self.sources.value == 'txt_rest':
view_options.children = [info_txt_rest, source_from_txt_rest()]
elif self.sources.value == 'shape':
view_options.children = [info_shape, source_from_shape()]
elif self.sources.value == 'json':
view_options.children = [info_json, source_from_shape('json')]
self.sources.observe(on_source_change, 'value')
def get():
"""Get the parcel's dataset for the given location or ids"""
debug = False
info = Label("1. Select the aoi to get parcel data.")
values = config.read()
ppoly_out = Output()
progress = Output()
def outlog(*text):
with progress:
print(*text)
def outlog_poly(*text):
with ppoly_out:
print(*text)
def aois_options():
values = config.read()
options = {}
if values['set']['data_source'] == 'api':
api_values = config.read('api_options.json')
for aoi in api_values['aois']:
options[(aoi.upper(), aoi)] = api_values['aois'][aoi]['years']
elif values['set']['data_source'] == 'direct':
values = config.read('api_options.json')
for aoi in values['dataset']:
options[(f"{aoi.upper()} ({aoi})", aoi)] = [aoi.split('_')[-1]]
return options
def aois_years():
values = config.read()
years = {}
if values['set']['data_source'] == 'api':
api_values = config.read('api_options.json')
for aoi in api_values['aois']:
years[aoi] = api_values['aois'][aoi]['years']
elif values['set']['data_source'] == 'direct':
values = config.read()
for aoi in values['dataset']:
years[aoi] = [aoi.split('_')[-1]]
return years
try:
aois = Dropdown(
options=tuple(aois_options()),
value=values['set']['dataset'],
description='AOI:',
)
except Exception:
aois = Dropdown(
options=tuple(aois_options()),
description='AOI:',
)
def years_disabled():
values = config.read()
if values['set']['data_source'] == 'direct':
return True
else:
return False
year = Dropdown(
options=next(iter(aois_options().values())),
description='Year:',
disabled=years_disabled(),
)
button_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
@button_refresh.on_click
def button_refresh_on_click(b):
values = config.read()
if values['set']['data_source'] == 'api':
from cbm.datas import api
available_options = json.loads(api.get_options())
try:
api_options = normpath(
join(config.path_conf, 'api_options.json'))
os.makedirs(dirname(api_options), exist_ok=True)
with open(api_options, "w") as f:
json.dump(available_options, f, indent=4)
outlog(f"File saved at: {api_options}")
except Exception as err:
outlog(f"Could not create the file 'api_options.json': {err}")
outlog(f"The API options are updated.")
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
year.disabled = years_disabled()
def table_options_change(change):
api_values = config.read('api_options.json')
id_examples = api_values['aois'][change.new]['id_examples']
try:
id_examples_label.value = ', '.join(str(x) for x in id_examples)
year.options = aois_years()[change.new]
year.disabled = years_disabled()
pid.value = str(id_examples[0])
except Exception:
id_examples_label.value = ', '.join(str(x) for x in id_examples)
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
year.disabled = years_disabled()
pid.value = str(id_examples[0])
aois.observe(table_options_change, 'value')
info_method = Label("2. Select a method to download parcel data.")
method = ToggleButtons(
options=[('Parcel ID', 2), ('Coordinates', 1), ('Map marker', 3),
('Polygon', 4)],
value=None,
description='',
disabled=False,
button_style='info',
tooltips=[
'Enter lon lat', 'Enter parcel ID', 'Select a point on a map',
'Get parcels id in a polygon'
],
)
plon = Text(value='5.664', placeholder='Add lon', description='Lon:')
plat = Text(value='52.694', placeholder='Add lat', description='Lat:')
wbox_lat_lot = VBox(children=[plat, plon])
api_values = config.read('api_options.json')
id_examples = api_values['aois'][aois.value]['id_examples']
id_examples_label = Label(', '.join(str(x) for x in id_examples))
info_pid = HBox(
[Label("Multiple parcel ids can be added, e.g.: "), id_examples_label])
pid = Textarea(
value=str(id_examples[0]),
placeholder='12345, 67890',
description='Parcel(s) ID:',
)
wbox_pids = VBox(children=[info_pid, pid])
bt_get_ids = Button(description="Find parcels",
disabled=False,
button_style='info',
tooltip='Find parcels within the polygon.',
icon='')
get_ids_box = HBox(
[bt_get_ids,
Label("Find the parcels that are in the polygon.")])
@bt_get_ids.on_click
def bt_get_ids_on_click(b):
with ppoly_out:
try:
# get_requests = data_source()
ppoly_out.clear_output()
polygon = get_maps.polygon_map.feature_collection['features'][
-1]['geometry']['coordinates'][0]
polygon_str = '-'.join(
['_'.join(map(str, c)) for c in polygon])
outlog_poly(f"Geting parcel ids within the polygon...")
polyids = parcel_info.by_polygon(aois.value, year.value,
polygon_str, ptype.value,
False, True)
outlog_poly(
f"'{len(polyids['ogc_fid'])}' parcels where found:")
outlog_poly(polyids['ogc_fid'])
file = normpath(
join(config.get_value(['paths', 'temp']),
'pids_from_polygon.txt'))
with open(file, "w") as text_file:
text_file.write('\n'.join(map(str, polyids['ogc_fid'])))
except Exception as err:
outlog("No parcel ids found:", err)
method_out = Output(layout=Layout(border='1px solid black'))
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 1:
display(wbox_lat_lot)
elif obj['new'] == 2:
display(wbox_pids)
elif obj['new'] == 3:
display(
get_maps.base_map(aois.value,
config.get_value(['set',
'data_source'])))
elif obj['new'] == 4:
display(
VBox([
get_maps.polygon(
aois.value,
config.get_value(['set', 'data_source'])),
get_ids_box, ppoly_out
]))
method.observe(method_options, 'value')
info_type = Label("3. Select datasets to download.")
ptype = Text(value=None,
placeholder='(Optional) Parcel Type',
description='pType:',
disabled=False)
table_options = HBox([aois, button_refresh, ptype, year])
# ########### Time series options #########################################
pts_bt = ToggleButton(
value=False,
description='Time series',
button_style='success', # success
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pts_bands = data_options.pts_bands()
pts_tstype = SelectMultiple(
options=[("Sentinel-2 Level 2A", 's2'),
("S1 Backscattering Coefficients", 'bs'),
("S1 6-day Coherence (20m)", 'c6')],
value=['s2'],
rows=3,
description='TS type:',
disabled=False,
)
pts_band = Dropdown(
options=list(pts_bands['s2']),
value='',
description='Band:',
disabled=False,
)
def pts_tstype_change(change):
if len(pts_tstype.value) <= 1:
pts_band.disabled = False
try:
pts_b = change.new[0]
pts_band.options = pts_bands[pts_b]
except Exception:
pass
else:
pts_band.value = ''
pts_band.disabled = True
pts_tstype.observe(pts_tstype_change, 'value')
pts_options = VBox(children=[pts_tstype, pts_band])
# ########### Chip images options #########################################
pci_bt = ToggleButton(value=False,
description='Chip images',
disabled=False,
button_style='success',
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pci_start_date = DatePicker(
value=datetime.date(2020, 6, 1),
description='Start Date',
)
pci_end_date = DatePicker(
value=datetime.date(2020, 6, 30),
description='End Date',
)
pci_plevel = RadioButtons(
options=['LEVEL2A', 'LEVEL1C'],
value='LEVEL2A',
description='Proces. level:', # Processing level
disabled=False,
layout=Layout(width='50%'))
pci_chipsize = IntSlider(value=640,
min=100,
max=5120,
step=10,
description='Chip size:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='d')
pci_bands = data_options.pci_bands()
pci_satellite = RadioButtons(options=list(pci_bands),
value='Sentinel 2',
disabled=True,
layout=Layout(width='100px'))
pci_band = SelectMultiple(options=list(pci_bands['Sentinel 2']),
value=['B04'],
rows=11,
description='Band:',
disabled=False)
sats_plevel = HBox([pci_satellite, pci_plevel])
def on_sat_change(change):
sat = change.new
pci_band.options = pci_bands[sat]
pci_satellite.observe(on_sat_change, 'value')
pci_options = VBox(children=[
pci_start_date, pci_end_date, sats_plevel, pci_chipsize, pci_band
])
# ########### General options #############################################
pts_wbox = VBox(children=[])
pci_wbox = VBox(children=[])
def pts_observe(button):
if button['new']:
pts_bt.icon = 'toggle-on'
pts_wbox.children = [pts_options]
else:
pts_bt.icon = 'toggle-off'
pts_wbox.children = []
def pci_observe(button):
if button['new']:
pci_bt.icon = 'toggle-on'
pci_wbox.children = [pci_options]
else:
pci_bt.icon = 'toggle-off'
pci_wbox.children = []
pts_bt.observe(pts_observe, names='value')
pci_bt.observe(pci_observe, names='value')
pts = VBox(children=[pts_bt, pts_wbox], layout=Layout(width='40%'))
pci = VBox(children=[pci_bt, pci_wbox], layout=Layout(width='40%'))
data_types = HBox(children=[pts, pci])
info_get = Label("4. Download the selected data.")
bt_get = Button(description='Download',
button_style='warning',
tooltip='Send the request',
icon='download')
path_temp = config.get_value(['paths', 'temp'])
path_data = config.get_value(['paths', 'data'])
info_paths = HTML("".join([
"<style>div.c {line-height: 1.1;}</style>",
"<div class='c';>By default data will be stored in the temp folder ",
f"({path_temp}), you will be asked to empty the temp folder each time ",
"you start the notebook.<br>In your personal data folder ",
f"({path_data}) you can permanently store the data.</div>"
]))
paths = RadioButtons(options=[
(f"Temporary folder: '{path_temp}'.", path_temp),
(f"Personal data folder: '{path_data}'.", path_data)
],
layout={'width': 'max-content'},
value=path_temp)
paths_box = Box([Label(value="Select folder:"), paths])
def file_len(fname):
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
def get_data(parcel):
get_requests = data_source()
pid = str(parcel['pid'][0])
source = config.get_value(['set', 'data_source'])
if source == 'api':
datapath = normpath(join(paths.value, aois.value, year.value, pid))
elif source == 'direct':
dataset = config.get_value(['set', 'dataset'])
datapath = normpath(join(paths.value, dataset, pid))
file_pinf = normpath(join(datapath, 'info.json'))
os.makedirs(dirname(file_pinf), exist_ok=True)
with open(file_pinf, "w") as f:
json.dump(parcel, f)
outlog(f"File saved at: {file_pinf}")
if pts_bt.value is True:
outlog(f"Getting time series for parcel: '{pid}',",
f"({pts_tstype.value} {pts_band.value}).")
for pts in pts_tstype.value:
ts = time_series.by_pid(aois.value, year.value, pid, pts,
ptype.value, pts_band.value)
band = ''
if pts_band.value != '':
band = f"_{pts_band.value}"
file_ts = normpath(
join(datapath, f'time_series_{pts}{band}.csv'))
if isinstance(ts, pd.DataFrame):
ts.to_csv(file_ts, index=True, header=True)
elif isinstance(ts, dict):
os.makedirs(os.path.dirname(file_ts), exist_ok=True)
df = pd.DataFrame.from_dict(ts, orient='columns')
df.to_csv(file_ts, index=True, header=True)
outlog("TS Files are saved.")
if pci_bt.value is True:
files_pci = normpath(join(datapath, 'chip_images'))
outlog(f"Getting '{pci_band.value}' chip images for parcel: {pid}")
with progress:
get_requests.rcbl(parcel, pci_start_date.value,
pci_end_date.value, pci_band.value,
pci_chipsize.value, files_pci)
filet = normpath(
join(datapath, 'chip_images',
f'images_list.{pci_band.value[0]}.csv'))
if file_len(filet) > 1:
outlog(
f"Completed, all GeoTIFFs for bands '{pci_band.value}' are ",
f"downloaded in the folder: '{datapath}/chip_images'")
else:
outlog(
"No files where downloaded, please check your configurations"
)
def get_from_location(lon, lat):
outlog(f"Finding parcel information for coordinates: {lon}, {lat}")
parcel = parcel_info.by_location(aois.value, year.value, lon, lat,
ptype.value, True, False, debug)
pid = str(parcel['pid'][0])
outlog(f"The parcel '{pid}' was found at this location.")
try:
get_data(parcel)
except Exception as err:
print(err)
def get_from_id(pids):
outlog(f"Getting parcels information for: '{pids}'")
for pid in pids:
try:
parcel = parcel_info.by_pid(aois.value, year.value, pid,
ptype.value, True, False, debug)
get_data(parcel)
except Exception as err:
print(err)
@bt_get.on_click
def bt_get_on_click(b):
progress.clear_output()
if method.value == 1:
try:
with progress:
lon, lat = plon.value, plat.value
get_from_location(lon, lat)
except Exception as err:
outlog("Could not get parcel information for location",
f"'{lon}', '{lat}': {err}")
elif method.value == 2:
try:
with progress:
pids = pid.value.replace(" ", "").split(",")
get_from_id(pids)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 3:
try:
marker = get_maps.base_map.map_marker
lon = str(round(marker.location[1], 2))
lat = str(round(marker.location[0], 2))
get_from_location(lon, lat)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 4:
try:
plimit = int(values['set']['plimit'])
file = normpath(
join(config.get_value(['paths', 'temp']),
'pids_from_polygon.txt'))
with open(file, "r") as text_file:
pids = text_file.read().split('\n')
outlog("Geting data form the parcels:")
outlog(pids)
if len(pids) <= plimit:
get_from_id(pids)
else:
outlog(
"You exceeded the maximum amount of selected parcels ",
f"({plimit}) to get data. Please select smaller area.")
except Exception as err:
outlog("No pids file found.", err)
else:
outlog(f"Please select method to get parcel information.")
return VBox([
info, table_options, info_method, method, method_out, info_type,
data_types, info_get, info_paths, paths_box, bt_get, progress
])
def view():
temppath = config.get_value(['paths', 'temp'])
datapath = config.get_value(['paths', 'data'])
paths = RadioButtons(options=[
(f"Temporary folder: '{temppath}'.", temppath),
(f"Personal data folder: '{datapath}'.", datapath)
],
layout={'width': 'max-content'},
value=temppath)
paths_box = Box([Label(value="Select folder:"), paths])
tables_first = [
f for f in os.listdir(paths.value)
if isdir(normpath(join(paths.value, f)))
]
select_table = Dropdown(
options=[f for f in tables_first if not f.startswith('.')],
value=None,
description='Select tabe:',
disabled=False,
)
button_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
select_option_box = HBox([select_table, button_refresh])
selection_single = Dropdown(
options=[],
value=None,
description='Select parcel:',
disabled=False,
)
view_source = ToggleButtons(
options=[('From local folder', 0), ('Download new data', 1)],
value=0,
description='',
disabled=False,
button_style='success',
tooltips=[],
)
view_method = ToggleButtons(
options=[],
value=None,
description='',
disabled=False,
button_style='info',
tooltips=[],
)
rm_parcel = Button(value=False,
disabled=False,
button_style='danger',
tooltip='Delete parcel data.',
icon='trash',
layout=Layout(width='35px'))
source_box = VBox([])
def on_source_change(obj):
if view_source.value == 1:
source_box.children = [get_panel.get()]
else:
source_box.children = []
view_source.observe(on_source_change, 'value')
code_info = Label()
single_box = HBox([selection_single, rm_parcel])
select_box = Box([single_box])
selection = VBox([
Label("Select a data source."), view_source, source_box, paths_box,
Label("Select a parcel to display."), select_option_box, select_box
])
view_box = Output(layout=Layout(border='1px solid black'))
@button_refresh.on_click
def button_refresh_on_click(b):
view_box.clear_output()
tables_first = [
f for f in os.listdir(paths.value)
if isdir(normpath(join(paths.value, f)))
]
select_table.options = [
f for f in tables_first if not f.startswith('.')
]
if select_table.value is not None:
parcels = normpath(join(paths.value, select_table.value))
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
else:
selection_single.options = []
selection_single.value = None
@rm_parcel.on_click
def rm_parcel_on_click(b):
try:
parcel_to_rm = normpath(
join(paths.value, select_table.value, selection_single.value))
try:
shutil.rmtree(f'{parcel_to_rm}')
except Exception:
pass
try:
os.remove(f'{parcel_to_rm}')
except Exception:
pass
# print(f"The parce: '{selection_single.value}' is deleted.")
parcels = normpath(join(paths.value, select_table.value))
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
view_box.clear_output()
except Exception:
pass
def on_datapath_change(change):
tables = [
f for f in os.listdir(paths.value)
if isdir(normpath(join(paths.value, f)))
]
tables = [f for f in tables if not f.startswith('.')]
select_table.options = tables
paths.observe(on_datapath_change, 'value')
def on_table_change(change):
if select_table.value is not None:
parcels = normpath(join(paths.value, select_table.value))
parcels_list = [
f for f in os.listdir(parcels) if not f.startswith('.')
]
selection_single.options = parcels_list
else:
selection_single.options = []
selection_single.value = None
view_method.options = []
select_table.observe(on_table_change, 'value')
def on_selection_change(obj):
code_info.value = "Select how to view the dataset."
options_list = [('Get example code', 1)]
if obj['new'] is not None:
parceldata = normpath(
join(paths.value, select_table.value, selection_single.value))
data_list = [
f for f in os.listdir(parceldata) if not f.startswith('.')
]
if any("time_series" in s for s in data_list):
options_list.append(('Plot time series', 2))
if any("chip_images" in s for s in data_list):
options_list.append(('View images', 3))
options_list.append(("Show on map", 4))
view_method.options = options_list
view_method.value = None
selection_single.observe(on_selection_change, 'value')
def method_options(obj):
view_box.clear_output()
data_path = normpath(
join(paths.value, select_table.value, selection_single.value))
with view_box:
if obj['new'] == 1:
display(view_code.code(data_path))
elif obj['new'] == 2:
display(
view_time_series.time_series_widget(
select_table.value, selection_single.value))
elif obj['new'] == 3:
display(view_grid.imgs_grid(data_path))
elif obj['new'] == 4:
display(view_map.widget_box(data_path))
selection_single.observe(method_options, 'value')
view_method.observe(method_options, 'value')
notes_info = Label("Add a note for the parcel")
notes_bt = Button(value=False,
description='Add note',
disabled=False,
button_style='info',
tooltip='Add a note.',
icon='sticky-note')
notes_box = VBox([])
@notes_bt.on_click
def notes_bt_on_click(b):
if notes_box.children == ():
notes_box.children = [
view_notes.notes(
normpath(join(paths.value, select_table.value)),
select_table.value, selection_single.value)
]
else:
notes_box.children = []
wbox = VBox([
selection, code_info, view_method, view_box,
HBox([notes_info, notes_bt]), notes_box
])
return wbox
def init_strip_eq_radio(on_value_change_eq_radio) -> Widget:
eq_radio = RadioButtons(options=['>', '=', '<'], value='=')
eq_radio.observe(on_value_change_eq_radio, names='value')
return eq_radio
class DatasetAnnotatorClassification:
supported_types = [
TaskType.CLASSIFICATION_BINARY, TaskType.CLASSIFICATION_SINGLE_LABEL,
TaskType.CLASSIFICATION_MULTI_LABEL
]
def __init__(self,
task_type,
observations,
output_path,
name,
classes,
show_name=True,
show_axis=False,
fig_size=(10, 10),
buttons_vertical=False,
custom_display_function=None,
is_image=True):
if task_type not in self.supported_types:
raise Exception(labels_str.warn_task_not_supported)
if len(observations) == 0:
raise Exception(labels_str.warn_no_images)
num_classes = len(classes)
if num_classes <= 1:
raise Exception(labels_str.warn_little_classes)
elif len(
classes
) > 2 and task_type.value == TaskType.CLASSIFICATION_BINARY.value:
raise Exception(labels_str.warn_binary_only_two)
self.is_image = is_image
self.key = "path" if self.is_image else "observation"
if not self.is_image and custom_display_function is None:
raise Exception(labels_str.warn_display_function_needed)
self.task_type = task_type
self.show_axis = show_axis
self.name = name
self.show_name = show_name
self.output_path = output_path
self.file_path = os.path.join(self.output_path, self.name + ".json")
print(labels_str.info_ds_output + self.file_path)
self.mapping, self.dataset = self.__create_results_dict(
self.file_path, classes)
self.classes = list(self.mapping["categories_id"].values())
if len(
self.classes
) > 2 and task_type.value == TaskType.CLASSIFICATION_BINARY.value:
raise Exception(labels_str.warn_binary_only_two +
" ".join(self.classes))
self.observations = observations
self.max_pos = len(self.observations) - 1
self.pos = 0
self.fig_size = fig_size
self.buttons_vertical = buttons_vertical
if custom_display_function is None:
self.image_display_function = self.__show_image
else:
self.image_display_function = custom_display_function
self.previous_button = self.__create_button(labels_str.str_btn_prev,
(self.pos == 0),
self.__on_previous_clicked)
self.next_button = self.__create_button(labels_str.str_btn_next,
(self.pos == self.max_pos),
self.__on_next_clicked)
self.save_button = self.__create_button(labels_str.str_btn_download,
False, self.__on_save_clicked)
self.save_function = self.__save_function # save_function
buttons = [self.previous_button, self.next_button, self.save_button]
label_total = Label(value='/ {}'.format(len(self.observations)))
self.text_index = BoundedIntText(value=1,
min=1,
max=len(self.observations))
self.text_index.layout.width = '80px'
self.text_index.layout.height = '35px'
self.text_index.observe(self.__selected_index)
self.out = Output()
self.out.add_class(name)
if self.__is_multilabel():
self.checkboxes = [
Checkbox(False,
description='{}'.format(self.classes[i]),
indent=False) for i in range(len(self.classes))
]
for cb in self.checkboxes:
cb.layout.width = '180px'
cb.observe(self.__checkbox_changed)
self.checkboxes_layout = VBox(
children=[cb for cb in self.checkboxes])
else:
self.checkboxes = RadioButtons(options=self.classes,
disabled=False,
indent=False)
self.checkboxes.layout.width = '180px'
self.checkboxes.observe(self.__checkbox_changed)
self.checkboxes_layout = VBox(children=[self.checkboxes])
output_layout = HBox(children=[self.out, self.checkboxes_layout])
if self.buttons_vertical:
self.all_widgets = HBox(children=[
VBox(children=[HBox([self.text_index, label_total])] +
buttons), output_layout
])
else:
self.all_widgets = VBox(children=[
HBox([self.text_index, label_total]),
HBox(children=buttons), output_layout
])
## loading js library to perform html screenshots
j_code = """
require.config({
paths: {
html2canvas: "https://html2canvas.hertzen.com/dist/html2canvas.min"
}
});
"""
display(Javascript(j_code))
def __create_results_dict(self, file_path, cc):
mapping = {
"categories_id": {},
"categories_name": {},
"observations": {}
}
if not os.path.exists(file_path):
dataset = {'categories': [], "observations": []}
for index, c in enumerate(cc):
category = {"supercategory": c, "name": c, "id": index + 1}
dataset["categories"].append(category)
else:
with open(file_path, 'r') as classification_file:
dataset = json.load(classification_file)
for index, img in enumerate(dataset['observations']):
mapping['observations'][img[self.key]] = index
for c in dataset['categories']:
mapping['categories_id'][c["id"]] = c["name"]
mapping['categories_name'][c["name"]] = c["id"]
index_categories = len(dataset['categories']) + 1
for c in cc:
if not c in mapping['categories_name'].keys():
mapping['categories_id'][index_categories] = c
mapping['categories_name'][c] = index_categories
category = {
"supercategory": c,
"name": c,
"id": index_categories
}
dataset["categories"].append(category)
index_categories += 1
return mapping, dataset
def __checkbox_changed(self, b):
if b['owner'].value is None or b['name'] != 'value':
return
class_name = b['owner'].description
value = b['owner'].value
current_index = self.mapping["observations"][self.observations[
self.pos]]
if self.__is_multilabel():
class_index = self.mapping["categories_name"][class_name]
if not class_index in self.dataset["observations"][current_index][
"categories"] and value:
self.dataset["observations"][current_index][
"categories"].append(class_index)
if class_index in self.dataset["observations"][current_index][
"categories"] and not value:
self.dataset["observations"][current_index][
"categories"].remove(class_index)
else:
class_index = self.mapping["categories_name"][value]
self.dataset["observations"][current_index][
"category"] = class_index
if self.pos == self.max_pos:
self.save_state()
def __is_multilabel(self):
return TaskType.CLASSIFICATION_MULTI_LABEL.value == self.task_type.value
def __create_button(self, description, disabled, function):
button = Button(description=description)
button.disabled = disabled
button.on_click(function)
return button
def __show_image(self, image_record):
if not 'path' in image_record:
print("missing path")
if not os.path.exists(image_record['path']):
print("Image cannot be load" + image_record['path'])
img = Image.open(image_record['path'])
if self.show_name:
print(os.path.basename(image_record['path']))
plt.figure(figsize=self.fig_size)
if not self.show_axis:
plt.axis('off')
plt.imshow(img)
plt.show()
def save_state(self):
with open(self.file_path, 'w') as output_file:
json.dump(self.dataset, output_file, indent=4)
def __save_function(self, image_path, index):
img_name = os.path.basename(image_path).split('.')[0]
j_code = """
require(["html2canvas"], function(html2canvas) {
var element = $(".p-Widget.jupyter-widgets-output-area.output_wrapper.$it_name$")[0];
console.log(element);
html2canvas(element).then(function (canvas) {
var myImage = canvas.toDataURL();
var a = document.createElement("a");
a.href = myImage;
a.download = "$img_name$.png";
a.click();
a.remove();
});
});
"""
j_code = j_code.replace('$it_name$', self.name)
j_code = j_code.replace('$img_name$', img_name)
tmp_out = Output()
with tmp_out:
display(Javascript(j_code))
tmp_out.clear_output()
def __perform_action(self):
if not self.observations[
self.pos] in self.mapping["observations"].keys():
observation = {self.key: self.observations[self.pos]}
if self.is_image:
observation["file_name"] = os.path.basename(
self.observations[self.pos])
observation["id"] = len(self.mapping["observations"]) + 1
if self.__is_multilabel():
observation["categories"] = []
self.dataset["observations"].append(observation)
self.mapping["observations"][observation[self.key]] = len(
self.dataset["observations"]) - 1
current_index = self.mapping["observations"][self.observations[
self.pos]]
self.next_button.disabled = (self.pos == self.max_pos)
self.previous_button.disabled = (self.pos == 0)
if self.__is_multilabel():
for cb in self.checkboxes:
cb.unobserve(self.__checkbox_changed)
if not "categories" in self.dataset["observations"][
current_index]:
self.dataset["observations"][current_index][
"categories"] = []
categories = self.dataset["observations"][current_index][
"categories"]
cb.value = self.mapping["categories_name"][
cb.description] in categories
cb.observe(self.__checkbox_changed)
else:
self.checkboxes.unobserve(self.__checkbox_changed)
obs = self.dataset["observations"][current_index]
category = obs["category"] if "category" in obs else None
if category:
for k in self.mapping["categories_name"]:
if self.mapping["categories_name"][k] == category:
category = k
break
self.checkboxes.value = category
self.checkboxes.observe(self.__checkbox_changed)
with self.out:
self.out.clear_output()
self.image_display_function(
self.dataset["observations"][current_index])
self.text_index.unobserve(self.__selected_index)
self.text_index.value = self.pos + 1
self.text_index.observe(self.__selected_index)
def __on_previous_clicked(self, b):
self.save_state()
self.pos -= 1
self.__perform_action()
def __on_next_clicked(self, b):
self.save_state()
self.pos += 1
self.__perform_action()
def __on_save_clicked(self, b):
self.save_state()
self.save_function(self.observations[self.pos], self.pos)
def __selected_index(self, t):
if t['owner'].value is None or t['name'] != 'value':
return
self.pos = t['new'] - 1
self.__perform_action()
def start_classification(self):
if self.max_pos < self.pos:
print("No available observation")
return
display(self.all_widgets)
self.__perform_action()
def print_statistics(self):
counter = dict()
for c in self.mapping["categories_id"]:
counter[c] = 0
for record in self.dataset["observations"]:
if self.__is_multilabel():
if "categories" in record:
for c in record["categories"]:
counter[c] += 1
elif "category" in record:
counter[record["category"]] += 1
table = []
for c in counter:
table.append([self.mapping["categories_id"][c], counter[c]])
table = sorted(table, key=lambda x: x[0])
print(
tabulate(table,
headers=[
labels_str.info_class_name, labels_str.info_ann_images
]))
def get():
"""Get the parcel's dataset for the given location or ids"""
info = Label(
"1. Select the region and the year to get parcel information.")
values = config.read()
# Set the max number of parcels that can be downloaded at once.
plimit = int(values['set']['plimit'])
def aois_options():
values = config.read()
options = {}
if values['set']['data_source'] == '0':
for desc in values['api']['options']['aois']:
aoi = f"{values['api']['options']['aois'][desc]}"
options[(desc, aoi)] = values['api']['options']['years'][aoi]
elif values['set']['data_source'] == '1':
for aoi in values['ds_conf']:
desc = f"{values['ds_conf'][aoi]['desc']}"
confgs = values['ds_conf'][aoi]['years']
options[(f'{desc} ({aoi})', aoi)] = [y for y in confgs]
return options
def aois_years():
values = config.read()
years = {}
if values['set']['data_source'] == '0':
for desc in values['api']['options']['aois']:
aoi = values['api']['options']['aois'][desc]
years[aoi] = values['api']['options']['years'][aoi]
elif values['set']['data_source'] == '1':
for aoi in values['ds_conf']:
desc = f"{values['ds_conf'][aoi]['desc']}"
years[aoi] = [y for y in values['ds_conf'][aoi]['years']]
return years
try:
aois = Dropdown(
options=tuple(aois_options()),
value=values['set']['ds_conf'],
description='AOI:',
disabled=False,
)
except:
aois = Dropdown(
options=tuple(aois_options()),
description='AOI:',
disabled=False,
)
year = Dropdown(
options=next(iter(aois_options().values())),
description='Year:',
disabled=False,
)
button_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
@button_refresh.on_click
def button_refresh_on_click(b):
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
def table_options_change(change):
try:
year.options = aois_years()[change.new]
except:
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
aois.observe(table_options_change, 'value')
info_method = Label("2. Select a method to get the data.")
method = ToggleButtons(
options=[('Parcel ID', 2), ('Coordinates', 1), ('Map marker', 3),
('Polygon', 4)],
value=None,
description='',
disabled=False,
button_style='info',
tooltips=[
'Enter lat lon', 'Enter parcel ID', 'Select a point on a map',
'Get parcels id in a polygon'
],
)
plon = Text(value='5.664',
placeholder='Add lon',
description='Lon:',
disabled=False)
plat = Text(value='52.694',
placeholder='Add lat',
description='Lat:',
disabled=False)
wbox_lat_lot = VBox(children=[plat, plon])
info_pid = Label(
"Multiple parcel id codes can be added (comma ',' separated, e.g.: 11111, 22222)."
)
pid = Textarea(value='34296',
placeholder='12345, 67890',
description='Parcel(s) ID:',
disabled=False)
wbox_pids = VBox(children=[info_pid, pid])
bt_get_ids = Button(description="Find parcels",
disabled=False,
button_style='info',
tooltip='Find parcels within the polygon.',
icon='')
get_ids_box = HBox(
[bt_get_ids,
Label("Find the parcels that are in the polygon.")])
ppoly_out = Output()
progress = Output()
def outlog(*text):
with progress:
print(*text)
def outlog_poly(*text):
with ppoly_out:
print(*text)
@bt_get_ids.on_click
def bt_get_ids_on_click(b):
with ppoly_out:
try:
get_requests = data_source()
ppoly_out.clear_output()
polygon = get_maps.polygon_map.feature_collection['features'][
-1]['geometry']['coordinates'][0]
polygon_str = '-'.join(
['_'.join(map(str, c)) for c in polygon])
outlog_poly(f"Geting parcel ids within the polygon...")
polyids = json.loads(
get_requests.ppoly(aois.value, year.value, polygon_str,
False, True))
outlog_poly(
f"'{len(polyids['ogc_fid'])}' parcels where found:")
outlog_poly(polyids['ogc_fid'])
file = config.get_value(['files', 'pids_poly'])
with open(file, "w") as text_file:
text_file.write('\n'.join(map(str, polyids['ogc_fid'])))
except Exception as err:
outlog("No parcel ids found:", err)
method_out = Output(layout=Layout(border='1px solid black'))
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 1:
display(wbox_lat_lot)
elif obj['new'] == 2:
display(wbox_pids)
elif obj['new'] == 3:
display(
get_maps.base_map(
aois.value,
int(config.get_value(['set', 'data_source']))))
elif obj['new'] == 4:
display(
VBox([
get_maps.polygon(
aois.value,
int(config.get_value(['set', 'data_source']))),
get_ids_box, ppoly_out
]))
method.observe(method_options, 'value')
info_type = Label("3. Select datasets to download.")
table_options = HBox([aois, button_refresh, year])
# ########### Time series options #########################################
pts_bt = ToggleButton(
value=False,
description='Time series',
disabled=False,
button_style='success', # success
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pts_bands = data_options.pts_bands()
pts_tstype = SelectMultiple(
options=data_options.pts_tstype(),
value=['s2'],
rows=3,
description='TS type:',
disabled=False,
)
pts_band = Dropdown(
options=list(pts_bands['s2']),
value='',
description='Band:',
disabled=False,
)
def pts_tstype_change(change):
if len(pts_tstype.value) <= 1:
pts_band.disabled = False
try:
pts_b = change.new[0]
pts_band.options = pts_bands[pts_b]
except:
pass
else:
pts_band.value = ''
pts_band.disabled = True
pts_tstype.observe(pts_tstype_change, 'value')
pts_options = VBox(children=[pts_tstype, pts_band])
# ########### Chip images options #########################################
pci_bt = ToggleButton(value=False,
description='Chip images',
disabled=False,
button_style='success',
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pci_start_date = DatePicker(value=datetime.date(2019, 6, 1),
description='Start Date',
disabled=False)
pci_end_date = DatePicker(value=datetime.date(2019, 6, 30),
description='End Date',
disabled=False)
pci_plevel = RadioButtons(
options=['LEVEL2A', 'LEVEL1C'],
value='LEVEL2A',
description='Proces. level:', # Processing level
disabled=False,
layout=Layout(width='50%'))
pci_chipsize = IntSlider(value=640,
min=100,
max=5120,
step=10,
description='Chip size:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='d')
pci_bands = data_options.pci_bands()
pci_satellite = RadioButtons(options=list(pci_bands),
value='Sentinel 2',
disabled=True,
layout=Layout(width='100px'))
pci_band = SelectMultiple(options=list(pci_bands['Sentinel 2']),
value=['B04'],
rows=11,
description='Band:',
disabled=False)
sats_plevel = HBox([pci_satellite, pci_plevel])
def on_sat_change(change):
sat = change.new
pci_band.options = pci_bands[sat]
pci_satellite.observe(on_sat_change, 'value')
pci_options = VBox(children=[
pci_start_date, pci_end_date, sats_plevel, pci_chipsize, pci_band
])
# ########### General options #############################################
pts_wbox = VBox(children=[])
pci_wbox = VBox(children=[])
def pts_observe(button):
if button['new']:
pts_bt.icon = 'toggle-on'
pts_wbox.children = [pts_options]
else:
pts_bt.icon = 'toggle-off'
pts_wbox.children = []
def pci_observe(button):
if button['new']:
pci_bt.icon = 'toggle-on'
pci_wbox.children = [pci_options]
else:
pci_bt.icon = 'toggle-off'
pci_wbox.children = []
pts_bt.observe(pts_observe, names='value')
pci_bt.observe(pci_observe, names='value')
pts = VBox(children=[pts_bt, pts_wbox], layout=Layout(width='40%'))
pci = VBox(children=[pci_bt, pci_wbox], layout=Layout(width='40%'))
data_types = HBox(children=[pts, pci])
info_get = Label("4. Download the selected data.")
bt_get = Button(description='Download',
disabled=False,
button_style='warning',
tooltip='Send the request',
icon='download')
path_temp = config.get_value(['paths', 'temp'])
path_data = config.get_value(['paths', 'data'])
info_paths = HTML("".join([
"<style>div.c {line-height: 1.1;}</style>",
"<div class='c';>By default data will be stored in the temp folder ",
f"({path_temp}), you will be asked to empty the temp folder each time ",
"you start the notebook.<br>In your personal data folder ",
f"({path_data}) you can permanently store the data.</div>"
]))
paths = RadioButtons(options=[
(f"Temporary folder: '{path_temp}'.", path_temp),
(f"Personal data folder: '{path_data}'.", path_data)
],
layout={'width': 'max-content'},
value=path_temp)
paths_box = Box([Label(value="Select folder:"), paths])
def file_len(fname):
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
def get_data(parcel):
values = config.read()
get_requests = data_source()
pid = parcel['ogc_fid'][0]
source = int(config.get_value(['set', 'data_source']))
if source == 0:
datapath = f'{paths.value}{aois.value}{year.value}/parcel_{pid}/'
elif source == 1:
ds_conf = config.get_value(['set', 'ds_conf'])
datapath = f'{paths.value}{ds_conf}/parcel_{pid}/'
file_pinf = f"{datapath}{pid}_information"
outlog(data_handler.export(parcel, 10, file_pinf))
if pts_bt.value is True:
outlog(f"Getting time series for parcel: '{pid}',",
f"({pts_tstype.value} {pts_band.value}).")
for pts in pts_tstype.value:
ts = json.loads(
get_requests.pts(aois.value, year.value, pid, pts,
pts_band.value))
band = ''
if pts_band.value != '':
band = f"_{pts_band.value}"
file_ts = f"{datapath}{pid}_time_series_{pts}{band}"
outlog(data_handler.export(ts, 11, file_ts))
if pci_bt.value is True:
files_pci = f"{datapath}{pid}_chip_images/"
outlog(f"Getting '{pci_band.value}' chip images for parcel: {pid}")
with progress:
get_requests.rcbl(parcel, pci_start_date.value,
pci_end_date.value, pci_band.value,
pci_satellite.value, pci_chipsize.value,
files_pci)
filet = f'{datapath}/{pid}_chip_images/{pid}_images_list.{pci_band.value[0]}.csv'
if file_len(filet) > 1:
outlog(
f"Completed, all GeoTIFFs for bands '{pci_band.value}' are ",
f"downloaded in the folder: '{datapath}/{pid}_chip_images'"
)
else:
outlog(
"No files where downloaded, please check your configurations"
)
def get_from_location(lon, lat):
get_requests = data_source()
outlog(f"Finding parcel information for coordinates: {lon}, {lat}")
parcel = json.loads(
get_requests.ploc(aois.value, year.value, lon, lat, True))
pid = parcel['ogc_fid'][0]
outlog(f"The parcel '{pid}' was found at this location.")
try:
get_data(parcel)
except Exception as err:
print(err)
def get_from_id(pids):
get_requests = data_source()
outlog(f"Getting parcels information for: '{pids}'")
for pid in pids:
try:
parcel = json.loads(
get_requests.pid(aois.value, year.value, pid, True))
get_data(parcel)
except Exception as err:
print(err)
@bt_get.on_click
def bt_get_on_click(b):
progress.clear_output()
if method.value == 1:
try:
with progress:
get_requests = data_source()
lon, lat = plon.value, plat.value
get_from_location(lon, lat)
except Exception as err:
outlog(
f"Could not get parcel information for location '{lon}', '{lat}': {err}"
)
elif method.value == 2:
try:
with progress:
pids = pid.value.replace(" ", "").split(",")
get_from_id(pids)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 3:
try:
marker = get_maps.base_map.map_marker
lon = str(round(marker.location[1], 2))
lat = str(round(marker.location[0], 2))
get_from_location(lon, lat)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 4:
try:
file = config.get_value(['files', 'pids_poly'])
with open(file, "r") as text_file:
pids = text_file.read().split('\n')
outlog("Geting data form the parcels:")
outlog(pids)
if len(pids) <= plimit:
get_from_id(pids)
else:
outlog(
"You exceeded the maximum amount of selected parcels ",
f"({plimit}) to get data. Please select smaller area.")
except Exception as err:
outlog("No pids file found.", err)
else:
outlog(f"Please select method to get parcel information.")
return VBox([
info, table_options, info_method, method, method_out, info_type,
data_types, info_get, info_paths, paths_box, bt_get, progress
])