def test_setting_non_empty_value(self):
# Simulate user setting a value for the upload from the kernel.
uploader = FileUpload()
content = memoryview(b'some content')
uploader.value = [{
'name':
'some-name.txt',
'type':
'text/plain',
'size':
561,
'last_modified':
dt.datetime(2020, 1, 9, 13, 58, 16, 434000,
tzinfo=dt.timezone.utc),
'content':
content
}]
state = uploader.get_state(key='value')
assert len(state['value']) == 1
[entry] = state['value']
assert entry['name'] == 'some-name.txt'
assert entry['type'] == 'text/plain'
assert entry['size'] == 561
assert entry['last_modified'] == 1578578296434
assert entry['content'] == content
def test_receive_single_file(self):
uploader = FileUpload()
message = {'value': [FILE_UPLOAD_FRONTEND_CONTENT]}
uploader.set_state(message)
assert len(uploader.value) == 1
(uploaded_file, ) = uploader.value
assert uploaded_file.name == 'file-name.txt'
assert uploaded_file.type == 'text/plain'
assert uploaded_file.size == 20760
assert uploaded_file.content.tobytes() == b'file content'
assert (uploaded_file.last_modified == dt.datetime(
2020, 1, 9, 13, 58, 16, 434000, tzinfo=dt.timezone.utc))
def upload():
text_file = FileUpload()
def cb(change):
global file_contents
decoded = io.StringIO(change['owner'].data[0].decode('utf-8'))
# filename = change['owner'].filename
# print('Uploaded `{}` ({:.2f} kB)'.format(
# filename, len(decoded.read()) / 2 **10))
file_contents = decoded.getvalue()
text_file.observe(cb, names='data')
display(text_file)
def _create_input_widget(self, inputs: HasTraits) -> Widget:
print("Creating widget")
# uploaded_file_details = Output()
file_upload = FileUpload(accept=".csv", multiple=False)
def set_module_value(change):
# inputs.files = list(change.new.values())
inputs.files = change.new
# file_upload.value.clear()
file_upload._counter = 0
file_upload.observe(set_module_value, names="value")
return file_upload
def test_receive_multiple_files(self):
uploader = FileUpload(multiple=True)
message = {
'value': [
FILE_UPLOAD_FRONTEND_CONTENT, {
**FILE_UPLOAD_FRONTEND_CONTENT,
**{
'name': 'other-file-name.txt'
}
}
]
}
uploader.set_state(message)
assert len(uploader.value) == 2
assert uploader.value[0].name == 'file-name.txt'
assert uploader.value[1].name == 'other-file-name.txt'
def get_files_dropdown(path,
ftypes='',
description='',
select_multi=False,
only_shp=False):
progress = Output()
def outlog(*text):
with progress:
print(*text)
select = FileUpload(description='Select file:',
icon='plus',
accept=ftypes,
multiple=select_multi)
clear = Button(value=False,
button_style='warning',
tooltip='Clear selection.',
icon='broom',
layout=Layout(width='40px'))
upload = Button(value=False,
button_style='info',
tooltip='Upload file.',
icon='fa-upload',
layout=Layout(width='40px'))
dist_folder = Label(f"Destination path: '{path}'")
flist = Dropdown(
options=filter_files(path, ftypes, only_shp),
description=description,
disabled=False,
)
refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
@refresh.on_click
def refresh_on_click(b):
flist.options = filter_files(path, ftypes, only_shp)
@clear.on_click
def clear_on_click(b):
select.value.clear()
select._counter = 0
@upload.on_click
def upload_on_click(b):
progress.clear_output()
os.makedirs(path, exist_ok=True)
for key in select.value:
content = select.value[key]['content']
with open(f'{path}/{key}', 'wb') as f:
f.write(content)
flist.options = filter_files(path, ftypes, only_shp)
outlog("The file is uploaded.")
return VBox([
HBox([select, clear, upload, dist_folder]),
HBox([flist, refresh]), progress
])
def __new__(cls, filename):
if filename in cls.images:
return cls.images[filename]
else:
obj = super().__new__(cls)
obj.widget = FileUpload()
obj.file = Path(filename)
cls.images[filename] = obj
return obj
def test_serialization_deserialization_integrity(self):
# The value traitlet needs to remain unchanged following
# a serialization / deserialization roundtrip, otherwise
# the kernel dispatches it back to the frontend following
# a state change, because it doesn't recognize that the
# property_lock entry is the same as the new value.
from ipykernel.comm import Comm
uploader = FileUpload()
mock_comm = MagicMock(spec=Comm)
mock_comm.kernel = 'does not matter'
mock_comm.send = MagicMock()
uploader.comm = mock_comm
message = {'value': [FILE_UPLOAD_FRONTEND_CONTENT]}
uploader.set_state(message)
# Check that no message is sent back to the frontend
# as a result of setting the state.
mock_comm.send.assert_not_called()
def create_file_upload(self) -> v.Btn:
file_uploader = FileUpload(description=".ics file", multiple=False)
def on_upload(change):
self.main_loading.indeterminate = True
value = change["new"]
filename = list(value.keys())[0]
uploaded_file = Path(self.app_dir) / filename
try:
with open(uploaded_file, "wb") as outfile:
outfile.write(value[filename]["content"])
hm = Heatmap(input_data=uploaded_file)
hm.draw(title=filename)
self.heatmap_plot.children = [hm.result.canvas]
finally:
Path(uploaded_file).exists() and Path(uploaded_file).unlink()
self.main_loading.indeterminate = False
file_uploader.observe(on_upload, "value")
btn_uploader = v.Btn(class_="mx-2", children=[file_uploader])
return btn_uploader
def on_selector_change(self, change):
if self.image_selector.value in self.image_dict:
self.change_image(self.image_dict[self.image_selector.value])
self.widget.children = [self.selector_box, self.display_box]
else:
self.upload_widget = FileUpload(accept='image/*', multiple=False)
self.name_widget = Text(description='<b>Image Name</b>', style={'description_width': 'initial'})
self.ok_widget = ToggleButton(
value=False,
description='Add',
disabled=False,
button_style='success',
tooltip='Description',
icon='check')
self.add_widget = HBox(children=[self.upload_widget, self.name_widget, self.ok_widget],
layout=Layout(justify_content='space-around'))
self.widget.children = [self.selector_box, self.add_widget]
self.upload_widget.observe(self.on_upload, 'value')
self.ok_widget.observe(self.on_add, 'value')
def test_resetting_value(self):
# Simulate an upload, then resetting the value from the
# kernel.
uploader = FileUpload()
message = {'value': [FILE_UPLOAD_FRONTEND_CONTENT]}
uploader.set_state(message)
uploader.value = [] # reset value to an empty file list
assert uploader.get_state(key='value') == {'value': []}
class Mint(MintBase):
def __init__(self, *args, **kwargs):
self.progress_callback = self.set_progress
super().__init__(progress_callback=self.progress_callback,
*args,
**kwargs)
fc = FileChooser()
fc.show_only_dirs = True
fc.default_path = tempfile.gettempdir()
self.ms_storage_path = fc
self.ms_upload = FileUpload()
self.peaklist_files_button = FileUpload(description='Peaklists',
accept='csv,xlsx',
multiple=False)
self.peaklist_files_button.observe(self.load_peaklist, names='value')
self.load_ms_button = Button(description='Load MS-files')
self.load_ms_button.on_click(self.search_files)
self.detect_peaks_button = Button(description="Detect Peaks")
self.detect_peaks_button.on_click(self.detect_peaks)
self.message_box = Textarea(
value='',
placeholder='Please select some files and click on Run.',
description='',
disabled=True,
layout={
'width': '90%',
'height': '500px',
'font_family': 'monospace'
})
self.run_button = Button(description="Run")
self.run_button.on_click(self.run)
self.run_button.style.button_color = 'lightgray'
self.optimize_rt_button = Button(description="Find closest peaks")
self.optimize_rt_button.on_click(self.action_optimize_rt)
self.download_button = Button(description="Export")
self.download_button.on_click(self.export_action)
self.download_button.style.button_color = 'lightgray'
self.progress_bar = Progress(min=0,
max=100,
layout=Layout(width='90%'),
description='Progress:',
bar_style='info')
self.output = widgets.Output()
tabs = Tab()
tabs.children = [
HBox([self.ms_storage_path, self.ms_upload, self.load_ms_button]),
HBox([
self.peaklist_files_button, self.detect_peaks_button,
self.optimize_rt_button
]),
]
tabs.set_title(0, 'MS-Files')
tabs.set_title(1, 'Peaklists')
self.layout = VBox([
tabs, self.message_box,
HBox([self.run_button, self.download_button]), self.progress_bar
])
def load_peaklist(self, value):
for fn, data in value['new'].items():
self.load(io.BytesIO(data['content']))
self.list_files()
def action_optimize_rt(self, b):
if (self.n_files > 0) and len(self.peaklist) > 0:
self.optimize_rt()
def message(self, text):
self.message_box.value = f'{text}\n' + self.message_box.value
def clear_messages(self):
self.message_box.value = ''
def search_files(self, b=None):
self.ms_files = (
glob(os.path.join(self.ms_storage_path.selected_path, '*mzXML')) +
glob(os.path.join(self.ms_storage_path.selected_path, '*mzML')) +
glob(os.path.join(self.ms_storage_path.selected_path, '*mzHDF')) +
glob(os.path.join(self.ms_storage_path.selected_path, '*mzxml')) +
glob(os.path.join(self.ms_storage_path.selected_path, '*mzml')) +
glob(os.path.join(self.ms_storage_path.selected_path, '*mzhdf')))
self.list_files()
def show(self):
display(
HTML("<style>textarea, input { font-family: monospace; }</style>"))
return self.layout
def files(self, files):
super(Mint, self).ms_files = files
self.ms_files_button.files = files
self.list_files()
def add_ms_files(self, fns):
if fns is not None:
self.ms_files = self.ms_files + fns
self.list_files()
def add_peaklist_files(self, fns):
if fns is not None:
self.peaklist_files = self.peaklist_files + fns
self.list_files()
def list_files(self, b=None):
text = f'{self.n_files} MS-files to process:\n'
for i, line in enumerate(self.ms_files):
text += line + '\n'
if i > 10:
text += line + '\n...\n'
break
text += '\nUsing peak list:\n'
if len(self.peaklist_files) != 0:
text += '\n'.join([str(i) for i in self.peaklist_files])
elif len(self.peaklist) != 0:
text += self.peaklist.to_string()
else:
text += '\nNo peaklist defined.'
self.message(text)
if (self.n_files != 0) and (self.n_peaklist_files != 0):
self.run_button.style.button_color = 'lightgreen'
else:
self.run_button.style.button_color = 'lightgray'
def run(self, b=None, **kwargs):
self.progress = 0
super(Mint, self).run(**kwargs)
self.message('Done processing MS-files.')
if self.results is not None:
self.download_button.style.button_color = 'lightgreen'
def detect_peaks(self, b=None, **kwargs):
self.message('\n\nRun peak detection.')
super(Mint, self).detect_peaks(**kwargs)
def set_progress(self, value):
self.progress_bar.value = value
def export_action(self, b=None, filename=None):
if filename is None:
filename = 'MINT__results.xlsx'
filename = os.path.join(HOME, filename)
self.export(filename)
self.message(f'\n\nExported results to: {filename}')
def __init__(self, *args, **kwargs):
self.progress_callback = self.set_progress
super().__init__(progress_callback=self.progress_callback,
*args,
**kwargs)
fc = FileChooser()
fc.show_only_dirs = True
fc.default_path = tempfile.gettempdir()
self.ms_storage_path = fc
self.ms_upload = FileUpload()
self.peaklist_files_button = FileUpload(description='Peaklists',
accept='csv,xlsx',
multiple=False)
self.peaklist_files_button.observe(self.load_peaklist, names='value')
self.load_ms_button = Button(description='Load MS-files')
self.load_ms_button.on_click(self.search_files)
self.detect_peaks_button = Button(description="Detect Peaks")
self.detect_peaks_button.on_click(self.detect_peaks)
self.message_box = Textarea(
value='',
placeholder='Please select some files and click on Run.',
description='',
disabled=True,
layout={
'width': '90%',
'height': '500px',
'font_family': 'monospace'
})
self.run_button = Button(description="Run")
self.run_button.on_click(self.run)
self.run_button.style.button_color = 'lightgray'
self.optimize_rt_button = Button(description="Find closest peaks")
self.optimize_rt_button.on_click(self.action_optimize_rt)
self.download_button = Button(description="Export")
self.download_button.on_click(self.export_action)
self.download_button.style.button_color = 'lightgray'
self.progress_bar = Progress(min=0,
max=100,
layout=Layout(width='90%'),
description='Progress:',
bar_style='info')
self.output = widgets.Output()
tabs = Tab()
tabs.children = [
HBox([self.ms_storage_path, self.ms_upload, self.load_ms_button]),
HBox([
self.peaklist_files_button, self.detect_peaks_button,
self.optimize_rt_button
]),
]
tabs.set_title(0, 'MS-Files')
tabs.set_title(1, 'Peaklists')
self.layout = VBox([
tabs, self.message_box,
HBox([self.run_button, self.download_button]), self.progress_bar
])
def test_empty_initial_value(self):
uploader = FileUpload()
assert uploader.value == ()
def test_construction_with_params(self):
uploader = FileUpload(accept='.txt', multiple=True, disabled=True)
assert uploader.accept == '.txt'
assert uploader.multiple
assert uploader.disabled
def test_construction(self):
uploader = FileUpload()
# Default
assert uploader.accept == ''
assert not uploader.multiple
assert not uploader.disabled
def foi_v2():
path_data = f"{config.get_value(['paths', 'temp'])}foi/"
progress = Output()
def outlog(*text):
with progress:
print(*text)
imp_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
foi_info = HTML(
value=
"""FOI procedures version 2 does not need direct access to the database.
""",
placeholder='FOI Information',
)
# Generate or upload image.
img_info = Label(
f"3. Upload or generate raster base image. (Only upload is currently available)"
)
img_option = ToggleButtons(
options=['Upload', 'Generate'],
value=None,
disabled=True,
button_style='info', # 'success', 'info', 'warning', 'danger' or ''
tooltips=['Upnload your base image', 'Get from object storage'])
img_dist_folder = Text(value=f"{path_data}raster/",
placeholder='tmp/',
description='Folder:',
disabled=False)
img_select = FileUpload(
description='Select file:',
icon='plus',
accept='.tif, .tiff',
multiple=True # True to accept multiple files upload else False
)
img_clear = Button(value=False,
disabled=False,
button_style='info',
tooltip='Upload foi reference data (.shp)',
icon='broom',
layout=Layout(width='40px'))
img_upload = Button(value=False,
disabled=False,
button_style='info',
tooltip='Upload foi base image (.tif)',
icon='fa-upload',
layout=Layout(width='40px'))
img_box = HBox(
[HBox([img_dist_folder, img_select, img_clear, img_upload])])
# YAML File upload
yml_select = FileUpload(description='Select file:',
icon='plus',
accept='.yml, .yaml, .txt',
multiple=False)
yml_clear = Button(value=False,
disabled=False,
button_style='info',
tooltip='Clear selection.',
icon='broom',
layout=Layout(width='40px'))
yml_upload = Button(value=False,
disabled=False,
button_style='info',
tooltip='Upload yaml file.',
icon='fa-upload',
layout=Layout(width='40px'))
yml_box = HBox([yml_select, yml_clear, yml_upload])
# Prepare procedures
pre_info = Label("4. Prepare FOI procedure.")
vector_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}vector/*') if '.shp' in s],
description='Vector file:',
disabled=False,
)
raster_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}raster/*') if '.tif' in s],
description='Raster file:',
disabled=False,
)
yaml_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}/*') if '.yml' in s],
description='yaml file:',
disabled=False,
)
# heterogeneity_threshold
pre_heto_chec = HTML("""
Minimum and maximum thresholds for heterogeneity checks. In the example,
any parcel with percentage of pixels for one class between 30 and 70 from
the total, will be considered heterogenous.
""")
pre_min_het = IntText(value=30,
description='MIN:',
tooltip="Minimum threshold for heterogeneity checks",
disabled=False,
layout=Layout(width='150px'))
pre_max_het = IntText(value=70,
description='MAX:',
tooltip="Maximum threshold for heterogeneity checks",
disabled=False,
layout=Layout(width='150px'))
pre_heto_chec_box = HBox([pre_min_het, pre_max_het])
pre_min_cluster_size = IntText(
value=20,
description='pixels:',
tooltip="Minimum area for clusters selection.",
disabled=False,
layout=Layout(width='200px'))
pre_pixel_connectivity = IntText(
value=8,
description='connectivity type:',
tooltip=
"Type of pixel connectivity in analysis. Accepted values: 4 or 8.",
disabled=False,
layout=Layout(width='200px'))
pre_negative_buffer = IntText(
value=-10,
description='negative buffer:',
tooltip="Negative buffer to be applied on the FOI",
disabled=False,
layout=Layout(width='200px'))
refresh_selections = Button(layout=Layout(width='35px'), icon='fa-refresh')
pre_box = VBox([
pre_info,
Label("Select the requared files:"),
HBox([
vector_file,
HTML(
"Spatial data to be tested - parcels that will be checked for heterogeneity and cardinality."
)
]),
HBox([
raster_file,
HTML("""Thematic raster - classification raster, or raster
from other source that will be used for testing heterogeneity and cardinality."""
)
]),
HTML(
"""YAML file that holds the classes form the thematic raster file -
can be also a simple list of values in the notebook
corespondence between pixel values and names for the classes"""),
HBox([yaml_file, yml_box]), pre_heto_chec, pre_heto_chec_box,
pre_pixel_connectivity, pre_negative_buffer,
HBox([
pre_min_cluster_size,
HTML(
"Minimum area for clusters selection - only clusters bigger from this threshold will be counted."
)
])
])
# Run procedures
run_info = Label("5. Run FOI procedure.")
run_proc = Button(
description='Run FOI procedure version 2',
value=False,
disabled=False,
button_style='info',
tooltip='Run v2 FOI',
icon='play',
)
run_box = HBox([run_proc])
def on_img_option_change(change):
if img_option.value == 'Upload':
img_box.children = [
HBox([img_dist_folder, img_select, img_clear, img_upload])
]
else:
img_box.children = ()
img_option.observe(on_img_option_change, 'value')
@refresh_selections.on_click
def refresh_selections_on_click(b):
vector_file.options = [
s for s in glob.glob(f'{path_data}vector/*') if '.shp' in s
]
raster_file.options = [
s for s in glob.glob(f'{path_data}raster/*') if '.tif' in s
]
yaml_file.options = [
s for s in glob.glob(f'{path_data}/*') if '.yml' in s
]
@img_clear.on_click
def img_clear_on_click(b):
img_select.value.clear()
img_select._counter = 0
@img_upload.on_click
def img_upload_on_click(b):
progress.clear_output()
os.makedirs(img_dist_folder.value, exist_ok=True)
for key in img_select.value:
content = img_select.value[key]['content']
with open(f'{img_dist_folder.value}{key}', 'wb') as f:
f.write(content)
outlog("All files are uploaded.")
@yml_clear.on_click
def yml_clear_on_click(b):
yml_select.value.clear()
yml_select._counter = 0
@yml_upload.on_click
def yml_upload_on_click(b):
progress.clear_output()
yml_dist_folder = f'{path_data}'
os.makedirs(yml_dist_folder, exist_ok=True)
for key in yml_select.value:
content = yml_select.value[key]['content']
with open(f'{yml_dist_folder}{key}', 'wb') as f:
f.write(content)
outlog("The yaml file is uploaded.")
@run_proc.on_click
def run_proc_on_click(b):
with progress:
foi_proc_v2.proc(vector_file.value, raster_file.value,
yaml_file.value, pre_negative_buffer.value,
pre_min_het.value, pre_max_het.value,
pre_pixel_connectivity.value,
pre_min_cluster_size.value)
wbox_v2 = VBox([
proc_func.upload_shp(path_data), img_info, img_option, img_box,
pre_box, run_info, run_box, progress
])
return wbox_v2
nbands=selection.result["nbands"],
functional=selection.result["functional"],
user_kpoints_settings=user_kpoints_settings,
user_incar_settings=user_incar_settings,
handlers=handlers,
cores_per_node=selection.result["cores_per_node"],
kpar_range=selection.result["max_kpt_range"]))
except TypeError as t:
print(t)
print("Incorrect node list input!")
style = {'description_width': 'initial'}
file = interactive(file_interface,
fu=FileUpload(),
fworker=Select(options=CLUSTER_DICT.keys(),
value='leibniz',
rows=3,
description='Fworker:'),
kpt_density=IntSlider(
value=300,
min=100,
max=10000,
step=100,
description='k-point density:',
style={'description_width': 'initial'}))
button = Button(description="Submit Workflow")
button.on_click(submit_workflows)
output = Output()
def __init__(self,
basemapName="Mapnik",
animate=False,
zoom=16,
width="100%",
height="720px"):
"""Parse the UBX File and put it into a DataFrame"""
self.OnLocationChanged = None
self.OnDataLoaded = None
self.OnDataFileChanged = None
self._animate = animate
# map
self._basemaps = {
"Mapnik": basemaps.OpenStreetMap.Mapnik,
"Satellite": basemaps.Esri.WorldImagery,
"WorldStreetMap": basemaps.Esri.WorldStreetMap,
"BlackAndWhite": basemaps.OpenStreetMap.BlackAndWhite,
"France": basemaps.OpenStreetMap.France,
"HOT": basemaps.OpenStreetMap.HOT,
"OpenTopoMap": basemaps.OpenTopoMap,
"WorldTopoMap": basemaps.Esri.WorldTopoMap,
}
self._basemapName = basemapName
self._basemap = self._basemaps[self._basemapName]
self._zoom = zoom
self._startLocation = (49, 8)
self._m = Map(basemap=self._basemap,
center=self._startLocation,
zoom=self._zoom,
close_popup_on_click=False)
self._m.layout.width = width
self._m.layout.height = height
self._addControls()
# parser
self._parser = UbxParser()
# GUI elements
self._fileSelector = widgets.Dropdown(value=self.UbxFiles[0],
placeholder="Log Files",
options=self.UbxFiles,
description="Log File",
ensureoption=False,
disabled=False)
self._fileSelector.observe(self._on_filename_changed, names='value')
self._waypointSlider = widgets.IntSlider(
value=0,
description="Waypoint",
min=0,
max=0,
layout=widgets.Layout(width='50%'))
self._waypointSlider.observe(self._on_waypoint_changed, names='value')
self._mapSelector = widgets.Dropdown(value=self._basemapName,
options=list(
self._basemaps.keys()),
description="Map",
ensureoption=True,
disabled=False)
self._mapSelector.observe(self._on_mapstyle_changed, names='value')
self._upload = FileUpload(accept='.pubx', multiple=True)
self._upload.observe(self._on_file_upload_changed, names="value")
self._header = HTML(
"<b><font size='2' color='orange'>pixelmaps</font></b>")
self._gui = widgets.VBox([
widgets.HBox([
self._header, self._fileSelector, self._waypointSlider,
self._mapSelector, self._upload
]), self._m
])
# extension: .py
# format_name: percent
# format_version: '1.3'
# jupytext_version: 1.4.1
# kernelspec:
# display_name: Python 3
# language: python
# name: python3
# ---
# %%
from ipywidgets import FileUpload
from IPython.display import display
file_upload = FileUpload(
accept='.csv', # Accepted file extension e.g. '.txt', '.pdf', 'image/*', 'image/*,.pdf'
multiple=False # True to accept multiple files upload else False
)
display(file_upload)
# %%
filename = next(iter(file_upload.value))
print(f'"{filename}" uploaded!')
from pathlib import Path
CSV_DIRECTORY = Path('csv')
CSV_DIRECTORY.mkdir(exist_ok=True)
content = file_upload.value[filename]['content']
with open(f'csv/{filename}', 'wb') as f: f.write(content)
def upload_shp(path_data, config_path=False):
# Upload
l_up = Label("a. Upload .shp to the server.")
accept_files = ".shp, .cpg, .dbf, .prj, .shx, .sbn, .sbx, .xml"
data_path = config.get_value(['paths', 'temp'])
shp_dist_folder = Text(value=normpath(join(path_data, 'vector')),
placeholder='tmp/',
description='Folder:',
disabled=config_path)
shp_select = FileUpload(
description='Select files:',
icon='plus',
accept=accept_files,
multiple=True # True to accept multiple files upload else False
)
shp_clear = Button(value=False,
disabled=False,
button_style='info',
tooltip='Clear selections.',
icon='broom',
layout=Layout(width='40px'))
shp_upload = Button(value=False,
disabled=False,
button_style='info',
tooltip='Upload foi reference data (.shp).',
icon='fa-upload',
layout=Layout(width='40px'))
progress = Output()
def outlog(*text):
with progress:
print(*text)
@shp_clear.on_click
def shp_clear_on_click(b):
shp_select.value.clear()
shp_select._counter = 0
@shp_upload.on_click
def shp_upload_on_click(b):
progress.clear_output()
os.makedirs(shp_dist_folder.value, exist_ok=True)
for key in shp_select.value:
content = shp_select.value[key]['content']
with open(normpath(join(shp_dist_folder.value, key)), 'wb') as f:
f.write(content)
outlog("All files are uploaded.")
shp_select.value.clear()
shp_select._counter = 0
shp_box = HBox([shp_dist_folder, shp_select, shp_clear, shp_upload])
# Import the .shp to the database
l_imp = Label("""b. Import uploaded .shp to the database. Add a short name,
max 15 characters for the parcels table e.g.:escat2020.""")
imp_select = Dropdown(options=[
s for s in glob.glob(normpath(join(shp_dist_folder.value, '*')))
if '.shp' in s
],
description='Select .shp:',
disabled=False)
imp_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
imp_proc = Button(
description='Import .shp file',
value=False,
disabled=False,
button_style='info',
tooltip='Run',
icon='fa-database',
)
imp_truncate = Checkbox(value=False,
description='Remove old entries',
disabled=False)
try:
with open(normpath(join(data_path, 'tb_prefix')), 'r') as f:
name_from_prfix = f.read()
except Exception:
name_from_prfix = None
imp_tb_name = Text(value=name_from_prfix,
placeholder='ms2010',
description='Table name:',
tooltip='A short name max 10 char.',
disabled=False)
def on_imp_tb_name(change):
with open(normpath(join(data_path, 'tb_prefix')), 'w+') as f:
f.write(imp_tb_name.value)
imp_tb_name.observe(on_imp_tb_name, 'value')
@imp_proc.on_click
def imp_proc_on_click(b):
if imp_tb_name is not None:
import subprocess
progress.clear_output()
# tb_name = imp_select.value.split('/')[-1].split('.')[0]
tb_name = imp_tb_name.value.replace(' ', '').lower()[:15]
outlog("Importing .shp to database...")
command = [
'ogr2ogr', '-f', 'PostgreSQL', 'PG:' + db.conn_str(), '-nln',
tb_name, '-nlt', 'PROMOTE_TO_MULTI', '-nlt', 'GEOMETRY',
imp_select.value
]
if imp_truncate.value is True:
command.extend(['--config', 'OGR_TRUNCATE', 'YES'])
if subprocess.call(command) == 0:
progress.clear_output()
outlog(
f"Completed. Total number of rows in table '{tb_name}':",
db.exact_count(tb_name))
else:
outlog("Could not import shp file.")
else:
outlog(
"Please add a name for the parcels table (MAX 15 char e.g.: escat2020)."
)
@imp_refresh.on_click
def imp_refresh_on_click(b):
imp_select.options = [
s for s in glob.glob(f'{shp_dist_folder.value}/*') if '.shp' in s
]
imp_box = HBox(
[imp_tb_name, imp_select, imp_refresh, imp_proc, imp_truncate])
return VBox([l_up, shp_box, l_imp, imp_box, progress])
class ImageSelector():
def __init__(self, image_dict, max_width=None):
self.max_width = max_width
self.image_dict = image_dict
self.default_image = list(image_dict.keys())[0]
self.image = self.image_dict[self.default_image]
# Image Selector
self.image_selector = Dropdown(options=list(self.image_dict) + ['+ Add new image'])
self.image_selector.observe(self.on_selector_change, 'value')
self.selector_box = HBox(children=[self.image_selector], layout=Layout(justify_content='space-around'))
# Image Display
width, height = self.get_size(*self.image.size)
self.image_display = Image(value=to_binary(self.image),
format='png',
width=width,
height=height)
self.display_box = HBox(children=[self.image_display], layout=Layout(justify_content='center'))
self.widget = VBox(children=[self.selector_box, self.display_box],
layout=Layout(align_content='inherit'))
def get_size(self, width, height):
if self.max_width is not None:
new_width = min(self.max_width, width)
height = int((new_width/width) * height)
return new_width, height
return width, height
def change_image(self, image):
self.image = image
self.image_display.width, self.image_display.height = self.get_size(*image.size)
self.image_display.value = to_binary(image)
def on_selector_change(self, change):
if self.image_selector.value in self.image_dict:
self.change_image(self.image_dict[self.image_selector.value])
self.widget.children = [self.selector_box, self.display_box]
else:
self.upload_widget = FileUpload(accept='image/*', multiple=False)
self.name_widget = Text(description='<b>Image Name</b>', style={'description_width': 'initial'})
self.ok_widget = ToggleButton(
value=False,
description='Add',
disabled=False,
button_style='success',
tooltip='Description',
icon='check')
self.add_widget = HBox(children=[self.upload_widget, self.name_widget, self.ok_widget],
layout=Layout(justify_content='space-around'))
self.widget.children = [self.selector_box, self.add_widget]
self.upload_widget.observe(self.on_upload, 'value')
self.ok_widget.observe(self.on_add, 'value')
#
def on_upload(self, change):
image_binary = list(self.upload_widget.value.values())[0]['content']
image = PImage.open(BytesIO(image_binary))
self.change_image(image)
self.widget.children = [self.selector_box, self.add_widget, self.display_box]
def on_add(self, change):
if self.upload_widget.value:
image_binary = list(self.upload_widget.value.values())[0]['content']
self.image_dict[self.name_widget.value] = PImage.open(BytesIO(image_binary))
self.image_selector.options = list(self.image_dict) + ['+ Add new image']
self.image_selector.value = self.name_widget.value
def foi():
path_plug = "cbm/foi/foi_db_func/"
path_data = f"{config.get_value(['paths', 'temp'])}foi/"
progress = Output()
def outlog(*text):
with progress:
print(*text)
foi_info = HTML(
value="""FOI procedures need direct access to the database. <br>
In case there no image is provided, access to object storage will be needed to generate the base image from sentinel images.
""",
placeholder='FOI Information',
)
# Connect to database
db_info = Label(f"1. Connect to database and object storage.")
db_select = Dropdown(options=[db for db in config.get_value(['db'])],
description='Configure:',
disabled=True,
layout=Layout(width='140px'))
db_config = Button(value=False,
disabled=False,
button_style='info',
tooltip='Configure db connection.',
icon='cogs',
layout=Layout(width='40px'))
db_box = HBox([db_select, db_config])
parcels_table = RadioButtons(
options=[('Upload .shp', 0), ('From database', 1)],
# value='pineapple', # Defaults to 'pineapple'
# layout={'width': 'max-content'}, # If the items' names are long
# description='Pizza topping:',
disabled=False)
par_box = HBox([])
def on_parcels_table(method):
if method.new == 0:
par_box.children = []
elif method.new == 1:
par_box.children = []
parcels_table.observe(on_parcels_table, 'value')
# Generate or upload image.
img_info = Label(
f"3. Upload or generate raster base image. (Only upload is currently available)"
)
img_option = ToggleButtons(
options=['Upload', 'Generate'],
value=None,
disabled=True,
button_style='info', # 'success', 'info', 'warning', 'danger' or ''
tooltips=['Upnload your base image', 'Get from object storage'])
img_dist_folder = Text(value=f"{path_data}raster/",
placeholder='tmp/',
description='Folder:',
disabled=False)
img_select = FileUpload(
description='Select file:',
icon='plus',
accept='.tif, .tiff',
multiple=True # True to accept multiple files upload else False
)
img_clear = Button(value=False,
disabled=False,
button_style='info',
tooltip='Clear selections.',
icon='broom',
layout=Layout(width='40px'))
img_upload = Button(value=False,
disabled=False,
button_style='info',
tooltip='Upload foi base image (.tif)',
icon='fa-upload',
layout=Layout(width='40px'))
img_box = HBox(
[HBox([img_dist_folder, img_select, img_clear, img_upload])])
# YAML File upload
yml_select = FileUpload(description='Select file:',
icon='plus',
accept='.yml, .yaml, .txt',
multiple=False)
yml_clear = Button(value=False,
disabled=False,
button_style='info',
tooltip='Clear selection.',
icon='broom',
layout=Layout(width='40px'))
yml_upload = Button(value=False,
disabled=False,
button_style='info',
tooltip='Upload yaml file.',
icon='fa-upload',
layout=Layout(width='40px'))
yml_box = HBox([yml_select, yml_clear, yml_upload])
# Prepare procedures
pre_info = Label("4. Prepare FOI procedure.")
pre_ins_func = Button(value=False,
disabled=False,
button_style='info',
tooltip='Insert functions to database.',
icon='fa-share-square',
layout=Layout(width='40px'))
pre_ins_func_box = HBox(
[Label("Add functions to database:"), pre_ins_func])
vector_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}vector/*') if '.shp' in s],
description='Vector file:',
disabled=False,
)
raster_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}raster/*') if '.tif' in s],
description='Raster file:',
disabled=False,
)
yaml_file = Dropdown(
options=[s for s in glob.glob(f'{path_data}/*') if '.yml' in s],
description='yaml file:',
disabled=False,
)
# heterogeneity_threshold
pre_heto_chec = HTML("""
Minimum and maximum thresholds for heterogeneity checks. In the example,
any parcel with percentage of pixels for one class between 30 and 70 from
the total, will be considered heterogenous.
""")
pre_min_het = IntText(value=30,
description='MIN:',
tooltip="Minimum threshold for heterogeneity checks",
disabled=False,
layout=Layout(width='150px'))
pre_max_het = IntText(value=70,
description='MAX:',
tooltip="Maximum threshold for heterogeneity checks",
disabled=False,
layout=Layout(width='150px'))
pre_heto_chec_box = HBox([pre_min_het, pre_max_het])
pre_min_area = IntText(value=2000,
description='area:',
tooltip="Minimum area for clusters selection.",
disabled=False,
layout=Layout(width='200px'))
refresh_selections = Button(layout=Layout(width='35px'), icon='fa-refresh')
pre_box = VBox([
pre_info, pre_ins_func_box,
HBox([Label("Select the requared files:"), refresh_selections]),
HTML("""a. Spatial data to be tested - parcels that will
be checked for heterogeneity and cardinality."""),
HBox([vector_file]),
HTML("""b. Thematic raster - classification raster, or raster
from other source that will be used for testing heterogeneity and cardinality."""
),
HBox([raster_file]),
HTML(
"""c. YAML file that holds the classes form the thematic raster file -
can be also a simple list of values in the notebook
corespondence between pixel values and names for the classes"""),
HBox([yaml_file, yml_box]), pre_heto_chec, pre_heto_chec_box,
HBox([
pre_min_area,
HTML(
"Minimum area for clusters selection - only clusters bigger from this threshold will be counted."
)
])
])
# Run procedures
run_info = Label("5. Run FOI procedure.")
run_proc = Button(
description='Run FOI procedure',
value=False,
disabled=False,
button_style='info',
tooltip='Run',
icon='play',
)
run_box = HBox([run_proc])
def on_img_option_change(change):
if img_option.value == 'Upload':
img_box.children = [
HBox([img_dist_folder, img_select, img_clear, img_upload])
]
else:
img_box.children = ()
img_option.observe(on_img_option_change, 'value')
@refresh_selections.on_click
def refresh_selections_on_click(b):
vector_file.options = [
s for s in glob.glob(f'{path_data}vector/*') if '.shp' in s
]
raster_file.options = [
s for s in glob.glob(f'{path_data}raster/*') if '.tif' in s
]
yaml_file.options = [
s for s in glob.glob(f'{path_data}/*') if '.yml' in s
]
@img_clear.on_click
def img_clear_on_click(b):
img_select.value.clear()
img_select._counter = 0
@img_upload.on_click
def img_upload_on_click(b):
progress.clear_output()
os.makedirs(img_dist_folder.value, exist_ok=True)
for key in img_select.value:
content = img_select.value[key]['content']
with open(f'{img_dist_folder.value}{key}', 'wb') as f:
f.write(content)
outlog("All files are uploaded.")
@yml_clear.on_click
def yml_clear_on_click(b):
yml_select.value.clear()
yml_select._counter = 0
@yml_upload.on_click
def yml_upload_on_click(b):
progress.clear_output()
yml_dist_folder = f'{path_data}'
os.makedirs(yml_dist_folder, exist_ok=True)
for key in yml_select.value:
content = yml_select.value[key]['content']
with open(f'{yml_dist_folder}{key}', 'wb') as f:
f.write(content)
outlog("The yaml file is uploaded.")
db_conf_box = HBox([])
@db_config.on_click
def db_config_on_click(b):
if db_conf_box.children == ():
db_conf_box.children = [settings.direct_conn()]
else:
db_conf_box.children = ()
@pre_ins_func.on_click
def pre_ins_func_on_click(b):
progress.clear_output()
try:
functions = glob.glob(f"{path_plug}*.func")
db = config.get_value(['set', 'db_conn'])
sche = config.get_value(['db', db, 'conn', 'sche'])
user = config.get_value(['db', db, 'conn', 'user'])
for f in functions:
database.insert_function(
open(f).read().format(schema=sche, owner=user))
finc_list = [
f"ipycbm_{f.split('/')[-1].split('.')[0]}, " for f in functions
]
outlog(
f"The functions: {('').join(finc_list)}where added to the database"
)
except Exception as err:
outlog("Could not add functions to dattabase.", err)
@run_proc.on_click
def run_proc_on_click(b):
with progress:
foi_proc.proc(vector_file.value, raster_file.value,
yaml_file.value, pre_min_het.value,
pre_max_het.value, pre_min_area.value)
wbox = VBox([
foi_info, db_info, db_box, db_conf_box,
proc_func.upload_shp(path_data), img_info, img_option, img_box,
pre_box, run_info, run_box, progress
])
return wbox
def remove_marker(self):
if self.marker is not None:
self.m.remove_layer(self.marker)
self.marker = None
self.marker_or_geojson = None
def remove_geojson(self):
if self.geojson is not None:
self.m.remove_layer(self.geojson)
self.geojson = None
self.marker_or_geojson = None
label = Label()
file_upload = FileUpload(accept='.geojson,.json',
multiple=False,
description='Upload GeoJSON')
m = Map(center=(-10, -60),
zoom=4,
interpolation='nearest',
basemap=basemaps.CartoDB.DarkMatter)
map_menu = Map_menu(m, label, file_upload)
m.on_interaction(map_menu.show)
file_upload.observe(map_menu.show_geojson, 'value')
def to_webmercator(source, affine, bounds):
with rasterio.Env():
rows, cols = source.shape
src_transform = affine
# In[6]:
if IN_COLAB:
from google.colab import files
else:
from ipywidgets import FileUpload
from IPython.display import display
get_ipython().system('jupyter nbextension enable --py widgetsnbextension')
# In[7]:
if IN_COLAB:
uploaded = files.upload()
else:
uploaded = FileUpload()
# Running the following cell should enable a clickable button to upload the .zip file. If no button appears, you might need to update / install nodeJS.
# In[8]:
display(uploaded)
# In[9]:
if IN_COLAB:
dataset = list(uploaded.keys())[0]
else:
dataset = list(uploaded.value.keys())[0]
# In[10]:
class UbxMap():
def __init__(self,
basemapName="Mapnik",
animate=False,
zoom=16,
width="100%",
height="720px"):
"""Parse the UBX File and put it into a DataFrame"""
self.OnLocationChanged = None
self.OnDataLoaded = None
self.OnDataFileChanged = None
self._animate = animate
# map
self._basemaps = {
"Mapnik": basemaps.OpenStreetMap.Mapnik,
"Satellite": basemaps.Esri.WorldImagery,
"WorldStreetMap": basemaps.Esri.WorldStreetMap,
"BlackAndWhite": basemaps.OpenStreetMap.BlackAndWhite,
"France": basemaps.OpenStreetMap.France,
"HOT": basemaps.OpenStreetMap.HOT,
"OpenTopoMap": basemaps.OpenTopoMap,
"WorldTopoMap": basemaps.Esri.WorldTopoMap,
}
self._basemapName = basemapName
self._basemap = self._basemaps[self._basemapName]
self._zoom = zoom
self._startLocation = (49, 8)
self._m = Map(basemap=self._basemap,
center=self._startLocation,
zoom=self._zoom,
close_popup_on_click=False)
self._m.layout.width = width
self._m.layout.height = height
self._addControls()
# parser
self._parser = UbxParser()
# GUI elements
self._fileSelector = widgets.Dropdown(value=self.UbxFiles[0],
placeholder="Log Files",
options=self.UbxFiles,
description="Log File",
ensureoption=False,
disabled=False)
self._fileSelector.observe(self._on_filename_changed, names='value')
self._waypointSlider = widgets.IntSlider(
value=0,
description="Waypoint",
min=0,
max=0,
layout=widgets.Layout(width='50%'))
self._waypointSlider.observe(self._on_waypoint_changed, names='value')
self._mapSelector = widgets.Dropdown(value=self._basemapName,
options=list(
self._basemaps.keys()),
description="Map",
ensureoption=True,
disabled=False)
self._mapSelector.observe(self._on_mapstyle_changed, names='value')
self._upload = FileUpload(accept='.pubx', multiple=True)
self._upload.observe(self._on_file_upload_changed, names="value")
self._header = HTML(
"<b><font size='2' color='orange'>pixelmaps</font></b>")
self._gui = widgets.VBox([
widgets.HBox([
self._header, self._fileSelector, self._waypointSlider,
self._mapSelector, self._upload
]), self._m
])
@property
def CurrentLocation(self):
return self._waypointMarker.location
@property
def RouteLine(self):
return self._routeLine
@property
def Map(self):
return self._m
@property
def WaypointSliderWidget(self):
return self._waypointSlider
@property
def FileSelectorWidget(self):
return self._fileSelector
@property
def MaxNumberOfWaypoints(self):
return len(self._df) - 1
@property
def UbxFiles(self):
ubxFiles = []
dir = os.listdir()
ubxFiles = []
for file in dir:
if file.endswith(".pubx"):
ubxFiles.append(file)
if len(ubxFiles) == 0:
ubxFiles.append("<NoLogFiles>")
return ubxFiles
def setCurrentWaypoint(self, value):
self._currentWaypoint = value
row = self._df.iloc[self._currentWaypoint]
speed = row["groundspeed"]
gz = row["gravityz"]
ml = row["miclevel"]
lat = row["latitude"]
lng = row["longitude"]
head = row["headingofmotion"]
self._waypointMessage.value = f"Moving {self._parser.direction(head)} with {speed:.0f} Km/h"
#self._waypointMarker.rotation_angle = head
self._waypointMarker.location = (lat, lng)
self._center()
def Append(self, filename):
self._filename = filename
self._df = self._df.append(
DataFrame.from_dict(self._parser.read(self._filename)))
self._updateData()
def Load(self, filename="<First>"):
if filename == "<First>":
self._filename = self.UbxFiles[0]
if self._filename == "<NoLogFiles>":
return
else:
self._filename = filename
self._df = DataFrame.from_dict(self._parser.read(self._filename))
self._updateData()
self._clearLayers()
def _updateData(self):
# Get the start location
self._startLocation = (self._df.at[0, 'latitude'],
self._df.at[0, 'longitude'])
# Calculate the route average speed
self._averageSpeed = self._df["groundspeed"].mean()
self._minSpeed = self._df["groundspeed"].min()
self._maxSpeed = self._df["groundspeed"].max()
self._averageHOF = self._df["headingofmotion"].mean()
# Get the route points to draw the route
self._routePoints = self._df.loc[:, ["latitude", "longitude"
]].values.tolist()
self._waypointSlider.max = self.MaxNumberOfWaypoints
if self.OnDataLoaded:
self.OnDataLoaded(self)
def _addControls(self):
# Add Controls
self._m.add_control(
MeasureControl(position='bottomleft',
active_color='orange',
primary_length_unit='kilometers'))
self._m.add_control(FullScreenControl())
self._m.add_control(ScaleControl(position='bottomleft'))
searchMarker = Marker(icon=AwesomeIcon(
name="check", marker_color='green', icon_color='darkred'))
self._m.add_control(
SearchControl(
position="topleft",
url=
'https://nominatim.openstreetmap.org/search?format=json&q={s}',
zoom=self._zoom,
marker=searchMarker))
def Display(self):
self._fileSelector.options = self.UbxFiles
self._waypointSlider.max = self.MaxNumberOfWaypoints
return self._gui
def move(self, location):
self._waypointMarker.location = location
def _on_filename_changed(self, change):
newFile = change["new"]
self.Load(newFile)
self.DrawRoute()
if self.OnDataFileChanged:
self.OnDataFileChanged(self, change["new"])
def _on_file_upload_changed(self, change):
newFile = change["new"]
with open(newFile, 'wb') as output_file:
for uploaded_filename in self._upload.value:
content = self._upload.value[uploaded_filename]['content']
output_file.write(content)
def _on_mapstyle_changed(self, change):
self._basemapName = change["new"]
self._m.layers = [basemap_to_tiles(self._basemaps[self._basemapName])]
self.DrawRoute()
def _on_waypoint_changed(self, change):
self._currentWaypoint = int(change['new'])
self.setCurrentWaypoint(self._currentWaypoint)
def _on_location_changed(self, event):
# Do some computation given the new marker location, accessible from `event['new']
print(event["new"])
def _addWaypoint(self):
# Waypoint
self._waypointMarker = Marker(location=self._startLocation,
draggable=False)
self._waypointMarker.observe(self._on_location_changed, 'location')
self._m.add_layer(self._waypointMarker)
# Waypoint Popup Message
self._waypointMessage = HTML()
self._waypointMarker.popup = self._waypointMessage
def DrawRoute(self):
# Draw the route
#self._clearLayers()
self._routeLine = AntPath(
locations=self._routePoints,
dash_array=[1, int(self._averageSpeed)],
delay=int(self._averageSpeed * 500),
color="red",
pulse_color="blue") #color='#7590ba',pulse_color='#3f6fba')
self._m.add_layer(self._routeLine)
self._addWaypoint()
self._center()
self._addStats()
if self._animate:
self._center()
def _center(self):
self._m.center = self._waypointMarker.location
def _centerTo(self, location):
self._m.center = location
def _clearLayers(self):
self._m.clear_layers()
self._m.layers = [basemap_to_tiles(self._basemap)]
def calcSpeedZGravityThreshold(self, speed, grvityz):
threshold = 10.5
if speed < 6: #walking
threshold = 12.5
elif speed >= 6 and speed < 10: # running
threshold = 14.5
elif speed > 10 and speed < 25: # bike
threshold = 16.5
return threshold
def _addStats(self):
oldloc = (0, 0)
# Iterate through the route points
for index, row in self._df.iterrows():
# Get various row info
speed = row["groundspeed"]
gz = row["gravityz"]
ml = row["miclevel"]
lat = row["latitude"]
lng = row["longitude"]
loc = (lat, lng)
# Add speed markers
if speed < 2 and self._parser.distance(loc, oldloc) > 0.01:
#m.add_layer(Marker(location=loc, draggable=False, title="Stopped"))
self._m.add_layer(
Circle(location=loc,
radius=5,
color="green",
fill_color="green"))
oldloc = loc
if self._animate:
self._centerTo(loc)
# Add Z Gravity markers (bumps)
if gz > self.calcSpeedZGravityThreshold(speed, gz):
self._m.add_layer(
Circle(location=loc,
radius=int(gz / 2),
color="red",
fill_color="red"))
if self._animate:
self._centerTo(loc)
# Add Sound Markers
if ml >= 1.5:
self._m.add_layer(
Circle(location=loc,
radius=int(ml * 5),
color="orange",
fill_color="orange"))
if self._animate:
self._centerTo(loc)