def __init__(self):
# initialize the base class
super().__init__()
# Add additional elements
self.wt_signals = Text(placeholder='Signal Table',
description='Signal Table:',
disabled=False)
self.wt_histtable = Text(placeholder='Hist. Table',
description='Hist. Table:',
disabled=False)
self.wt_sm_table = Text(placeholder='Sen. Table',
description='Sen. Table:',
disabled=False)
self.wrb_cloud = RadioButtons(
options=['True', 'False'],
value='True', # Defaults to 'pineapple'
description='Cloud Free:',
disabled=False)
self.wdp_start_date = DatePicker(description='Start date:',
disabled=False)
self.wdp_end_date = DatePicker(description='Stop date:',
disabled=False)
children = [*super().children, self.wt_signals, self.wt_histtable, \
self.wt_sm_table, self.wrb_cloud, self.wdp_start_date, \
self.wdp_end_date]
self.children = children
def getCP(self):
self.setLabel()
self.plon = 54.43
self.plat = 24.42
self.dateSelection = datetime.now()
self.dateLast = datetime(1950, 1, 1)
self.selectVar = 'AOD'
self.selectTime = 0
self.sdateSW = DatePicker(description='Start Date',
layout=Layout(width='auto'),
value=self.startDate,
disabled=False)
self.edateSW = DatePicker(description='End Date',
layout=Layout(width='auto'),
value=self.endDate,
disabled=False)
self.varSW = Dropdown(options=[
'AOD', 'DUST_PM', 'SALT_PM', 'ORG_CARB', 'BLK_CARB', 'SO4', 'PM2.5'
],
value='AOD',
layout=Layout(width='280px'),
description='Variable:',
disabled=False)
self.latSW = Text(description='Latitude:',
disabled=False,
value='24.42',
layout=Layout(width='180px'))
self.lonSW = Text(description='Longitude:',
disabled=False,
value='54.43',
layout=Layout(width='180px'))
self.plotPB = Button(description='Time Series Plot',
disabled=False,
layout={
'width': 'auto',
'border': '3px outset'
})
self.inpUSR.children += (HBox([
VBox([self.sdateSW, self.edateSW, self.varSW]),
VBox([self.latSW, self.lonSW, self.plotPB])
],
layout={'overflow': 'visible'}), )
self.sdateSW.observe(self.sdateSWCB)
self.edateSW.observe(self.edateSWCB)
self.varSW.observe(self.varSWCB, names='value')
self.latSW.observe(self.latSWCB, names='value')
self.lonSW.observe(self.lonSWCB, names='value')
self.plotPB.on_click(self.plotTS)
return self.cp
def DateWidget(*args, **kw):
"""
DateWidget to pick datetime type, if bounds but no default is
defined default to min.
"""
if kw.get('value') is None and 'min' in kw:
kw['value'] = kw['min']
return DatePicker(*args, **kw)
def __init__(self, signal_components: dict, md_type="", parent=None):
self.type = md_type
self.parent = parent
if self.parent is not None:
try:
self.parent.set_title_from_widget(self)
except:
print("Unable to set the Tab title.")
# Retain only the signals (at this stage a single component should be present)
self.signals = list(signal_components.keys())
self.wsm_signals = SelectMultiple(options=["", *self.signals],
value=[""],
description="Signals:",
placeholder="Signals",
disabled=False)
# Components
self.components = signal_components
# start date
wdp_start_date = DatePicker(description='Start date:', disabled=False)
# stop date
wdp_stop_date = DatePicker(description='Stop date:', disabled=False)
self.whb_dates = HBox([wdp_start_date, wdp_stop_date])
# Max number of markers to detect
self.wit_max_num = BoundedIntText(value=0,
min=0,
step=1,
description="Max Number of Markers:")
# Initialize the
super().__init__([
HTML(value=f"<B>Processor type: {self.type}</B>"),
self.wsm_signals, self.whb_dates,
HTML(value="<B>Options :</B>")
],
layout=Layout(border='1px solid black'))
self.options = []
def data_ui():
form_item_layout = Layout(
display='flex',
flex_flow='row',
justify_content='space-between',
)
start_date = DatePicker(value=datetime.datetime(year=2010, month=1, day=1))
end_date = DatePicker(value=datetime.datetime(year=2019, month=1, day=1))
def format_angle(a):
if a < 0:
return f" {a} "
if a > 0:
return f" +{a} "
return " 0 "
lon_range = SelectionRangeSlider(options=[(format_angle(i - 180), i)
for i in range(0, 361)],
index=(0, 360))
lat_range = SelectionRangeSlider(options=[(format_angle(i - 90), i)
for i in range(0, 181)],
index=(0, 180))
form_items = [
Box([Label(value='Dataset name'), Text()], layout=form_item_layout),
Box([Label(value='Start date'), start_date], layout=form_item_layout),
Box([Label(value='End date'), end_date], layout=form_item_layout),
Box([Label(value='Longitude'), lon_range], layout=form_item_layout),
Box([Label(value='Latitude'), lat_range], layout=form_item_layout),
Box([Label(value=''),
Button(description="Search", icon="search")],
layout=form_item_layout),
]
form = Box(form_items,
layout=Layout(display='flex',
flex_flow='column',
border='solid 1px lightgray',
align_items='stretch',
width='50%'))
return form
def dias_cat():
aoi_method = RadioButtons(
# options=[('Predefined MS polygons', 1), ('Get polygon from dataset extent', 2), ('Draw polygon on a map', 3)],
options=[('Predefined MS polygons', 1),
('Get polygon from dataset extent', 2)],
value=2,
description='AOI:',
disabled=False,
)
mss = data_options.ms_polygons()
ms = Dropdown(
options=[(m, mss[m]) for m in mss],
tooltip="AOI",
description='AOI:',
)
tb = Dropdown(
options=database.tables(),
tooltip="Select table",
description='Table:',
)
tb_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
aoi_box = VBox([HBox([tb, tb_refresh])])
dias = Dropdown(
options=data_options.dias_providers(),
value='EOSC',
description='DIAS Provider:',
disabled=True,
)
start = DatePicker(description='Start date', disabled=False)
end = DatePicker(description='End date', disabled=False)
plevel = RadioButtons(
options=['LEVEL2A', 'LEVEL2AP'],
value='LEVEL2A',
description='pLevel:',
disabled=False,
)
ptype = RadioButtons(
options=['CARD-COH6', 'CARD-BS'],
description='pType:',
disabled=False,
)
card_options = VBox([plevel])
card = RadioButtons(
options=[('Sentinel 1', 's1'), ('Sentinel 2', 's2')],
value='s2',
description='card:',
disabled=False,
)
bt_card2db = Button(description='Add CARD to db',
value=False,
disabled=False,
button_style='info',
tooltip='Add CARD catalogue to database',
icon='database')
progress = Output()
def outlog(*text):
with progress:
print(*text)
def on_aoi_method_change(change):
if aoi_method.value == 1:
aoi_box.children = [ms]
elif aoi_method.value == 2:
aoi_box.children = [HBox([tb, tb_refresh])]
elif aoi_method.value == 3:
aoi_box.children = []
aoi_method.observe(on_aoi_method_change, 'value')
def on_card_change(change):
if card.value == 's2':
card_options.children = [plevel]
else:
card_options.children = [ptype]
card.observe(on_card_change, 'value')
@tb_refresh.on_click
def tb_refresh_on_click(b):
tb.options = database.tables()
@bt_card2db.on_click
def bt_card2db_on_click(b):
progress.clear_output()
try:
with open(f"{config.get_value(['paths','temp'])}tb_prefix",
'r') as f:
tb_prefix = f.read()
except Exception:
tb_prefix = ''
dc_table = f'{tb_prefix}_dias_catalogue'
if database.tb_exist(dc_table) is True:
if aoi_method.value == 1:
polygon = ms.value.replace(' ', '+')
elif aoi_method.value == 2:
polygon = database.tb_extent(tb.value)
elif aoi_method.value == 3:
polygon = ms.value.replace(' ', '+')
# print(polygon)
if card.value == 's2':
option = plevel.value
else:
option = ptype.value
outlog("Inserting CARD catalogue to database ...")
with progress:
meta2DB.dias_cat(tb_prefix, f"POLYGON(({polygon}))",
start.value, end.value, card.value, option)
outlog("Completed.")
else:
outlog(f"Table {dc_table} does not exist.")
wbox = VBox([
dias,
HBox([aoi_method, aoi_box]),
HBox([start, end]),
HBox([card, card_options]), bt_card2db, progress
])
return wbox
def __init__(self, signal_components: dict):
"""
Summary:
Object constructor for the base_data_displayer_widget.
Arguments:
signal_components - dictionary with the signals available for plotting.
Returns :
The newly created object
Note :
Currently, a data displayer has the following syntax
{
"signals": ["ndvi","coh_norm"],
"bottom_bar": "b08_b11_b04",
"bottom_bar_components": ["B08_mean","B11_mean","B04_mean"],
"start_date": "2018-04-01",
"stop_date": "2018-10-31",
"output_folder" : "./nl/results_rest",
"file_name_prefix" : "s1_s2_compo",
"legend" : ["NDVI","Coherence Norm"],
"resolution" : 72,
"marker_signals" : ["coh_norm"],
"marker_plot_type" : "tl",
"marker_colors" : {"drop" : "b", "peak" : "y"},
"ylabel" : "label",
"add_months" : true,
}
"""
# start creating the different widgets forming the base_data_displayer_widget
self.signals = ["", *list(signal_components.keys())]
self.wsm_signals = SelectMultiple(options=self.signals,
value=[""],
description="Signals:",
placeholder="Signals",
disabled=False)
self.wdd_bottom_bar = Dropdown(options=self.signals,
value="",
description="Bottom Bar Signal:",
placeholder="Bottom Bar Signal",
disabled=False,
layout={'width': 'max-content'})
self.wdd_red = Dropdown(options=[""],
description="RED:",
placeholder="RED",
disabled=False,
layout={'width': 'max-content'})
self.wdd_green = Dropdown(options=[""],
description="GREEN:",
placeholder="GREEN",
disabled=False,
layout={'width': 'max-content'})
self.wdd_blue = Dropdown(options=[""],
description="BLUE:",
placeholder="BLUE",
disabled=False,
layout={'width': 'max-content'})
def on_bottom_bar_change(change):
components = [""]
if self.wdd_bottom_bar.value in signal_components:
components = signal_components[self.wdd_bottom_bar.value]
self.wdd_red.options = components
self.wdd_green.options = components
self.wdd_blue.options = components
self.wdd_bottom_bar.observe(on_bottom_bar_change, 'value')
self.wh_bb_components = HBox(
[self.wdd_red, self.wdd_green, self.wdd_blue])
# start date
wdp_start_date = DatePicker(description='Start date:', disabled=False)
# stop date
wdp_stop_date = DatePicker(description='Stop date:', disabled=False)
self.whb_dates = HBox([wdp_start_date, wdp_stop_date])
# output folder
self.wfc_outdir = FileChooser(placeholder='Output directory',
description='Output directory:',
disabled=False)
self.wfc_outdir.show_only_dirs = True
# file name prefix
self.wt_prefix = Text(placeholder='File name prefix',
description='File name prefix:',
disabled=False)
self.whb_output_prop = HBox([
VBox([HTML(value="<B>Output folder:<B>"), self.wfc_outdir]),
self.wt_prefix
])
self.wsm_markers = SelectMultiple(options=self.signals,
value=[""],
description="Marker signals:",
placeholder="Marker signals",
disabled=False,
layout={'width': 'max-content'})
# Marker plot type
self.whb_marker_plot_type = VBox([
HTML(value="<B>Marker plot type:</B>"),
Checkbox(value=False,
description='Triangle',
disabled=False,
indent=False),
Checkbox(
value=False, description='Line', disabled=False, indent=False)
],
layout=Layout(
border='1px solid black',
width='25%'))
self.whb_marker_colors = VBox([
HTML(value="<B>Marker color:</B>"),
ColorPicker(concise=True,
description='Drop',
value="#ffffff",
disabled=False,
indent=False),
ColorPicker(concise=True,
description='Peak',
value="#ffffff",
disabled=False,
indent=False)
],
layout=Layout(border='1px solid black',
width='25%'))
self.whb_markers = HBox([
self.wsm_markers, self.whb_marker_plot_type, self.whb_marker_colors
],
layout=Layout(align_items='center',
justify_content='space-around'))
self.wit_resolution = IntText(value=72,
description='Image Resolution:',
disabled=False,
layout=Layout(width='25%',
height='35px'))
self.wt_ymin = FloatText(value=0,
description='Y min:',
disabled=False,
tooltip="Minimum vertical value in plot",
layout=Layout(width='25%', height='35px'))
self.wt_ymax = FloatText(value=0,
description='Y max:',
disabled=False,
tooltip="Maximum vertical value in plot",
layout=Layout(width='25%', height='35px'))
self.wcb_add_months = Checkbox(description='Add month names',
value=True,
indent=False,
layout=Layout(width='25%',
height='35px'))
self.whb_plot_prop = HBox([
self.wit_resolution,
Text(description='Legend',
disabled=False,
layout=Layout(width='25%', height='35px')),
Text(description='Y label',
disabled=False,
layout=Layout(width='25%', height='35px'))
],
layout=Layout(
align_items='center',
justify_content='space-around'))
self.whb_plot_prop2 = HBox(
[self.wt_ymin, self.wt_ymax, self.wcb_add_months],
layout=Layout(align_items='center',
justify_content='space-around'))
super().__init__([
self.wsm_signals, self.wdd_bottom_bar, self.wh_bb_components,
self.whb_dates, self.whb_output_prop, self.whb_markers,
self.whb_plot_prop, self.whb_plot_prop2
])
def get():
"""Get the parcel's dataset for the given location or ids"""
debug = False
info = Label("1. Select the aoi to get parcel data.")
values = config.read()
ppoly_out = Output()
progress = Output()
def outlog(*text):
with progress:
print(*text)
def outlog_poly(*text):
with ppoly_out:
print(*text)
def aois_options():
values = config.read()
options = {}
if values['set']['data_source'] == 'api':
api_values = config.read('api_options.json')
for aoi in api_values['aois']:
options[(aoi.upper(), aoi)] = api_values['aois'][aoi]['years']
elif values['set']['data_source'] == 'direct':
values = config.read('api_options.json')
for aoi in values['dataset']:
options[(f"{aoi.upper()} ({aoi})", aoi)] = [aoi.split('_')[-1]]
return options
def aois_years():
values = config.read()
years = {}
if values['set']['data_source'] == 'api':
api_values = config.read('api_options.json')
for aoi in api_values['aois']:
years[aoi] = api_values['aois'][aoi]['years']
elif values['set']['data_source'] == 'direct':
values = config.read()
for aoi in values['dataset']:
years[aoi] = [aoi.split('_')[-1]]
return years
try:
aois = Dropdown(
options=tuple(aois_options()),
value=values['set']['dataset'],
description='AOI:',
)
except Exception:
aois = Dropdown(
options=tuple(aois_options()),
description='AOI:',
)
def years_disabled():
values = config.read()
if values['set']['data_source'] == 'direct':
return True
else:
return False
year = Dropdown(
options=next(iter(aois_options().values())),
description='Year:',
disabled=years_disabled(),
)
button_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
@button_refresh.on_click
def button_refresh_on_click(b):
values = config.read()
if values['set']['data_source'] == 'api':
from cbm.datas import api
available_options = json.loads(api.get_options())
try:
api_options = normpath(
join(config.path_conf, 'api_options.json'))
os.makedirs(dirname(api_options), exist_ok=True)
with open(api_options, "w") as f:
json.dump(available_options, f, indent=4)
outlog(f"File saved at: {api_options}")
except Exception as err:
outlog(f"Could not create the file 'api_options.json': {err}")
outlog(f"The API options are updated.")
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
year.disabled = years_disabled()
def table_options_change(change):
api_values = config.read('api_options.json')
id_examples = api_values['aois'][change.new]['id_examples']
try:
id_examples_label.value = ', '.join(str(x) for x in id_examples)
year.options = aois_years()[change.new]
year.disabled = years_disabled()
pid.value = str(id_examples[0])
except Exception:
id_examples_label.value = ', '.join(str(x) for x in id_examples)
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
year.disabled = years_disabled()
pid.value = str(id_examples[0])
aois.observe(table_options_change, 'value')
info_method = Label("2. Select a method to download parcel data.")
method = ToggleButtons(
options=[('Parcel ID', 2), ('Coordinates', 1), ('Map marker', 3),
('Polygon', 4)],
value=None,
description='',
disabled=False,
button_style='info',
tooltips=[
'Enter lon lat', 'Enter parcel ID', 'Select a point on a map',
'Get parcels id in a polygon'
],
)
plon = Text(value='5.664', placeholder='Add lon', description='Lon:')
plat = Text(value='52.694', placeholder='Add lat', description='Lat:')
wbox_lat_lot = VBox(children=[plat, plon])
api_values = config.read('api_options.json')
id_examples = api_values['aois'][aois.value]['id_examples']
id_examples_label = Label(', '.join(str(x) for x in id_examples))
info_pid = HBox(
[Label("Multiple parcel ids can be added, e.g.: "), id_examples_label])
pid = Textarea(
value=str(id_examples[0]),
placeholder='12345, 67890',
description='Parcel(s) ID:',
)
wbox_pids = VBox(children=[info_pid, pid])
bt_get_ids = Button(description="Find parcels",
disabled=False,
button_style='info',
tooltip='Find parcels within the polygon.',
icon='')
get_ids_box = HBox(
[bt_get_ids,
Label("Find the parcels that are in the polygon.")])
@bt_get_ids.on_click
def bt_get_ids_on_click(b):
with ppoly_out:
try:
# get_requests = data_source()
ppoly_out.clear_output()
polygon = get_maps.polygon_map.feature_collection['features'][
-1]['geometry']['coordinates'][0]
polygon_str = '-'.join(
['_'.join(map(str, c)) for c in polygon])
outlog_poly(f"Geting parcel ids within the polygon...")
polyids = parcel_info.by_polygon(aois.value, year.value,
polygon_str, ptype.value,
False, True)
outlog_poly(
f"'{len(polyids['ogc_fid'])}' parcels where found:")
outlog_poly(polyids['ogc_fid'])
file = normpath(
join(config.get_value(['paths', 'temp']),
'pids_from_polygon.txt'))
with open(file, "w") as text_file:
text_file.write('\n'.join(map(str, polyids['ogc_fid'])))
except Exception as err:
outlog("No parcel ids found:", err)
method_out = Output(layout=Layout(border='1px solid black'))
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 1:
display(wbox_lat_lot)
elif obj['new'] == 2:
display(wbox_pids)
elif obj['new'] == 3:
display(
get_maps.base_map(aois.value,
config.get_value(['set',
'data_source'])))
elif obj['new'] == 4:
display(
VBox([
get_maps.polygon(
aois.value,
config.get_value(['set', 'data_source'])),
get_ids_box, ppoly_out
]))
method.observe(method_options, 'value')
info_type = Label("3. Select datasets to download.")
ptype = Text(value=None,
placeholder='(Optional) Parcel Type',
description='pType:',
disabled=False)
table_options = HBox([aois, button_refresh, ptype, year])
# ########### Time series options #########################################
pts_bt = ToggleButton(
value=False,
description='Time series',
button_style='success', # success
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pts_bands = data_options.pts_bands()
pts_tstype = SelectMultiple(
options=[("Sentinel-2 Level 2A", 's2'),
("S1 Backscattering Coefficients", 'bs'),
("S1 6-day Coherence (20m)", 'c6')],
value=['s2'],
rows=3,
description='TS type:',
disabled=False,
)
pts_band = Dropdown(
options=list(pts_bands['s2']),
value='',
description='Band:',
disabled=False,
)
def pts_tstype_change(change):
if len(pts_tstype.value) <= 1:
pts_band.disabled = False
try:
pts_b = change.new[0]
pts_band.options = pts_bands[pts_b]
except Exception:
pass
else:
pts_band.value = ''
pts_band.disabled = True
pts_tstype.observe(pts_tstype_change, 'value')
pts_options = VBox(children=[pts_tstype, pts_band])
# ########### Chip images options #########################################
pci_bt = ToggleButton(value=False,
description='Chip images',
disabled=False,
button_style='success',
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pci_start_date = DatePicker(
value=datetime.date(2020, 6, 1),
description='Start Date',
)
pci_end_date = DatePicker(
value=datetime.date(2020, 6, 30),
description='End Date',
)
pci_plevel = RadioButtons(
options=['LEVEL2A', 'LEVEL1C'],
value='LEVEL2A',
description='Proces. level:', # Processing level
disabled=False,
layout=Layout(width='50%'))
pci_chipsize = IntSlider(value=640,
min=100,
max=5120,
step=10,
description='Chip size:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='d')
pci_bands = data_options.pci_bands()
pci_satellite = RadioButtons(options=list(pci_bands),
value='Sentinel 2',
disabled=True,
layout=Layout(width='100px'))
pci_band = SelectMultiple(options=list(pci_bands['Sentinel 2']),
value=['B04'],
rows=11,
description='Band:',
disabled=False)
sats_plevel = HBox([pci_satellite, pci_plevel])
def on_sat_change(change):
sat = change.new
pci_band.options = pci_bands[sat]
pci_satellite.observe(on_sat_change, 'value')
pci_options = VBox(children=[
pci_start_date, pci_end_date, sats_plevel, pci_chipsize, pci_band
])
# ########### General options #############################################
pts_wbox = VBox(children=[])
pci_wbox = VBox(children=[])
def pts_observe(button):
if button['new']:
pts_bt.icon = 'toggle-on'
pts_wbox.children = [pts_options]
else:
pts_bt.icon = 'toggle-off'
pts_wbox.children = []
def pci_observe(button):
if button['new']:
pci_bt.icon = 'toggle-on'
pci_wbox.children = [pci_options]
else:
pci_bt.icon = 'toggle-off'
pci_wbox.children = []
pts_bt.observe(pts_observe, names='value')
pci_bt.observe(pci_observe, names='value')
pts = VBox(children=[pts_bt, pts_wbox], layout=Layout(width='40%'))
pci = VBox(children=[pci_bt, pci_wbox], layout=Layout(width='40%'))
data_types = HBox(children=[pts, pci])
info_get = Label("4. Download the selected data.")
bt_get = Button(description='Download',
button_style='warning',
tooltip='Send the request',
icon='download')
path_temp = config.get_value(['paths', 'temp'])
path_data = config.get_value(['paths', 'data'])
info_paths = HTML("".join([
"<style>div.c {line-height: 1.1;}</style>",
"<div class='c';>By default data will be stored in the temp folder ",
f"({path_temp}), you will be asked to empty the temp folder each time ",
"you start the notebook.<br>In your personal data folder ",
f"({path_data}) you can permanently store the data.</div>"
]))
paths = RadioButtons(options=[
(f"Temporary folder: '{path_temp}'.", path_temp),
(f"Personal data folder: '{path_data}'.", path_data)
],
layout={'width': 'max-content'},
value=path_temp)
paths_box = Box([Label(value="Select folder:"), paths])
def file_len(fname):
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
def get_data(parcel):
get_requests = data_source()
pid = str(parcel['pid'][0])
source = config.get_value(['set', 'data_source'])
if source == 'api':
datapath = normpath(join(paths.value, aois.value, year.value, pid))
elif source == 'direct':
dataset = config.get_value(['set', 'dataset'])
datapath = normpath(join(paths.value, dataset, pid))
file_pinf = normpath(join(datapath, 'info.json'))
os.makedirs(dirname(file_pinf), exist_ok=True)
with open(file_pinf, "w") as f:
json.dump(parcel, f)
outlog(f"File saved at: {file_pinf}")
if pts_bt.value is True:
outlog(f"Getting time series for parcel: '{pid}',",
f"({pts_tstype.value} {pts_band.value}).")
for pts in pts_tstype.value:
ts = time_series.by_pid(aois.value, year.value, pid, pts,
ptype.value, pts_band.value)
band = ''
if pts_band.value != '':
band = f"_{pts_band.value}"
file_ts = normpath(
join(datapath, f'time_series_{pts}{band}.csv'))
if isinstance(ts, pd.DataFrame):
ts.to_csv(file_ts, index=True, header=True)
elif isinstance(ts, dict):
os.makedirs(os.path.dirname(file_ts), exist_ok=True)
df = pd.DataFrame.from_dict(ts, orient='columns')
df.to_csv(file_ts, index=True, header=True)
outlog("TS Files are saved.")
if pci_bt.value is True:
files_pci = normpath(join(datapath, 'chip_images'))
outlog(f"Getting '{pci_band.value}' chip images for parcel: {pid}")
with progress:
get_requests.rcbl(parcel, pci_start_date.value,
pci_end_date.value, pci_band.value,
pci_chipsize.value, files_pci)
filet = normpath(
join(datapath, 'chip_images',
f'images_list.{pci_band.value[0]}.csv'))
if file_len(filet) > 1:
outlog(
f"Completed, all GeoTIFFs for bands '{pci_band.value}' are ",
f"downloaded in the folder: '{datapath}/chip_images'")
else:
outlog(
"No files where downloaded, please check your configurations"
)
def get_from_location(lon, lat):
outlog(f"Finding parcel information for coordinates: {lon}, {lat}")
parcel = parcel_info.by_location(aois.value, year.value, lon, lat,
ptype.value, True, False, debug)
pid = str(parcel['pid'][0])
outlog(f"The parcel '{pid}' was found at this location.")
try:
get_data(parcel)
except Exception as err:
print(err)
def get_from_id(pids):
outlog(f"Getting parcels information for: '{pids}'")
for pid in pids:
try:
parcel = parcel_info.by_pid(aois.value, year.value, pid,
ptype.value, True, False, debug)
get_data(parcel)
except Exception as err:
print(err)
@bt_get.on_click
def bt_get_on_click(b):
progress.clear_output()
if method.value == 1:
try:
with progress:
lon, lat = plon.value, plat.value
get_from_location(lon, lat)
except Exception as err:
outlog("Could not get parcel information for location",
f"'{lon}', '{lat}': {err}")
elif method.value == 2:
try:
with progress:
pids = pid.value.replace(" ", "").split(",")
get_from_id(pids)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 3:
try:
marker = get_maps.base_map.map_marker
lon = str(round(marker.location[1], 2))
lat = str(round(marker.location[0], 2))
get_from_location(lon, lat)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 4:
try:
plimit = int(values['set']['plimit'])
file = normpath(
join(config.get_value(['paths', 'temp']),
'pids_from_polygon.txt'))
with open(file, "r") as text_file:
pids = text_file.read().split('\n')
outlog("Geting data form the parcels:")
outlog(pids)
if len(pids) <= plimit:
get_from_id(pids)
else:
outlog(
"You exceeded the maximum amount of selected parcels ",
f"({plimit}) to get data. Please select smaller area.")
except Exception as err:
outlog("No pids file found.", err)
else:
outlog(f"Please select method to get parcel information.")
return VBox([
info, table_options, info_method, method, method_out, info_type,
data_types, info_get, info_paths, paths_box, bt_get, progress
])
def extraction():
start = DatePicker(description='Start date:', disabled=False)
end = DatePicker(description='End date:', disabled=False)
dates = HBox([start, end])
bt_sig_s2 = Button(description='Sentinel 2',
value=False,
disabled=False,
button_style='info',
tooltip='Run Sentinel 2 extraction',
icon='fa-play')
bt_sig_bs = Button(description='S1 Backscattering',
value=False,
disabled=True,
button_style='info',
tooltip='Run S1 Backscattering extraction',
icon='fa-play')
bt_sig_c6 = Button(description='S1 6-day coherence',
value=False,
disabled=True,
button_style='info',
tooltip='Run S1 6-day coherence extraction',
icon='fa-play')
progress = Output()
def outlog(*text):
with progress:
print(*text)
bt_sg_box = VBox([bt_sig_s2, bt_sig_bs, bt_sig_c6])
tab_box = Tab(children=[bt_sg_box, Text()])
tab_box.set_title(0, 'Procedures')
tab_box.set_title(1, 'VM Workers config')
@bt_sig_s2.on_click
def bt_sig_s2_on_click(b):
progress.clear_output()
with progress:
try:
delta = end.value - start.value
for d in range(delta.days):
pgS2Extract.main(start.value, end.value)
except Exception as err:
outlog(err)
@bt_sig_bs.on_click
def bt_sig_bs_on_click(b):
progress.clear_output()
with progress:
try:
delta = end.value - start.value
for d in range(delta.days):
pgS1bsExtract.extractS1bs(start.value, end.value)
except Exception as err:
outlog(err)
@bt_sig_c6.on_click
def bt_sig_c6_on_click(b):
progress.clear_output()
with progress:
outlog(
"6-day coherence extraction not yet available please check for updates."
)
return VBox([dates, bt_sg_box, progress])
layergeojson=GeoJSON(data=layer)
map.add_layer(layergeojson)
p = Point([-106.578677,35.062485])
cursor.execute("SELECT ST_AsGeoJSON(incidents.geom), ST_Distance(incidents.geom::geography,ST_GeometryFromText('{}')::geography) from incidents ORDER BY incidents.geom<->ST_GeometryFromText('{}') LIMIT 15".format(p.wkt,p.wkt))
c=cursor.fetchall()
for x in c:
layer=json.loads(x[0])
layergeojson=GeoJSON(data=layer)
map.add_layer(layergeojson)
from ipywidgets import interact, interactive, fixed, interact_manual,DatePicker
import ipywidgets as widgets
@widgets.interact(x=DatePicker())
def theDate(x):
if x:
for l in map.layers[1:]:
map.remove_layer(l)
nohyphen=str(x).replace("-","")
d=datetime.datetime.strptime(nohyphen,'%Y%m%d').date()
cursor.execute("SELECT ST_AsGeoJSON(geom) from incidents where date = '{}' ".format(str(d)))
c=cursor.fetchall()
for x in c:
layer=json.loads(x[0])
layergeojson=GeoJSON(data=layer)
map.add_layer(layergeojson)
return len(c)
else:
pass
class NasaGibsViewer(PanelObject):
def __init__(self,ptype='time',*args,**kwargs):
self.title = 'NASA GIBS Image Viewer'
PanelObject.__init__(self,*args, **kwargs)
self.url = 'https://gibs.earthdata.nasa.gov/wmts/epsg4326/best/wmts.cgi'
self.plat = 24.42
self.plon = 54.43
self.height = 1.0
self.width = 1.0
def getCP(self):
self.setLabel()
self.imgDate = datetime.now()
self.selectVar ='MODIS_Terra_SurfaceReflectance_Bands143'
self.dateSW = DatePicker(description='Date',
layout=Layout(width='220px'),
value = self.imgDate,
disabled=False)
self.vardict = {'TerraRGB':'MODIS_Terra_SurfaceReflectance_Bands143',
'AquaRGB' :'MODIS_Aqua_SurfaceReflectance_Bands143',
'AquaLST Day':'MODIS_Aqua_Land_Surface_Temp_Day'}
self.varSW = Dropdown(options=['TerraRGB','AquaRGB','AquaLST Day'],
value='TerraRGB',
layout=Layout(width='220px'),
description='Var:',
disabled=False
)
self.latSW = Text(description='Lat:',disabled=False,value='24.42',layout=Layout(width='220px'))
self.lonSW = Text(description='Lon:',disabled=False,value='54.43',layout=Layout(width='220px'))
self.plotPB =Button(description='Plot',disabled=False,layout={'width':'auto','border':'3px outset'})
self.latRS = FloatSlider( value=5,
min=2.0,
max=15,
step=0.2,
description='Width',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.1f',
)
self.lonRS = FloatSlider( value=5.0,
min=2.0,
max=15.0,
step=0.2,
description='Height:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.1f'
)
self.inpUSR.children+= (HBox([
VBox([self.plotPB]),
VBox([self.dateSW,self.latSW,self.lonSW]),
VBox([self.latRS,self.lonRS,self.varSW ])
],
layout={'overflow':'visible'}
),)
self.dateSW.observe(self.dateSWCB)
self.varSW.observe(self.varSWCB,names='value')
self.latSW.observe(self.latSWCB,names='value')
self.lonSW.observe(self.lonSWCB,names='value')
self.latRS.observe(self.latRSCB,names='value')
self.lonRS.observe(self.lonRSCB,names='value')
self.plotPB.on_click(self.plotGIBS)
return self.cp
def dateSWCB(self,change):
if change['type'] == 'change':
self.imgDate = self.dateSW.value
def varSWCB(self,b):
self.selectVar = self.vardict[self.varSW.value]
def latSWCB(self,change):
self.plat = float(self.latSW.value)
def lonSWCB(self,change):
self.plon = float(self.lonSW.value)
def latRSCB(self,change):
self.height = self.latRS.value*0.5
def lonRSCB(self,change):
self.width = self.lonRS.value*0.5
def plotGIBS(self,b):
self.lat1 = self.plat - self.height
self.lat2 = self.plat + self.height
self.lon1 = self.plon - self.width
self.lon2 = self.plon + self.width
with self.out_cp:
plt.ioff()
layer = self.selectVar
date_str = '{}-{}-{}'.format(self.imgDate.strftime('%Y'),
self.imgDate.strftime('%m'),
self.imgDate.strftime('%d')
)
print(layer,date_str)
fig = plt.figure(figsize=(8, 8))
ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
ax.add_wmts(self.url, layer,wmts_kwargs={'time': date_str})
self.out_cp.clear_output()
ax.set_extent([self.lon1, self.lon2, self.lat1, self.lat2], crs=ccrs.PlateCarree())
ax.coastlines(resolution='50m', color='yellow')
plt.plot(self.plon, self.plat, marker='o', color='red', markersize=15,
alpha=1.0)
plt.show()
def get():
"""Get the parcel's dataset for the given location or ids"""
info = Label(
"1. Select the region and the year to get parcel information.")
values = config.read()
# Set the max number of parcels that can be downloaded at once.
plimit = int(values['set']['plimit'])
def aois_options():
values = config.read()
options = {}
if values['set']['data_source'] == '0':
for desc in values['api']['options']['aois']:
aoi = f"{values['api']['options']['aois'][desc]}"
options[(desc, aoi)] = values['api']['options']['years'][aoi]
elif values['set']['data_source'] == '1':
for aoi in values['ds_conf']:
desc = f"{values['ds_conf'][aoi]['desc']}"
confgs = values['ds_conf'][aoi]['years']
options[(f'{desc} ({aoi})', aoi)] = [y for y in confgs]
return options
def aois_years():
values = config.read()
years = {}
if values['set']['data_source'] == '0':
for desc in values['api']['options']['aois']:
aoi = values['api']['options']['aois'][desc]
years[aoi] = values['api']['options']['years'][aoi]
elif values['set']['data_source'] == '1':
for aoi in values['ds_conf']:
desc = f"{values['ds_conf'][aoi]['desc']}"
years[aoi] = [y for y in values['ds_conf'][aoi]['years']]
return years
try:
aois = Dropdown(
options=tuple(aois_options()),
value=values['set']['ds_conf'],
description='AOI:',
disabled=False,
)
except:
aois = Dropdown(
options=tuple(aois_options()),
description='AOI:',
disabled=False,
)
year = Dropdown(
options=next(iter(aois_options().values())),
description='Year:',
disabled=False,
)
button_refresh = Button(layout=Layout(width='35px'), icon='fa-refresh')
@button_refresh.on_click
def button_refresh_on_click(b):
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
def table_options_change(change):
try:
year.options = aois_years()[change.new]
except:
aois.options = tuple(aois_options())
year.options = aois_years()[aois.value]
aois.observe(table_options_change, 'value')
info_method = Label("2. Select a method to get the data.")
method = ToggleButtons(
options=[('Parcel ID', 2), ('Coordinates', 1), ('Map marker', 3),
('Polygon', 4)],
value=None,
description='',
disabled=False,
button_style='info',
tooltips=[
'Enter lat lon', 'Enter parcel ID', 'Select a point on a map',
'Get parcels id in a polygon'
],
)
plon = Text(value='5.664',
placeholder='Add lon',
description='Lon:',
disabled=False)
plat = Text(value='52.694',
placeholder='Add lat',
description='Lat:',
disabled=False)
wbox_lat_lot = VBox(children=[plat, plon])
info_pid = Label(
"Multiple parcel id codes can be added (comma ',' separated, e.g.: 11111, 22222)."
)
pid = Textarea(value='34296',
placeholder='12345, 67890',
description='Parcel(s) ID:',
disabled=False)
wbox_pids = VBox(children=[info_pid, pid])
bt_get_ids = Button(description="Find parcels",
disabled=False,
button_style='info',
tooltip='Find parcels within the polygon.',
icon='')
get_ids_box = HBox(
[bt_get_ids,
Label("Find the parcels that are in the polygon.")])
ppoly_out = Output()
progress = Output()
def outlog(*text):
with progress:
print(*text)
def outlog_poly(*text):
with ppoly_out:
print(*text)
@bt_get_ids.on_click
def bt_get_ids_on_click(b):
with ppoly_out:
try:
get_requests = data_source()
ppoly_out.clear_output()
polygon = get_maps.polygon_map.feature_collection['features'][
-1]['geometry']['coordinates'][0]
polygon_str = '-'.join(
['_'.join(map(str, c)) for c in polygon])
outlog_poly(f"Geting parcel ids within the polygon...")
polyids = json.loads(
get_requests.ppoly(aois.value, year.value, polygon_str,
False, True))
outlog_poly(
f"'{len(polyids['ogc_fid'])}' parcels where found:")
outlog_poly(polyids['ogc_fid'])
file = config.get_value(['files', 'pids_poly'])
with open(file, "w") as text_file:
text_file.write('\n'.join(map(str, polyids['ogc_fid'])))
except Exception as err:
outlog("No parcel ids found:", err)
method_out = Output(layout=Layout(border='1px solid black'))
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 1:
display(wbox_lat_lot)
elif obj['new'] == 2:
display(wbox_pids)
elif obj['new'] == 3:
display(
get_maps.base_map(
aois.value,
int(config.get_value(['set', 'data_source']))))
elif obj['new'] == 4:
display(
VBox([
get_maps.polygon(
aois.value,
int(config.get_value(['set', 'data_source']))),
get_ids_box, ppoly_out
]))
method.observe(method_options, 'value')
info_type = Label("3. Select datasets to download.")
table_options = HBox([aois, button_refresh, year])
# ########### Time series options #########################################
pts_bt = ToggleButton(
value=False,
description='Time series',
disabled=False,
button_style='success', # success
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pts_bands = data_options.pts_bands()
pts_tstype = SelectMultiple(
options=data_options.pts_tstype(),
value=['s2'],
rows=3,
description='TS type:',
disabled=False,
)
pts_band = Dropdown(
options=list(pts_bands['s2']),
value='',
description='Band:',
disabled=False,
)
def pts_tstype_change(change):
if len(pts_tstype.value) <= 1:
pts_band.disabled = False
try:
pts_b = change.new[0]
pts_band.options = pts_bands[pts_b]
except:
pass
else:
pts_band.value = ''
pts_band.disabled = True
pts_tstype.observe(pts_tstype_change, 'value')
pts_options = VBox(children=[pts_tstype, pts_band])
# ########### Chip images options #########################################
pci_bt = ToggleButton(value=False,
description='Chip images',
disabled=False,
button_style='success',
tooltip='Get parcel information',
icon='toggle-off',
layout=Layout(width='50%'))
pci_start_date = DatePicker(value=datetime.date(2019, 6, 1),
description='Start Date',
disabled=False)
pci_end_date = DatePicker(value=datetime.date(2019, 6, 30),
description='End Date',
disabled=False)
pci_plevel = RadioButtons(
options=['LEVEL2A', 'LEVEL1C'],
value='LEVEL2A',
description='Proces. level:', # Processing level
disabled=False,
layout=Layout(width='50%'))
pci_chipsize = IntSlider(value=640,
min=100,
max=5120,
step=10,
description='Chip size:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='d')
pci_bands = data_options.pci_bands()
pci_satellite = RadioButtons(options=list(pci_bands),
value='Sentinel 2',
disabled=True,
layout=Layout(width='100px'))
pci_band = SelectMultiple(options=list(pci_bands['Sentinel 2']),
value=['B04'],
rows=11,
description='Band:',
disabled=False)
sats_plevel = HBox([pci_satellite, pci_plevel])
def on_sat_change(change):
sat = change.new
pci_band.options = pci_bands[sat]
pci_satellite.observe(on_sat_change, 'value')
pci_options = VBox(children=[
pci_start_date, pci_end_date, sats_plevel, pci_chipsize, pci_band
])
# ########### General options #############################################
pts_wbox = VBox(children=[])
pci_wbox = VBox(children=[])
def pts_observe(button):
if button['new']:
pts_bt.icon = 'toggle-on'
pts_wbox.children = [pts_options]
else:
pts_bt.icon = 'toggle-off'
pts_wbox.children = []
def pci_observe(button):
if button['new']:
pci_bt.icon = 'toggle-on'
pci_wbox.children = [pci_options]
else:
pci_bt.icon = 'toggle-off'
pci_wbox.children = []
pts_bt.observe(pts_observe, names='value')
pci_bt.observe(pci_observe, names='value')
pts = VBox(children=[pts_bt, pts_wbox], layout=Layout(width='40%'))
pci = VBox(children=[pci_bt, pci_wbox], layout=Layout(width='40%'))
data_types = HBox(children=[pts, pci])
info_get = Label("4. Download the selected data.")
bt_get = Button(description='Download',
disabled=False,
button_style='warning',
tooltip='Send the request',
icon='download')
path_temp = config.get_value(['paths', 'temp'])
path_data = config.get_value(['paths', 'data'])
info_paths = HTML("".join([
"<style>div.c {line-height: 1.1;}</style>",
"<div class='c';>By default data will be stored in the temp folder ",
f"({path_temp}), you will be asked to empty the temp folder each time ",
"you start the notebook.<br>In your personal data folder ",
f"({path_data}) you can permanently store the data.</div>"
]))
paths = RadioButtons(options=[
(f"Temporary folder: '{path_temp}'.", path_temp),
(f"Personal data folder: '{path_data}'.", path_data)
],
layout={'width': 'max-content'},
value=path_temp)
paths_box = Box([Label(value="Select folder:"), paths])
def file_len(fname):
with open(fname) as f:
for i, l in enumerate(f):
pass
return i + 1
def get_data(parcel):
values = config.read()
get_requests = data_source()
pid = parcel['ogc_fid'][0]
source = int(config.get_value(['set', 'data_source']))
if source == 0:
datapath = f'{paths.value}{aois.value}{year.value}/parcel_{pid}/'
elif source == 1:
ds_conf = config.get_value(['set', 'ds_conf'])
datapath = f'{paths.value}{ds_conf}/parcel_{pid}/'
file_pinf = f"{datapath}{pid}_information"
outlog(data_handler.export(parcel, 10, file_pinf))
if pts_bt.value is True:
outlog(f"Getting time series for parcel: '{pid}',",
f"({pts_tstype.value} {pts_band.value}).")
for pts in pts_tstype.value:
ts = json.loads(
get_requests.pts(aois.value, year.value, pid, pts,
pts_band.value))
band = ''
if pts_band.value != '':
band = f"_{pts_band.value}"
file_ts = f"{datapath}{pid}_time_series_{pts}{band}"
outlog(data_handler.export(ts, 11, file_ts))
if pci_bt.value is True:
files_pci = f"{datapath}{pid}_chip_images/"
outlog(f"Getting '{pci_band.value}' chip images for parcel: {pid}")
with progress:
get_requests.rcbl(parcel, pci_start_date.value,
pci_end_date.value, pci_band.value,
pci_satellite.value, pci_chipsize.value,
files_pci)
filet = f'{datapath}/{pid}_chip_images/{pid}_images_list.{pci_band.value[0]}.csv'
if file_len(filet) > 1:
outlog(
f"Completed, all GeoTIFFs for bands '{pci_band.value}' are ",
f"downloaded in the folder: '{datapath}/{pid}_chip_images'"
)
else:
outlog(
"No files where downloaded, please check your configurations"
)
def get_from_location(lon, lat):
get_requests = data_source()
outlog(f"Finding parcel information for coordinates: {lon}, {lat}")
parcel = json.loads(
get_requests.ploc(aois.value, year.value, lon, lat, True))
pid = parcel['ogc_fid'][0]
outlog(f"The parcel '{pid}' was found at this location.")
try:
get_data(parcel)
except Exception as err:
print(err)
def get_from_id(pids):
get_requests = data_source()
outlog(f"Getting parcels information for: '{pids}'")
for pid in pids:
try:
parcel = json.loads(
get_requests.pid(aois.value, year.value, pid, True))
get_data(parcel)
except Exception as err:
print(err)
@bt_get.on_click
def bt_get_on_click(b):
progress.clear_output()
if method.value == 1:
try:
with progress:
get_requests = data_source()
lon, lat = plon.value, plat.value
get_from_location(lon, lat)
except Exception as err:
outlog(
f"Could not get parcel information for location '{lon}', '{lat}': {err}"
)
elif method.value == 2:
try:
with progress:
pids = pid.value.replace(" ", "").split(",")
get_from_id(pids)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 3:
try:
marker = get_maps.base_map.map_marker
lon = str(round(marker.location[1], 2))
lat = str(round(marker.location[0], 2))
get_from_location(lon, lat)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 4:
try:
file = config.get_value(['files', 'pids_poly'])
with open(file, "r") as text_file:
pids = text_file.read().split('\n')
outlog("Geting data form the parcels:")
outlog(pids)
if len(pids) <= plimit:
get_from_id(pids)
else:
outlog(
"You exceeded the maximum amount of selected parcels ",
f"({plimit}) to get data. Please select smaller area.")
except Exception as err:
outlog("No pids file found.", err)
else:
outlog(f"Please select method to get parcel information.")
return VBox([
info, table_options, info_method, method, method_out, info_type,
data_types, info_get, info_paths, paths_box, bt_get, progress
])
class MerraAQTseries(PanelObject):
def __init__(self, ptype='time', *args, **kwargs):
self.title = 'MERRA2 AQ Time Series'
self.stateChange = True
self.init = True
self.progVal = -1
self.progMax = 1
PanelObject.__init__(self, *args, **kwargs)
self.ptype = ptype
self.baseURL = 'https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXAER.5.12.4'
self.lat = 24.42
self.lon = 54.43
self.endDate = datetime.now()
self.startDate = self.endDate - timedelta(days=1)
self.endDate = datetime(2020, 4, 30)
self.startDate = datetime(2020, 4, 20)
self.varlab = {
'AOD': 'AOD (Unitless)',
'DUST_PM': r'Dust PM$_{2.5}(\mu g m^{-3})$',
'SALT_PM': r'Salt PM$_{2.5}(\mu g m^{-3})$',
'ORG_CARB': r'Org Carbon PM$_{2.5} (\mu g m^{-3})$',
'BLK_CARB': r'Black Carbon PM$_{2.5}(\mu g m^{-3})$',
'SO4': r'SO$_4 PM_{2.5} (\mu g m^{-3})$',
'PM2.5': r'Total PM$_{2.5}(\mu g m^{-3})$'
}
def initSession(self):
username = self.pwdDict['NASA Earth Data']['user']
password = self.pwdDict['NASA Earth Data']['password']
self.session = setup_session(username,
password,
check_url=self.baseURL)
firstDay = datetime(1980, 1, 1)
dataset = open_url(self.getUrlMERRA(firstDay), session=self.session)
self.lon = dataset['lon'][:]
self.lat = dataset['lat'][:]
def getCP(self):
self.setLabel()
self.plon = 54.43
self.plat = 24.42
self.dateSelection = datetime.now()
self.dateLast = datetime(1950, 1, 1)
self.selectVar = 'AOD'
self.selectTime = 0
self.sdateSW = DatePicker(description='Start Date',
layout=Layout(width='auto'),
value=self.startDate,
disabled=False)
self.edateSW = DatePicker(description='End Date',
layout=Layout(width='auto'),
value=self.endDate,
disabled=False)
self.varSW = Dropdown(options=[
'AOD', 'DUST_PM', 'SALT_PM', 'ORG_CARB', 'BLK_CARB', 'SO4', 'PM2.5'
],
value='AOD',
layout=Layout(width='280px'),
description='Variable:',
disabled=False)
self.latSW = Text(description='Latitude:',
disabled=False,
value='24.42',
layout=Layout(width='180px'))
self.lonSW = Text(description='Longitude:',
disabled=False,
value='54.43',
layout=Layout(width='180px'))
self.plotPB = Button(description='Time Series Plot',
disabled=False,
layout={
'width': 'auto',
'border': '3px outset'
})
self.inpUSR.children += (HBox([
VBox([self.sdateSW, self.edateSW, self.varSW]),
VBox([self.latSW, self.lonSW, self.plotPB])
],
layout={'overflow': 'visible'}), )
self.sdateSW.observe(self.sdateSWCB)
self.edateSW.observe(self.edateSWCB)
self.varSW.observe(self.varSWCB, names='value')
self.latSW.observe(self.latSWCB, names='value')
self.lonSW.observe(self.lonSWCB, names='value')
self.plotPB.on_click(self.plotTS)
return self.cp
def sdateSWCB(self, change):
if change['type'] == 'change':
self.startDate = self.sdateSW.value
self.stateChange = True
def edateSWCB(self, change):
if change['type'] == 'change':
self.endDate = self.edateSW.value
self.stateChange = True
def varSWCB(self, b):
self.selectVar = self.varSW.value
def latSWCB(self, change):
self.plat = float(self.latSW.value)
self.stateChange = True
def lonSWCB(self, change):
self.plon = float(self.lonSW.value)
self.stateChange = True
def lonlatToIndex(self, plon, plat):
self.ilat = int(
np.interp(plat, (self.lat.data.min(), self.lat.data.max()),
(0, self.lat.shape[0] - 1)))
self.ilon = int(
np.interp(plon, (self.lon.data.min(), self.lon.data.max()),
(0, self.lon.shape[0] - 1)))
def updateProg(self, prog):
while self.progVal < self.progMax:
if self.progVal != prog.value:
prog.value = self.progVal
def getTS(self, startDate, endDate):
ndays = (endDate - startDate).days + 1
if self.init:
ndays += 1
currentDate = startDate
delta = timedelta(days=1)
self.df = pd.DataFrame(columns=[
'datetime', 'AOD', 'DUST_PM', 'SALT_PM', 'ORG_CARB', 'BLK_CARB',
'SO4', 'PM2.5'
])
with self.out_cp:
self.out_cp.clear_output()
pbar = IntProgress(min=0, max=int(ndays))
pbar.description = 'Progress:'
info1 = Label('0%')
info2 = Label(' ')
display(
VBox([
HBox([pbar, info1]),
HBox([info2], layout=Layout(justify_content='center'))
]))
progVal = 0
if self.init:
info2.value = 'Initializing NASA Earth Data Connection..'
self.initSession()
self.init = False
pbar.value += 1
progVal += 1
info1.value = '{:.1f}%'.format(
(float(progVal) / float(ndays)) * 100.0)
self.lonlatToIndex(self.plon, self.plat)
while currentDate <= endDate:
url = self.getUrlMERRA(currentDate)
info2.value = 'Accessing data for {}'.format(currentDate)
dataset = open_url(url, session=self.session)
aod = np.squeeze(dataset['TOTEXTTAU'][:, self.ilat, self.ilon])
dust_pm = np.squeeze(
dataset['DUSMASS25'][:, self.ilat,
self.ilon]) * 1000000000.0
salt_pm = np.squeeze(
dataset['SSSMASS25'][:, self.ilat,
self.ilon]) * 1000000000.0
org_carb = np.squeeze(
dataset['OCSMASS'][:, self.ilat, self.ilon]) * 1000000000.0
blk_carb = np.squeeze(
dataset['BCSMASS'][:, self.ilat, self.ilon]) * 1000000000.0
so4 = np.squeeze(dataset['SO4SMASS'][:, self.ilat,
self.ilon]) * 1000000000.0
pm25 = (1.375 * so4 + 1.6 * org_carb + blk_carb + dust_pm +
salt_pm)
dt = pd.date_range(currentDate, periods=24, freq='H')
vardict = {
'datetime': dt,
'AOD': aod,
'DUST_PM': dust_pm,
'SALT_PM': salt_pm,
'ORG_CARB': org_carb,
'BLK_CARB': blk_carb,
'SO4': so4,
'PM2.5': pm25
}
df_add = pd.DataFrame(vardict)
self.df = pd.concat([self.df, df_add])
currentDate += delta
progVal += 1
info1.value = '{:.1f}%'.format(
(float(progVal) / float(ndays)) * 100.0)
pbar.value += 1
self.stateChange = False
def plotTS(self, b):
if self.stateChange:
self.getTS(self.startDate, self.endDate)
with self.out_cp:
plt.ioff()
fig, ax = plt.subplots(1, figsize=(12, 6))
ax = self.df.plot(ax=ax, x='datetime', y=self.selectVar)
ax.set_xlabel('Time (UTC)')
ax.set_ylabel(self.varlab[self.selectVar])
self.out_cp.clear_output()
plt.show()
def getUrlMERRA(self, dayObj):
if dayObj.year < 1992: Vnumber = '100'
elif dayObj.year < 2001: Vnumber = '200'
elif dayObj.year < 2011: Vnumber = '300'
else: Vnumber = '400'
yy = dayObj.strftime('%Y')
mm = dayObj.strftime('%m')
dd = dayObj.strftime('%d')
self.opendap_url = self.baseURL + '/{}/{}/MERRA2_{}.tavg1_2d_aer_Nx.{}{}{}.nc4'.format(
yy, mm, Vnumber, yy, mm, dd)
return self.opendap_url
def getCP(self):
self.setLabel()
self.plon = 0.0
self.plat = 0.0
self.dateSelection = datetime.now()
self.dateSelection = datetime(2020, 4, 23)
self.dateLast = datetime(1950, 1, 1)
self.selectVar = 'AOD'
self.selectTime = 0
self.dateSW = DatePicker(description='Date',
layout=Layout(width='250px'),
value=self.dateSelection,
disabled=False)
self.myWidget1 = Dropdown(options=[
'AOD', 'DUST_PM', 'SALT_PM', 'ORG_CARB', 'BLK_CARB', 'SO4', 'PM2.5'
],
value='AOD',
layout=Layout(width='250px'),
description='Varibale:',
disabled=False)
self.myWidget2 = Dropdown(options=[],
value=None,
layout=Layout(width='250px'),
description='Varibale:',
disabled=False)
self.myWidget3 = Dropdown(options=[],
value=None,
description='Date:',
disabled=False)
self.myWidget4 = Dropdown(options=[],
value=None,
layout=Layout(width='250px'),
description='Time:',
disabled=False)
self.latSW = FloatSlider(min=-90.0,
max=90.0,
step=0.25,
description='Lat:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.2f')
self.lonSW = FloatSlider(min=-180.0,
max=180.0,
step=0.25,
description='Lon:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.2f')
self.plotBW = Button(description='Spatial Plot',
disabled=False,
layout={
'width': '200px',
'border': '3px outset'
})
if self.ptype == 'space':
self.inpUSR.children += (HBox([
VBox([self.dateSW, self.myWidget4, self.myWidget1]),
VBox([self.plotBW])
],
layout={'overflow': 'visible'}), )
else:
self.inpUSR.children += (HBox([
VBox([self.dateSW, self.myWidget1]),
VBox([self.timeplot, self.latSW, self.lonSW])
],
layout={'overflow': 'visible'}), )
self.year = np.arange(41) + 1980
self.mm = np.arange(12) + 1
# formatter = "{:02d}".format
# self.months=[formatter(item) for item in mm]
self.dateSW.observe(self.dateSWCB)
self.myWidget2.options = self.mm
self.myWidget1.observe(self.myCallback1, names='value')
self.myWidget2.observe(self.myCallback2, names='value')
self.date = np.arange(1, 30) + 1
self.time = np.arange(24)
self.myWidget3.options = self.date
self.myWidget4.options = self.time
self.myWidget3.observe(self.myCallback3, names='value')
self.myWidget4.observe(self.myCallback4, names='value')
self.latSW.observe(self.latSWCB, names='value')
self.lonSW.observe(self.lonSWCB, names='value')
self.plotBW.on_click(self.plotObs)
return self.cp
class MerraAQSpatial(PanelObject):
def __init__(self, ptype='space', *args, **kwargs):
self.title = 'MERRA2 AQ Spatial Plots'
PanelObject.__init__(self, *args, **kwargs)
self.ptype = ptype
def getCP(self):
self.setLabel()
self.plon = 0.0
self.plat = 0.0
self.dateSelection = datetime.now()
self.dateSelection = datetime(2020, 4, 23)
self.dateLast = datetime(1950, 1, 1)
self.selectVar = 'AOD'
self.selectTime = 0
self.dateSW = DatePicker(description='Date',
layout=Layout(width='250px'),
value=self.dateSelection,
disabled=False)
self.myWidget1 = Dropdown(options=[
'AOD', 'DUST_PM', 'SALT_PM', 'ORG_CARB', 'BLK_CARB', 'SO4', 'PM2.5'
],
value='AOD',
layout=Layout(width='250px'),
description='Varibale:',
disabled=False)
self.myWidget2 = Dropdown(options=[],
value=None,
layout=Layout(width='250px'),
description='Varibale:',
disabled=False)
self.myWidget3 = Dropdown(options=[],
value=None,
description='Date:',
disabled=False)
self.myWidget4 = Dropdown(options=[],
value=None,
layout=Layout(width='250px'),
description='Time:',
disabled=False)
self.latSW = FloatSlider(min=-90.0,
max=90.0,
step=0.25,
description='Lat:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.2f')
self.lonSW = FloatSlider(min=-180.0,
max=180.0,
step=0.25,
description='Lon:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.2f')
self.plotBW = Button(description='Spatial Plot',
disabled=False,
layout={
'width': '200px',
'border': '3px outset'
})
if self.ptype == 'space':
self.inpUSR.children += (HBox([
VBox([self.dateSW, self.myWidget4, self.myWidget1]),
VBox([self.plotBW])
],
layout={'overflow': 'visible'}), )
else:
self.inpUSR.children += (HBox([
VBox([self.dateSW, self.myWidget1]),
VBox([self.timeplot, self.latSW, self.lonSW])
],
layout={'overflow': 'visible'}), )
self.year = np.arange(41) + 1980
self.mm = np.arange(12) + 1
# formatter = "{:02d}".format
# self.months=[formatter(item) for item in mm]
self.dateSW.observe(self.dateSWCB)
self.myWidget2.options = self.mm
self.myWidget1.observe(self.myCallback1, names='value')
self.myWidget2.observe(self.myCallback2, names='value')
self.date = np.arange(1, 30) + 1
self.time = np.arange(24)
self.myWidget3.options = self.date
self.myWidget4.options = self.time
self.myWidget3.observe(self.myCallback3, names='value')
self.myWidget4.observe(self.myCallback4, names='value')
self.latSW.observe(self.latSWCB, names='value')
self.lonSW.observe(self.lonSWCB, names='value')
self.plotBW.on_click(self.plotObs)
return self.cp
def dateSWCB(self, change):
if change['type'] == 'change':
self.dateSelection = self.dateSW.value
def myCallback1(self, b):
self.selectVar = self.myWidget1.value
# print(self.selectYear)
def myCallback2(self, b):
formatter = "{:02d}".format
self.selectMonth = self.myWidget2.value
self.selectMonth = formatter(self.selectMonth)
# print(self.selectMonth)
def myCallback3(self, b):
formatter = "{:02d}".format
self.selectDate = formatter(self.myWidget3.value)
def myCallback4(self, b):
self.selectTime = self.myWidget4.value
def latSWCB(self, change):
self.plat = self.latSW.value
def lonSWCB(self, change):
self.plon = self.lonSW.value
def plotObs(self, b):
if self.dateSelection.year < 1992: self.Vnumber = '100'
elif self.dateSelection.year < 2001: self.Vnumber = '200'
elif self.dateSelection.year < 2011: self.Vnumber = '300'
else: self.Vnumber = '400'
self.selectYear = self.dateSelection.strftime('%Y')
self.selectMonth = self.dateSelection.strftime('%m')
self.selectDate = self.dateSelection.strftime('%d')
if self.dateSelection != self.dateLast:
username = self.pwdDict['NASA Earth Data']['user']
password = self.pwdDict['NASA Earth Data']['password']
with self.out_cp:
print('Accessing data...')
self.opendap_url = 'https://goldsmr4.gesdisc.eosdis.nasa.gov/opendap/MERRA2/M2T1NXAER.5.12.4/' + self.selectYear + '/' + self.selectMonth + '/MERRA2_' + self.Vnumber + '.tavg1_2d_aer_Nx.' + self.selectYear + self.selectMonth + self.selectDate + '.nc4'
session = setup_session(username,
password,
check_url=self.opendap_url)
dataset = open_url(self.opendap_url, session=session)
#dataset = open_url(opendap_url)
lon = dataset['lon'][:]
lat = dataset['lat'][:]
self.lons, self.lats = np.meshgrid(lon, lat)
t = int(self.selectTime)
aod = np.squeeze(dataset['TOTEXTTAU'][t, :, :], axis=0)
dust_pm = np.squeeze(dataset['DUSMASS25'][t, :, :],
axis=0) * 1000000000.0
salt_pm = np.squeeze(dataset['SSSMASS25'][t, :, :],
axis=0) * 1000000000.0
org_carb = np.squeeze(dataset['OCSMASS'][t, :, :],
axis=0) * 1000000000.0
blk_carb = np.squeeze(dataset['BCSMASS'][t, :, :],
axis=0) * 1000000000.0
so4 = np.squeeze(dataset['SO4SMASS'][t, :, :],
axis=0) * 1000000000.0
pm25 = (1.375 * so4 + 1.6 * org_carb + blk_carb + dust_pm +
salt_pm)
self.vardict = {
'AOD': aod,
'DUST_PM': dust_pm,
'SALT_PM': salt_pm,
'ORG_CARB': org_carb,
'BLK_CARB': blk_carb,
'SO4': so4,
'PM2.5': pm25
}
self.varlab = {
'AOD': 'Unitless',
'DUST_PM': r'$\mu g m^{-3}$',
'SALT_PM': r'$\mu g m^{-3}$',
'ORG_CARB': r'$\mu g m^{-3}$',
'BLK_CARB': r'$\mu g m^{-3}$',
'SO4': r'$\mu g m^{-3}$',
'PM2.5': r'$\mu g m^{-3}$'
}
var = self.vardict[self.selectVar]
# Set the figure size, projection, and extent
with self.out_cp:
plt.ioff()
self.out_cp.clear_output()
fig = plt.figure(figsize=(8, 4))
ax = plt.axes(projection=ccrs.PlateCarree())
ax.set_global()
#ax.set_extent([65, 100, 5.0,35.0])
ax.coastlines(resolution="110m", linewidth=1)
ax.gridlines(linestyle='--', color='black')
# Set contour levels, then draw the plot and a colorbar
clevs = np.arange(230, 311, 5)
plt.contourf(self.lons,
self.lats,
var,
transform=ccrs.PlateCarree(),
cmap=plt.cm.jet)
formatter = "{:02d}".format
tt = formatter(self.selectTime)
plt.title('MERRA2 ' + self.selectVar + ',' + tt + ':00 UTC ' +
self.selectYear + '-' + self.selectMonth + '-' +
self.selectDate,
size=14)
cb = plt.colorbar(ax=ax,
orientation="vertical",
pad=0.02,
aspect=16,
shrink=0.8)
cb.set_label(self.varlab[self.selectVar],
size=12,
rotation=90,
labelpad=15)
cb.ax.tick_params(labelsize=10)
plt.show()
self.dateLast = self.dateSelection
def main(conn):
def clear_map():
if 'm' in globals():
m.close()
else:
pass
def clear_lat_longs():
if 'lat_longs' in globals():
lat_longs = None
else:
pass
def clear_time_slider():
if 'time_slider' in globals():
time_slider = None
else:
pass
def draw_update_map(b):
clear_map()
clear_lat_longs()
plate = plate_dropdown.value
start_date_time = str(
start_date.value) + ' ' + start_time_dropdown.value
end_date_time = str(end_date.value) + ' ' + end_time_dropdown.value
global locations
locations = sfmta.get_points_for_shuttle(conn, plate, start_date_time,
end_date_time)
zip_locations = zip(locations['lng'], locations['lat'],
locations.index)
global lat_longs
lat_longs = [(x[0], x[1]) for x in zip_locations]
index_value = find_index(lat_longs, time_slider.value)
global m
m = draw_map(lat_longs[:index_value])
display(m)
button = Button(description="Draw/Update Map")
button.on_click(draw_update_map)
time_slider = IntSlider(min=0,
max=100,
step=1,
description='% of Data',
value=100)
def find_index(lat_longs, value):
length = len(lat_longs)
index = int(length * value / 100)
return index
def draw_map(lat_longs):
center = [37.79481, -122.41186]
zoom = 12
m = Map(center=center, zoom=zoom, basemap=basemaps.Hydda.Full)
m.layout.height = '650px'
pl = Polyline(locations=lat_longs)
pl.color = path_color.value
pl.fill_color = path_color.value
m.add_layer(pl)
return m
plates = sfmta.get_all_shuttles(conn)['LICENSE_PLATE'].unique()
plate_dropdown = Dropdown(options=plates, description='Plate')
def show_restrictions(b):
polygons = './vehiclerestrictions_wgs.json'
with open(polygons) as f:
polygons_json = json.load(f)
global geojson
geojson = GeoJSON(data=polygons_json)
m.add_layer(geojson)
def remove_restrictions(b):
if 'geojson' in globals():
m.remove_layer(geojson)
else:
pass
def download_data(b):
if 'lat_longs' in globals():
df = pd.DataFrame(data=lat_longs)
df.to_excel('output.xlsx')
else:
pass
button_restrictions = Button(description="Show Restrictions")
button_restrictions.on_click(show_restrictions)
button_remove_restrictions = Button(description="Hide Restrictions")
button_remove_restrictions.on_click(remove_restrictions)
path_color = ColorPicker(description='Color')
export_data = Button(description="Download Data")
export_data.on_click(download_data)
def top_n_stops(b):
from collections import Counter
count_lat_longs = Counter(lat_longs).most_common()
for lat_long in count_lat_longs[0:10]:
location = lat_long[0]
circle = Circle(location=location, radius=300)
m.add_layer(circle)
return m
button_add_top_stops = Button(description="Display Bus Idling")
button_add_top_stops.on_click(top_n_stops)
# Remove map button needs its own function with a single argument
def remove_map(b):
clear_map()
button_clear_map = Button(description="Remove Map")
button_clear_map.on_click(remove_map)
start = datetime.date.today()
end = datetime.date.today()
start_date = DatePicker(description='Start Date', value=start)
end_date = DatePicker(description='End Date', value=end)
start_time_dropdown = time_drop_down()
start_time_dropdown.description = 'Start Time'
end_time_dropdown = time_drop_down()
end_time_dropdown.description = 'End Time'
companies = sfmta.get_shuttles_companies(conn)['NAME'].unique()
company = Dropdown(options=companies,
value='WeDriveU',
description='Company')
return VBox([
HBox(
[company, button, button_restrictions,
button_remove_restrictions]),
HBox([
plate_dropdown, export_data, button_add_top_stops, button_clear_map
]),
HBox([path_color]),
HBox([start_date, start_time_dropdown]),
HBox([end_date, end_time_dropdown]),
HBox([time_slider]),
])
def getCP(self):
self.setLabel()
self.imgDate = datetime.now()
self.selectVar ='MODIS_Terra_SurfaceReflectance_Bands143'
self.dateSW = DatePicker(description='Date',
layout=Layout(width='220px'),
value = self.imgDate,
disabled=False)
self.vardict = {'TerraRGB':'MODIS_Terra_SurfaceReflectance_Bands143',
'AquaRGB' :'MODIS_Aqua_SurfaceReflectance_Bands143',
'AquaLST Day':'MODIS_Aqua_Land_Surface_Temp_Day'}
self.varSW = Dropdown(options=['TerraRGB','AquaRGB','AquaLST Day'],
value='TerraRGB',
layout=Layout(width='220px'),
description='Var:',
disabled=False
)
self.latSW = Text(description='Lat:',disabled=False,value='24.42',layout=Layout(width='220px'))
self.lonSW = Text(description='Lon:',disabled=False,value='54.43',layout=Layout(width='220px'))
self.plotPB =Button(description='Plot',disabled=False,layout={'width':'auto','border':'3px outset'})
self.latRS = FloatSlider( value=5,
min=2.0,
max=15,
step=0.2,
description='Width',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.1f',
)
self.lonRS = FloatSlider( value=5.0,
min=2.0,
max=15.0,
step=0.2,
description='Height:',
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True,
readout_format='.1f'
)
self.inpUSR.children+= (HBox([
VBox([self.plotPB]),
VBox([self.dateSW,self.latSW,self.lonSW]),
VBox([self.latRS,self.lonRS,self.varSW ])
],
layout={'overflow':'visible'}
),)
self.dateSW.observe(self.dateSWCB)
self.varSW.observe(self.varSWCB,names='value')
self.latSW.observe(self.latSWCB,names='value')
self.lonSW.observe(self.lonSWCB,names='value')
self.latRS.observe(self.latRSCB,names='value')
self.lonRS.observe(self.lonRSCB,names='value')
self.plotPB.on_click(self.plotGIBS)
return self.cp
