def by_location(aoi, year, lon, lat, chipsize=512, extend=512,
tms=['Google'], ptype=None, axis=True, debug=False):
"""Download the background image with parcels polygon overlay by selected
location. This function will get an image from the center of the polygon.
Examples:
from cbm.view import background
background.by_location(aoi, lon, lat, 512, 512, 'Google',
True, True)
Arguments:
aoi, the area of interest (str)
year, the year of parcels table
lon, lat, longitude and latitude in decimal degrees (float).
chipsize, size of the chip in pixels (int).
extend, size of the chip in meters (float).
tms, tile map server Google or Bing (str).
debug, print or not procedure information (Boolean).
"""
get_requests = data_source()
if type(tms) is str:
tms = [tms]
try:
parcel = parcel_info.by_location(aoi, year, lon, lat, ptype,
True, False, debug)
if type(parcel['pid']) is list:
pid = parcel['pid'][0]
else:
pid = parcel['pid']
workdir = normpath(join(config.get_value(['paths', 'temp']),
aoi, str(year), str(pid)))
if debug:
print('pid: ', pid)
print('workdir: ', workdir)
except Exception as err:
workdir = normpath(join(config.get_value(['paths', 'temp']), aoi,
str(year), f'_{lon}_{lat}'.replace('.', '_')))
if debug:
print("No parcel information found.", err)
bg_path = normpath(join(workdir, 'backgrounds'))
os.makedirs(bg_path, exist_ok=True)
with open(f"{bg_path}/chipsize_extend_{chipsize}_{extend}", "w") as f:
f.write('')
if debug:
print('bg_path: ', bg_path)
print('lon, lat:', lon, lat)
for t in tms:
if debug:
print('lon, lat, chipsize, extend, t, bg_path, debug')
print(lon, lat, chipsize, extend, t, bg_path, debug)
get_requests.background(lon, lat, chipsize, extend, t, bg_path, debug)
def get_data(parcel):
get_requests = data_source()
pid = parcel['pid'][0]
source = config.get_value(['set', 'data_source'])
if source == 'api':
datapath = normpath(
join(paths.value, aois.value, year.value, str(pid)))
elif source == 'direct':
dataset = config.get_value(['set', 'dataset'])
datapath = normpath(join(paths.value, dataset, str(pid)))
file_pinf = normpath(join(datapath, 'info.json'))
os.makedirs(dirname(file_pinf), exist_ok=True)
with open(file_pinf, "w") as f:
json.dump(parcel, f)
outlog(f"File saved at: {file_pinf}")
if pts_bt.value is True:
outlog(f"Getting time series for parcel: '{pid}',",
f"({pts_tstype.value} {pts_band.value}).")
for pts in pts_tstype.value:
ts = time_series.by_pid(aois.value, year.value, pid, pts,
pts_band.value)
band = ''
if pts_band.value != '':
band = f"_{pts_band.value}"
file_ts = normpath(
join(datapath, f'time_series_{pts}{band}.csv'))
if isinstance(ts, pd.DataFrame):
ts.to_csv(file_ts, index=True, header=True)
elif isinstance(ts, dict):
os.makedirs(os.path.dirname(file_ts), exist_ok=True)
df = pd.DataFrame.from_dict(ts, orient='columns')
df.to_csv(file_ts, index=True, header=True)
outlog("TS Files are saved.")
if pci_bt.value is True:
files_pci = normpath(join(datapath, 'chip_images'))
outlog(f"Getting '{pci_band.value}' chip images for parcel: {pid}")
with progress:
get_requests.rcbl(parcel, pci_start_date.value,
pci_end_date.value, pci_band.value,
pci_chipsize.value, files_pci)
filet = normpath(
join(datapath, 'chip_images',
f'images_list.{pci_band.value[0]}.csv'))
if file_len(filet) > 1:
outlog(
f"Completed, all GeoTIFFs for bands '{pci_band.value}' are ",
f"downloaded in the folder: '{datapath}/chip_images'")
else:
outlog(
"No files where downloaded, please check your configurations"
)
def method_options(obj):
with method_out:
method_out.clear_output()
if obj['new'] == 1:
display(wbox_lat_lot)
elif obj['new'] == 2:
display(wbox_pids)
elif obj['new'] == 3:
display(get_maps.base_map(aois.value,
int(config.get_value(['set', 'data_source']))))
elif obj['new'] == 4:
display(VBox([get_maps.polygon(aois.value,
int(config.get_value(['set', 'data_source']))),
get_ids_box, ppoly_out]))
def peers(aoi,
year,
pid,
distance=1000.0,
maxPeers=10,
ptype=None,
debug=False):
"""Get the parcel peers for the selected parcel
Examples:
import cbm
cbm.get.parcels_list.peers(aoi, year, pid)
Arguments:
aoi, the area of interest (str)
year, the year of parcels table
pid, the parcel id (str).
distance, distance from the given parcel
maxPeers, the max number of parcels to be returned
"""
get_requests = data_source()
parcels = json.loads(
get_requests.parcel_peers(aoi, year, pid, distance, maxPeers, ptype,
debug))
workdir = normpath(
join(config.get_value(['paths', 'temp']), aoi, str(year), str(pid)))
json_file = normpath(join(workdir, f'parcel_peers.json'))
os.makedirs(workdir, exist_ok=True)
with open(json_file, "w") as f:
json.dump(parcels, f)
return parcels
def by_location(aoi,
year,
lon,
lat,
ptype=None,
geom=False,
wgs84=False,
debug=False):
"""Download the time series for the selected year
Examples:
import cbm
cbm.get.parcel.by_location(aoi, year, lon, lat)
Arguments:
aoi, the area of interest (str)
year, the year of parcels table
lon, lat, the the coords of the parcel (float).
"""
get_requests = data_source()
parcel = json.loads(
get_requests.parcel_by_loc(aoi, year, lon, lat, ptype, geom, wgs84,
debug))
if type(parcel['pid']) is list:
pid = parcel['pid'][0]
else:
pid = parcel['pid']
workdir = normpath(
join(config.get_value(['paths', 'temp']), aoi, str(year), str(pid)))
json_file = normpath(join(workdir, 'info.json'))
os.makedirs(workdir, exist_ok=True)
with open(json_file, "w") as f:
json.dump(parcel, f)
return parcel
def maps(aoi, pid, chipsize=512, extend=512, tms='Google'):
workdir = config.get_value(['paths', 'temp'])
path = f'{workdir}/{aoi}/{pid}/backgrounds/'
for t in tms:
if not os.path.isfile(f'{path}{t.lower()}.png'):
bg.by_pid(aoi, pid, chipsize, extend, t, True)
columns = 5
rows = int(len(tms) // columns + (len(tms) % columns > 0))
fig = plt.figure(figsize=(25, 5 * rows))
grid = ImageGrid(
fig,
111, # similar to subplot(111)
nrows_ncols=(rows, columns), # creates grid of axes
axes_pad=0.1, # pad between axes in inch.
)
for ax, im in zip(grid, tms):
# Iterating over the grid returns the Axes.
ax.axis('off')
ax.imshow(plt.imread(f'{path}{im.lower()}.png', 3))
ax.set_title(im)
plt.show()
def by_pid(aoi, year, pid, tstype, ptype=None, band='', debug=False):
"""Download the time series for the selected year
Examples:
import cbm
cbm.get.time_series.by_pid(aoi, pid, tstype, band, save)
Arguments:
aoi, the area of interest and year e.g.: es2019, nld2020 (str)
pid, the parcel id (int).
"""
get_requests = data_source()
workdir = config.get_value(['paths', 'temp'])
file_ts = normpath(
join(workdir, aoi, year, str(pid), f'time_series_{tstype}{band}.csv'))
ts = json.loads(
get_requests.parcel_ts(aoi, year, pid, tstype, ptype, band, debug))
if isinstance(ts, pd.DataFrame):
ts.to_csv(file_ts, index=True, header=True)
elif isinstance(ts, dict):
os.makedirs(os.path.dirname(file_ts), exist_ok=True)
df = pd.DataFrame.from_dict(ts, orient='columns')
df.to_csv(file_ts, index=True, header=True)
if debug:
print(f"File saved at: {file_ts}")
return ts
def main():
global __version__
if sys.version_info < (3, 6):
print("Not supoted python version, cbm needs python version > 3.6")
sys.exit()
try:
_dist = get_distribution('cbm')
# Normalize case for Windows systems
dist_loc = normcase(_dist.location)
here = normcase(__file__)
if not here.startswith(normpath(join(dist_loc, 'cbm'))):
# not installed, but there is another version that *is*
raise DistributionNotFound
except DistributionNotFound:
__version__ = 'Please install this project with setup.py'
else:
__version__ = _dist.version
paths = config.get_value(['paths'])
for p in paths:
try:
if not exists(normpath(join(config.path_work, paths[p]))):
os.makedirs(normpath(join(config.path_work, paths[p])))
print(f"The folder {p} is created successfully.")
except OSError as err:
print(f"The folder {p} can not be created: ", err)
config.create()
update.check()
def main():
source = config.get_value(['s3', 'dias'])
dias = Dropdown(
options=data_options.dias_providers(),
value='EOSC',
description='DIAS Provider:',
)
dias_box = VBox()
def dias_set(d):
if d in ['CREODIAS', 'EOSC']:
from cbm.ipycbm.ipy_ext.dias import creodias
dias_box.children = [creodias.main()]
elif d in ['SOBLOO']:
from cbm.ipycbm.ipy_ext.dias import sobloo
dias_box.children = [sobloo.main()]
elif d in ['MUNDI']:
dias_box.children = [Label('This provider is not supported yet')]
elif d in ['ONDA']:
dias_box.children = [Label('This provider is not supported yet')]
elif d in ['WEKEO']:
dias_box.children = [Label('This provider is not supported yet')]
dias_set(source)
def on_dias_change(change):
dias_box.children = []
dias_set(dias.value)
dias.observe(on_dias_change, names='value')
return VBox([dias, dias_box])
def set_api_account(url, username, password):
from cbm.utils import config
config.set_value(['api', 'url'], url)
config.set_value(['api', 'user'], username)
config.set_value(['api', 'pass'], password)
print("The 'api' key in config/main.json file is updated:")
print(config.get_value(['api']))
def by_pid(aoi,
year,
pid,
ptype,
start_date,
end_date,
band,
chipsize,
debug=False):
"""Download the chip image by selected parcel id.
Examples:
import cbm
cbm.get.chip_images.by_pid(aoi, pid, start_date, end_date,
band, chipsize)
Arguments:
aoi, the area of interest and year e.g.: es2019, nld2020 (str)
pid, the parcel id (int).
start_date, Start date '2019-06-01' (str)
end_date, End date '2019-06-01' (str)
band, 3 Sentinel-2 band names. One of [‘B02’, ‘B03’, ‘B04’, ‘B08’]
(10 m bands) or [‘B05’, ‘B06’, ‘B07’, ‘B8A’, ‘B11’, ‘B12’]
(20 m bands). 10m and 20m bands can be combined.
The first band determines the resolution in the output
composite. Defaults to B08_B04_B03.
chipsize, size of the chip in pixels (int).
"""
workdir = normpath(join(config.get_value(['paths', 'temp']), aoi,
str(pid)))
get_requests = data_source()
pfile = normpath(join(workdir, 'info.json'))
if not isfile(pfile):
os.makedirs(dirname(pfile), exist_ok=True)
parcel = json.loads(
get_requests.parcel_by_id(aoi, year, ptype, pid, True))
with open(pfile, "w") as f:
json.dump(parcel, f)
if debug:
print(f"File saved at: {pfile}")
else:
with open(pfile, "r") as f:
parcel = json.load(f)
images_dir = normpath(join(workdir, 'chip_images'))
if debug:
print(f"Getting '{band}' chip images for parcel: {pid}")
get_requests.rcbl(parcel, start_date, end_date, [band], chipsize,
images_dir)
images_list = normpath(
join(workdir, 'chip_images', f'images_list.{band}.csv'))
if file_len(images_list) > 1:
if debug:
print(f"Completed, all GeoTIFFs for band '{band}' are downloaded",
f" in the folder: '{workdir}/chip_images'")
else:
print("No files where downloaded, please check your configurations")
def import_db_functions():
from os.path import dirname, abspath, join, normpath, basename
from cbm.datas import db
from cbm.utils import config
import glob
path_foi_func = normpath(join(dirname(abspath(__file__)), 'foi_db_func'))
functions = glob.glob(f"{path_foi_func}/*.func")
db_conn = config.get_value(['set', 'db_conn'])
schema = config.get_value(['db', db_conn, 'schema'])
user = config.get_value(['db', db_conn, 'user'])
for f in functions:
db.insert_function(open(f).read().format(schema=schema, owner=user))
if db.db_func_exist(basename(f).split('.')[0]):
print(f"The '{basename(f)}' Was imported to the database.")
else:
print(f"Could not add function '{basename(f)}' to dattabase.")
def data_source():
source = int(config.get_value(['set', 'data_source']))
if source == 0:
from cbm.sources import rest_api
return rest_api
elif source == 1:
from cbm.sources import direct
return direct
def extract():
l_connec = Label("1. Connect to database and object storage.")
l_create = Label(
"2. Create the essential CbM tables. The parcels table name will be added as prefix."
)
l_upload = Label(
"3. Upload .shp, with all the required files (.shp, .cpg, .dbf, .prj, .shx)."
)
l_carddb = Label(
"4. Add CARD metadata to databse tabel 'xx_dias_catalogue'.")
l_extrac = Label("5. Run parcel extraction routines.")
path_data = normpath(join(config.get_value(['paths', 'temp']), 'extract'))
# Connect
db_select = Dropdown(options=[db for db in config.get_value(['db'])],
description='Configure:',
disabled=True,
layout=Layout(width='140px'))
db_config = Button(value=False,
disabled=False,
button_style='info',
tooltip='Configure db connection.',
icon='cogs',
layout=Layout(width='40px'))
db_box = HBox([db_select, db_config])
db_conf_box = HBox([])
@db_config.on_click
def db_config_on_click(b):
if db_conf_box.children == ():
db_conf_box.children = [settings_ds.direct_conn()]
else:
db_conf_box.children = ()
wbox = VBox([
l_connec,
VBox([db_box, db_conf_box]), l_create,
ext_func.create_tables(), l_upload,
ext_func.upload_shp(path_data), l_carddb,
ext_card2db.main(), l_extrac,
ext_func.extraction()
])
return wbox
def data_source():
source = config.get_value(['set', 'data_source'])
if source == 'api':
from cbm.datas import api
return api
elif source == 'direct':
from cbm.datas import direct
return direct
def slider(aoi, pid, chipsize=512, extend=512, tms=['Google']):
workdir = config.get_value(['paths', 'temp'])
path = f'{workdir}/{aoi}/{pid}/'
bg_path = f'{path}/backgrounds/'
for t in tms:
if not os.path.isfile(f'{bg_path}{t.lower()}.tif'):
bg.by_pid(aoi, pid, chipsize, extend, t, True)
with open(f'{path}info.json', "r") as f:
json_data = json.load(f)
def overlay_parcel(img, geom):
patche = [
PolygonPatch(feature,
edgecolor="yellow",
facecolor="none",
linewidth=2) for feature in geom['geom']
]
return patche
with rasterio.open(f'{bg_path}{tms[0].lower()}.tif') as img:
img_epsg = img.crs.to_epsg()
geom = spatial_utils.transform_geometry(json_data, img_epsg)
patches = overlay_parcel(img, geom)
selection = SelectionSlider(options=tms,
value=tms[0],
disabled=False,
continuous_update=False,
orientation='horizontal',
readout=True)
output = Output()
fig, ax = plt.subplots(figsize=(10, 10))
with output:
with rasterio.open(f'{bg_path}{selection.value.lower()}.tif') as img:
for patch in patches:
ax.add_patch(copy(patch))
show(img, ax=ax)
plt.show()
def on_value_change(change):
with output:
output.clear_output()
fig, ax = plt.subplots(figsize=(10, 10))
with rasterio.open(
f'{bg_path}{selection.value.lower()}.tif') as im:
for patch in patches:
ax.add_patch(copy(patch))
show(im, ax=ax)
plt.show()
selection.observe(on_value_change, names='value')
return VBox([selection, output])
def pre_ins_func_on_click(b):
progress.clear_output()
try:
functions = glob.glob(f"{path_plug}*.func")
db = config.get_value(['set', 'db_conn'])
sche = config.get_value(['db', db, 'conn', 'sche'])
user = config.get_value(['db', db, 'conn', 'user'])
for f in functions:
database.insert_function(
open(f).read().format(schema=sche, owner=user))
finc_list = [
f"ipycbm_{f.split('/')[-1].split('.')[0]}, " for f in functions
]
outlog(
f"The functions: {('').join(finc_list)}where added to the database"
)
except Exception as err:
outlog("Could not add functions to dattabase.", err)
def get_data(parcel):
values = config.read()
get_requests = data_source()
pid = parcel['ogc_fid'][0]
source = int(config.get_value(['set', 'data_source']))
if source == 0:
datapath = f'{paths.value}{aois.value}{year.value}/parcel_{pid}/'
elif source == 1:
ds_conf = config.get_value(['set', 'ds_conf'])
datapath = f'{paths.value}{ds_conf}/parcel_{pid}/'
file_pinf = f"{datapath}{pid}_information"
outlog(data_handler.export(parcel, 10, file_pinf))
if pts_bt.value is True:
outlog(f"Getting time series for parcel: '{pid}',",
f"({pts_tstype.value} {pts_band.value}).")
for pts in pts_tstype.value:
ts = json.loads(get_requests.pts(aois.value, year.value,
pid, pts,
pts_band.value))
band = ''
if pts_band.value != '':
band = f"_{pts_band.value}"
file_ts = f"{datapath}{pid}_time_series_{pts}{band}"
outlog(data_handler.export(ts, 11, file_ts))
if pci_bt.value is True:
files_pci = f"{datapath}{pid}_chip_images/"
outlog(f"Getting '{pci_band.value}' chip images for parcel: {pid}")
with progress:
get_requests.rcbl(parcel, pci_start_date.value,
pci_end_date.value, pci_band.value,
pci_satellite.value,
pci_chipsize.value, files_pci)
filet = f'{datapath}/{pid}_chip_images/{pid}_images_list.{pci_band.value[0]}.csv'
if file_len(filet) > 1:
outlog(f"Completed, all GeoTIFFs for bands '{pci_band.value}' are ",
f"downloaded in the folder: '{datapath}/{pid}_chip_images'")
else:
outlog("No files where downloaded, please check your configurations")
def by_pid(aoi,
year,
pid,
chipsize=512,
extend=512,
tms=['Google'],
ptype=None,
axis=True,
debug=False):
"""Download the background image with parcels polygon overlay by selected
location.
Examples:
from cbm.view import background
background.by_pid(aoi, pid, 512, 512, 'Google',
True, True)
Arguments:
aoi, the area of interest (str)
year, the year of parcels table
pid, the parcel id (str).
chipsize, size of the chip in pixels (int).
extend, size of the chip in meters (float).
tms, tile map server Google or Bing (str).
debug, print or not procedure information (Boolean).
"""
get_requests = data_source()
if type(tms) is str:
tms = [tms]
workdir = normpath(
join(config.get_value(['paths', 'temp']), aoi, str(year), str(pid)))
parcel = parcel_info.by_pid(aoi, year, pid, ptype, True, False, debug)
if debug:
print('workdir: ', workdir)
lon = parcel['clon'][0]
lat = parcel['clat'][0]
bg_path = normpath(join(workdir, 'backgrounds'))
os.makedirs(bg_path, exist_ok=True)
with open(f"{bg_path}/chipsize_extend_{chipsize}_{extend}", "w") as f:
f.write('')
if debug:
print('bg_path: ', bg_path)
print('lon, lat:', lon, lat)
for t in tms:
if debug:
print('lon', 'lat', 'chipsize', 'extend', 't', 'bg_path', 'debug')
print(lon, lat, chipsize, extend, t, bg_path, debug)
get_requests.background(lon, lat, chipsize, extend, t, bg_path, debug)
def startup():
import sys
if sys.version_info < (3, 6):
print("Not supoted python version, ipycbm needs python version > 3.6")
return
config.update_keys()
paths = config.get_value(['paths'])
for p in paths:
os.makedirs(paths[p], exist_ok=True)
update.check()
display(config.clean_temp(True))
def get_value(dict_keys, var_name=None):
"""Get value for tables.
Example:
database.get_value(['database', 'table'], variable_table_name)
Arguments:
dict_keys, list of keys to get value from.
var_name, the name ofthe variable
"""
if dict_keys[0][-1] == '2':
config_value = config.get_value(dict_keys)
value = config.autoselect(config_value, tables(2), True)
else:
config_value = config.get_value(dict_keys)
value = config.autoselect(config_value, tables(1), True)
if var_name is not None:
if value == None:
print(f"!WARNING! The value for table '{var_name}' is: '{value}'.")
else:
print(f"The value for table '{var_name}' is: '{value}'.")
return value
def bt_get_on_click(b):
progress.clear_output()
if method.value == 1:
try:
with progress:
lon, lat = plon.value, plat.value
get_from_location(lon, lat)
except Exception as err:
outlog("Could not get parcel information for location",
f"'{lon}', '{lat}': {err}")
elif method.value == 2:
try:
with progress:
pids = pid.value.replace(" ", "").split(",")
get_from_id(pids)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 3:
try:
marker = get_maps.base_map.map_marker
lon = str(round(marker.location[1], 2))
lat = str(round(marker.location[0], 2))
get_from_location(lon, lat)
except Exception as err:
outlog(f"Could not get parcel information: {err}")
elif method.value == 4:
try:
plimit = int(values['set']['plimit'])
file = normpath(
join(config.get_value(['paths', 'temp']),
'pids_from_polygon.txt'))
with open(file, "r") as text_file:
pids = text_file.read().split('\n')
outlog("Geting data form the parcels:")
outlog(pids)
if len(pids) <= plimit:
get_from_id(pids)
else:
outlog(
"You exceeded the maximum amount of selected parcels ",
f"({plimit}) to get data. Please select smaller area.")
except Exception as err:
outlog("No pids file found.", err)
else:
outlog(f"Please select method to get parcel information.")
def widget_box():
source = config.get_value(['set', 'data_source'])
sources = RadioButtons(
options=[
("DIAS API.", 'dias_api'),
("Direct access to database and object storage.", 'direct')
],
layout={'width': 'max-content'}
)
sources_box = Box([
Label(value="Available sources:"),
sources]
)
info_api = Label("DIAS API Settings.")
info_direct = Label("Direct access settings")
view_options = VBox([info_direct])
if source == 'direct':
view_options.children = [info_direct]
else:
view_options.children = [info_api, dias_api()]
def on_source_change(change):
view_options.children = []
if sources.value == 'dias_api':
view_options.children = [info_api, dias_api()]
elif sources.value == 'direct':
view_options.children = [info_direct]
config.update(['preferences', 'data_source'], str(sources.value))
sources.observe(on_source_change, 'value')
wbox_sources = VBox([sources_box, view_options],
layout=Layout(border='1px solid black'))
info_general = Label(value="General settings:")
wbox = VBox([wbox_sources, info_general, settings.widget_box()])
return wbox
def bt_get_ids_on_click(b):
with ppoly_out:
try:
get_requests = data_source()
ppoly_out.clear_output()
polygon = get_maps.polygon_map.feature_collection[
'features'][-1]['geometry']['coordinates'][0]
polygon_str = '-'.join(['_'.join(map(str, c))
for c in polygon])
outlog_poly(f"Geting parcel ids within the polygon...")
polyids = json.loads(get_requests.ppoly(aois.value, year.value,
polygon_str, False,
True))
outlog_poly(f"'{len(polyids['ogc_fid'])}' parcels where found:")
outlog_poly(polyids['ogc_fid'])
file = config.get_value(['files', 'pids_poly'])
with open(file, "w") as text_file:
text_file.write('\n'.join(map(str, polyids['ogc_fid'])))
except Exception as err:
outlog("No parcel ids found:", err)
def by_location(aoi, year, lon, lat, tstype, ptype=None, band='', debug=False):
"""Download the time series for the selected year
Examples:
import cbm
cbm.get.time_series.by_location(aoi, year, lon, lat, tstype)
Arguments:
aoi, the area of interest and year e.g.: es2019, nld2020 (str)
lon, lat, the the coords of the parcel (float).
"""
get_requests = data_source()
parcel = parcel_info.by_location(aoi, year, lon, lat, ptype, True, False,
debug)
if type(parcel['pid']) is list:
pid = parcel['pid'][0]
else:
pid = parcel['pid']
workdir = config.get_value(['paths', 'temp'])
file_ts = normpath(
join(workdir, aoi, year, str(pid), f'time_series_{tstype}{band}.csv'))
if debug:
print(file_ts)
print(parcel)
ts = json.loads(
get_requests.parcel_ts(aoi, year, pid, tstype, ptype, band, debug))
try:
if isinstance(ts, pd.DataFrame):
ts.to_csv(file_ts, index=True, header=True)
elif isinstance(ts, dict):
os.makedirs(os.path.dirname(file_ts), exist_ok=True)
df = pd.DataFrame.from_dict(ts, orient='columns')
df.to_csv(file_ts, index=True, header=True)
if debug:
print(f"File saved at: {file_ts}")
return ts
except Exception as err:
return f"Could not create the file: {err}"
return ts
def widget_box():
source = int(config.get_value(['set', 'data_source']))
sources = RadioButtons(options=[
("JRC RESTful API.", 0),
("Direct access to database and object storage.", 1)
],
value=source,
layout={'width': 'max-content'})
sources_box = Box([Label(value="Data sources:"), sources])
info_api = Label("RESTful API Settings.")
info_direct = Label("Direct access settings")
view_options = VBox([info_direct])
if source == 0:
view_options.children = [info_api, rest_api()]
elif source == 1:
view_options.children = [info_direct, direct()]
def on_source_change(change):
view_options.children = []
if sources.value == 0:
view_options.children = [info_api, rest_api()]
elif sources.value == 1:
view_options.children = [info_direct, direct()]
config.update(['set', 'data_source'], str(sources.value))
sources.observe(on_source_change, 'value')
wbox_sources = VBox([sources_box, view_options],
layout=Layout(border='1px solid black'))
info_general = Label(value="General settings:")
wbox = VBox([wbox_sources, info_general, settings.widget_box()])
return wbox
def by_pid(aoi, year, pid, ptype=None, geom=False, wgs84=False, debug=False):
"""Download the time series for the selected year
Examples:
import cbm
cbm.get.parcel.by_pid(aoi, pid)
Arguments:
aoi, the area of interest (str)
year, the year of parcels table
pid, the parcel id (int).
"""
get_requests = data_source()
workdir = config.get_value(['paths', 'temp'])
file_pinf = normpath(join(workdir, aoi, str(year), str(pid), 'info.json'))
parcel = json.loads(
get_requests.parcel_by_id(aoi, year, pid, ptype, geom, wgs84, debug))
os.makedirs(dirname(file_pinf), exist_ok=True)
with open(file_pinf, "w") as f:
json.dump(parcel, f)
return parcel
def clean_temp(hide=False):
import shutil
progress = Output()
def outlog(*text):
progress.clear_output()
with progress:
print(*text)
temppath = config.get_value(['paths', 'temp'])
directory = os.listdir(normpath(join(config.path_work, temppath)))
if len(directory) > 0:
outlog(f"Your temp folder '{temppath}' has old files:",
f" '{directory}', do you want to delete them? ")
bt_clean = Button(value=False,
description='Empty temp folder',
disabled=False,
button_style='danger',
tooltip='Delete all data from the temporary folder.',
icon='trash')
clean_box = HBox([bt_clean, progress])
@bt_clean.on_click
def bt_clean_on_click(b):
for i in directory:
try:
shutil.rmtree(join(temppath, i))
except Exception:
os.remove(join(temppath, i))
outlog(f"The '{temppath}' folder is now empty.")
if hide is False:
return clean_box
elif hide is True and len(directory) > 0:
return clean_box
else:
return HBox([])
def data_source():
source = config.get_value(['set', 'data_source'])
sources = RadioButtons(options=[
("RESTful API for CbM.", 'api'),
("Direct access to database and object storage.", 'direct')
],
value=source,
layout={'width': 'max-content'},
disabled=True)
sources_box = Box([Label(value="Data sources:"), sources])
info_api = Label("RESTful API Settings.")
info_direct = Label("Direct access settings")
view_options = VBox([info_direct])
if source == 'api':
view_options.children = [info_api, settings_ds.api()]
elif source == 'direct':
view_options.children = [info_direct, settings_ds.direct()]
def on_source_change(change):
view_options.children = []
if sources.value == 'api':
view_options.children = [info_api, settings_ds.api()]
elif sources.value == 'direct':
view_options.children = [info_direct, settings_ds.direct()]
config.set_value(['set', 'data_source'], sources.value)
sources.observe(on_source_change, 'value')
wbox_sources = VBox([sources_box, view_options],
layout=Layout(border='1px solid black'))
return wbox_sources
def by_location(aoi,
year,
lon,
lat,
dates,
band,
chipsize,
columns=5,
quiet=True):
from cbm.datas import api
"""Plot chip image with parcel polygon overlay.
Examples:
import cbm
cbm.get.chip_images.by_pid(aoi, pid, start_date, end_date,
band, chipsize)
Arguments:
aoi, the area of interest and year e.g.: es2020, cat2020 (str)
pid, the parcel id (int).
dates, the date of the image (str) or start_date and end_date (list)
'2019-06-01' or ['2019-06-01', '2019-06-30']
band, 3 Sentinel-2 band names. One of [‘B02’, ‘B03’, ‘B04’, ‘B08’]
(10 m bands) or [‘B05’, ‘B06’, ‘B07’, ‘B8A’, ‘B11’, ‘B12’]
(20 m bands). 10m and 20m bands can be combined.
The first band determines the resolution in the output
composite. Defaults to B08_B04_B03.
chipsize, size of the chip in pixels (int).
columns, (int)
"""
if type(dates) is list:
start_date, end_date = dates[0], dates[1]
else:
start_date, end_date = dates, dates
json_data = json.loads(api.parcel_by_loc(aoi, lon, lat, True))
if type(json_data['ogc_fid']) is list:
pid = json_data['ogc_fid'][0]
else:
pid = json_data['ogc_fid']
workdir = normpath(join(config.get_value(['paths', 'temp']), aoi,
str(pid)))
chip_imgs.by_pid(aoi, pid, start_date, end_date, band, chipsize, quiet)
chips_dir = normpath(join(workdir, 'chip_images'))
if type(dates) is list:
chips = normpath(join(chips_dir, f"*{band}.tif"))
else:
chips = normpath(
join(chips_dir, f"*{start_date.replace('-', '')}*{band}.tif"))
chips_list = glob.glob(chips)
if len(chips_list) > 0:
if len(chips_list) < columns:
columns = len(chips_list)
with rasterio.open(chips_list[0]) as img:
img_epsg = img.crs.to_epsg()
geom = spatial_utils.transform_geometry(json_data, img_epsg)
patches = overlay_parcel(img, geom)
if not quiet:
for chip in chips_list:
print(chip)
rows = int(
len(chips_list) // columns + (len(chips_list) % columns > 0))
fig = plt.figure(figsize=(30, 10 * rows))
grid = ImageGrid(
fig,
111, # similar to subplot(111)
nrows_ncols=(rows, columns), # creates grid of axes
axes_pad=0.4, # pad between axes in inch.
)
for ax, t in zip(grid, chips_list):
with rasterio.open(t) as img:
for patch in patches:
ax.add_patch(copy(patch))
show(img, ax=ax, cmap=data_options.cmaps(band))
ax.set_title(t.split('_')[-1].split('.')[0], fontsize=20)
if len(chips_list) > columns:
for ax in grid[-((columns * rows - len(chips_list))):]:
ax.remove()
plt.show()
else:
print("! No images to show.")
