def get_options(debug=False):
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/info"""
response = requests.get(requrl.format(api_url), auth=(api_user, api_pass))
if debug:
print(requrl.format(api_url), response)
return response.content
def background(lon,
lat,
chipsize=512,
extend=512,
tms='Google',
bg_path='',
debug=False):
# aoi='undefined', year='', pid='0000', quiet=True):
"""Download the background image.
Examples:
background(lon, lat, 512, 512, 'Google', 'temp/test.tif', True)
Arguments:
lon, lat, longitude and latitude in decimal degrees (float).
chipsize, size of the chip in pixels (int).
extend, size of the chip in meters (float).
tms, tile map server Google or Bing (str).
bk_file, the name of the output file (str).
"""
# Get the api credentials
api_url, api_user, api_pass = config.credentials('api')
# The url to get the background image
requrl = f"lon={lon}&lat={lat}&chipsize={chipsize}&extend={extend}"
# print(f"{api_url}/query/backgroundByLocation?{requrl}&tms={tms}&iformat=tif")
response = requests.get(
f"{api_url}/query/backgroundByLocation?{requrl}&tms={tms}&iformat=tif",
auth=(api_user, api_pass))
# print(response)
# Try to get the image link from the html response
try:
img_url = response.content.decode("utf-8")
# print(type(img_url), img_url)
if img_url == '{}':
if debug:
print("Image not found...")
print(
f"{api_url}/query/backgroundByLocation?{requrl}&tms={tms}&iformat=tif",
response)
return response
else:
if debug:
print(
f"{api_url}/query/backgroundByLocation?{requrl}&tms={tms}&iformat=tif",
response)
res = requests.get(img_url, stream=True)
image_name = img_url.split('/')[-1].lower()
bg_file = normpath(join(bg_path, image_name))
with open(bg_file, "wb") as handle:
for chunk in res.iter_content(chunk_size=1024):
if chunk: # filter out keep-alive new chunks
handle.write(chunk)
return bg_file
except AttributeError as err:
return err
def wb_fupdate_on_click(b):
git_url, git_user, git_pass = config.credentials('git')
url = git_url.replace('http://', '')
url = url.replace('https://', '')
with progress:
run_update(['git', '-C', './', 'reset', '--hard', 'HEAD'])
run_update([
'git', '-C', './', 'pull',
f'http://{git_user}:{git_pass}@{url}'
])
def pts(area, year, pid, tstype, band=''):
if area == 'es_ns':
area = f'es'
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/parcelTimeSeries?aoi={}&year={}&pid={}&tstype={}&band={}"""
response = requests.get(requrl.format(api_url, area, year, pid, tstype,
band),
auth=(api_user, api_pass))
return response.content
def wb_update_on_click(b):
git_url, git_user, git_pass = config.credentials('git')
url = git_url.replace('http://', '')
url = url.replace('https://', '')
with progress:
if git_pass == '':
outlog("Please add the password for the git server.")
else:
run_update([
'git', '-C', './', 'pull',
f'http://{git_user}:{git_pass}@{url}'
], wb_fupdate)
def parcel_ts(aoi, year, pid, tstype='s2', ptype=None, band='', debug=False):
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/parcelTimeSeries?aoi={}&year={}&pid={}&tstype={}"""
if ptype not in [None, '']:
requrl = f"{requrl}&ptype={ptype}"
if band not in [None, '']:
requrl = f"{requrl}&band={band}"
response = requests.get(requrl.format(api_url, aoi, year, pid, tstype,
band),
auth=(api_user, api_pass))
if debug:
print(requrl.format(api_url, aoi, year, pid, tstype, band), response)
return response.content
def pid(area, year, pid, geom=False):
if area == 'es_ns':
parcels = f'es{year}_nour_subset'
else:
parcels = f'{area}{year}'
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/parcelById?parcels={}&parcelid={}"""
if geom is True:
requrl = f"{requrl}&withGeometry=True"
response = requests.get(requrl.format(api_url, parcels, pid),
auth=(api_user, api_pass))
return response.content
def cbl(lon, lat, start_date, end_date, bands=None, lut=None, chipsize=None):
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/chipsByLocation?lon={}&lat={}&start_date={}&end_date={}"""
band = '_'.join(bands)
if band is not None:
requrl = f"{requrl}&band={band}"
if chipsize is not None:
requrl = f"{requrl}&chipsize={chipsize}"
if lut != '':
requrl = f"{requrl}&lut={lut}"
# print(requrl.format(api_url, lon, lat, start_date, end_date))
response = requests.get(requrl.format(api_url, lon, lat, start_date,
end_date),
auth=(api_user, api_pass))
return response
def ppoly(area, year, polygon, geom=False, only_ids=True):
if area == 'es_ns':
parcels = f'es{year}_nour_subset'
else:
parcels = f'{area}{year}'
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/parcelsByPolygon?parcels={}&polygon={}"""
if geom is True:
requrl = f"{requrl}&withGeometry=True"
if only_ids is True:
requrl = f"{requrl}&only_ids=True"
response = requests.get(requrl.format(api_url, parcels, polygon),
auth=(api_user, api_pass))
return response.content
def parcel_peers(aoi,
year,
pid,
distance=1000.0,
maxPeers=10,
ptype=None,
debug=False):
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/parcelPeers?aoi={}&year={}&pid={}&distance={}&max={}"""
if ptype not in [None, '']:
requrl = f"{requrl}&ptype={ptype}"
response = requests.get(requrl.format(api_url, aoi, year, pid, distance,
maxPeers),
auth=(api_user, api_pass))
if debug:
print(requrl.format(api_url, aoi, year, pid, distance, maxPeers),
response)
return response.content
def parcel_by_id(aoi,
year,
pid,
ptype=None,
geom=False,
wgs84=False,
debug=False):
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/parcelByID?aoi={}&year={}&pid={}"""
if geom is True:
requrl = f"{requrl}&withGeometry=True"
if ptype not in [None, '']:
requrl = f"{requrl}&ptype={ptype}"
if wgs84 is True:
requrl = f"{requrl}&wgs84={wgs84}"
# print(requrl.format(api_url, aoi, year, pid))
response = requests.get(requrl.format(api_url, aoi, year, pid),
auth=(api_user, api_pass))
if debug:
print(requrl.format(api_url, aoi, year, pid), response)
return response.content
def check():
progress = Output()
def outlog(*text):
with progress:
print(*text)
git_url, git_user, git_pass = config.credentials('git')
url = git_url.replace('http://', '')
url = url.replace('https://', '')
command = [
'git', '-C', './', 'remote', 'show',
f'http://{git_user}:{git_pass}@{url}'
]
out = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
stdout, stderr = out.communicate()
if 'out of date' in stdout.decode():
display(progress)
outlog("There is an new version CbM.",
"Do you want to update your local CbM repository?")
display(btn_update())
def rcbl(parcel,
start_date,
end_date,
bands,
chipsize,
filespath,
debug=False):
"""Get parcel raw chip images from RESTful API by location"""
import os
import os.path
import pandas as pd
from osgeo import osr, ogr
import time
start = time.time()
api_url, api_user, api_pass = config.credentials('api')
for band in bands:
requrl = """{}/query/rawChipByLocation?lon={}&lat={}&start_date={}&end_date={}"""
if band is not None:
requrl = f"{requrl}&band={band}"
if chipsize is not None:
requrl = f"{requrl}&chipsize={chipsize}"
# Create a valid geometry from the returned JSON withGeometry
geom = ogr.CreateGeometryFromJson(parcel.get('geom')[0])
source = osr.SpatialReference()
source.ImportFromEPSG(parcel.get('srid')[0])
# Assign this projection to the geometry
geom.AssignSpatialReference(source)
target = osr.SpatialReference()
target.ImportFromEPSG(4326)
transform = osr.CoordinateTransformation(source, target)
# And get the lon, lat for its centroid, so that we can center the chips
# on the parcel
centroid = geom.Centroid()
centroid.Transform(transform)
# Use pid for next request
# pid = parcel['pid'][0]
# cropname = parcel['cropname'][0]
# Set up the rawChip request
cen_x, cen_y = str(centroid.GetX()), str(centroid.GetY())
response = requests.get(requrl.format(api_url, cen_y, cen_x,
start_date, end_date, band,
chipsize),
auth=(api_user, api_pass))
if debug:
print(
"Request url:",
requrl.format(api_url, cen_y, cen_x, start_date, end_date,
band, chipsize))
print("Centroid", centroid)
print("Response:", response)
# Create a pandas DataFrame from the json response
df = pd.read_json(response.content)
os.makedirs(filespath, exist_ok=True)
# Download the GeoTIFFs that were just created in the user cache
for c in df.chips:
url = f"{api_url}{c}"
outf = normpath(join(filespath, c.split('/')[-1]))
res = requests.get(url, stream=True)
if debug:
print(f"Downloading {c.split('/')[-1]}")
with open(outf, "wb") as handle:
for chunk in res.iter_content(chunk_size=512):
if chunk: # filter out keep-alive new chunks
handle.write(chunk)
# Sore DataFrame to file
df_file = normpath(join(filespath, f'images_list.{band}.csv'))
if isfile(df_file):
df_old = pd.read_csv(df_file, index_col=[0])
df = df.append(df_old, ignore_index=True)
df = df.drop_duplicates(subset=['chips'])
df = df.sort_values(by="dates")
df = df.reset_index(drop=True)
df.to_csv(df_file, index=True, header=True)
if debug:
print(f"Downloaded Images for '{band}'.")
print(f"The response table is saved to: {df_file}")
if debug:
print("\n------Total time------")
print(f"Total time required for {len(bands)}",
f"bands: {time.time() - start} seconds.")
def get_options():
api_url, api_user, api_pass = config.credentials('api')
requrl = """{}/query/options"""
response = requests.get(requrl.format(api_url), auth=(api_user, api_pass))
return response.content
def widget_box():
"""Update the repository.
Args:
None
Returns:
update_widget : A widget for general settings.
Raises:
Error:
Example:
"""
# User settings
user_info = Label("General settings.")
values = config.read()
user_name = Text(value=values['set']['user'],
placeholder='user name',
description='User:'******'set']['email'],
placeholder='[email protected]',
description='email:',
disabled=False)
user_institution = Text(value=values['set']['institution'],
placeholder='EU-',
description='Institution:',
disabled=False)
ms_list = data_options.eu_ms()
ms = Dropdown(
options=[(ms_list[m], m) for m in ms_list] + [('', '')],
value=values['set']['member_state'],
description='Member state:',
disabled=False,
)
wbox_user = VBox([user_info, user_name, user_email, user_institution, ms],
layout=Layout(border='1px solid black'))
# System settings
sys_info = Label("System settings.")
paths_info = Label(
"Select the personal data folder and the temporary folder.")
jupyterlab = Checkbox(
value=eval(values['set']['jupyterlab']),
description=
'Workin in Jupyter Lab (Uncheck for Voila and classical jupyter environment)',
disabled=False,
indent=False)
def on_jupyterlab_change(change):
config.update(['set', 'jupyterlab'], str(jupyterlab.value))
jupyterlab.observe(on_jupyterlab_change, 'value')
path_data = Text(value=values['paths']['data'], description='Data path:')
path_temp = Text(value=values['paths']['temp'], description='Temp path:')
files_info = Label("Select where to store the parcel IDs list file from:")
file_pids_poly = Text(value=values['files']['pids_poly'],
description='Polygon:')
file_pids_dist = Text(value=values['files']['pids_dist'],
description='Distance:')
plimit_info = Label(
"Warning: No more than 25 parcels are tested, unexpected results may occur."
)
plimit = BoundedIntText(value=int(values['set']['plimit']),
max=100_000_000,
min=1,
step=1,
description='Max parcels that can be downloaded:',
disabled=False)
wbox_sys = VBox([
sys_info, jupyterlab, plimit_info, plimit, paths_info, path_data,
path_temp, files_info, file_pids_poly, file_pids_dist
],
layout=Layout(border='1px solid black'))
# Git settings
git_info = Label(
"Git Settings. (To easily get the latest version of notebooks and scripts.)"
)
git_url, git_user, git_pass = config.credentials('git')
git_url = Text(value=git_url, description='Url:')
git_user = Text(value=git_user, description='User name:')
git_pass = Password(value=git_pass,
placeholder='******',
description='Password:'******'1px solid black'))
btn_save = Button(description='Save', disabled=False, icon='save')
progress = Output()
def outlog(*text):
with progress:
print(*text)
@btn_save.on_click
def btn_save_on_click(b):
progress.clear_output()
config.update(['set', 'user'], str(user_name.value))
config.update(['set', 'email'], str(user_email.value))
config.update(['set', 'institution'], str(user_institution.value))
config.update(['set', 'member_state'], str(user_email.value))
config.update(['set', 'plimit'], str(plimit.value))
config.update(['git', 'url'], str(git_url.value))
config.update(['git', 'user'], str(git_user.value))
config.update(['paths', 'data'], str(path_data.value))
config.update(['paths', 'temp'], str(path_temp.value))
config.update(['files', 'pids_poly'], str(file_pids_poly.value))
config.update(['files', 'pids_dist'], str(file_pids_dist.value))
if git_pass.value != '':
config.update(['git', 'pass'], str(git_pass.value))
outlog("The new settings are saved.")
wbox = VBox([
config.clean_temp(), wbox_user, wbox_sys, wbox_git,
HBox([btn_save, update.btn_update()]), progress
])
return wbox
def rcbl(parcel, start_date, end_date, bands, sat, chipsize, filespath):
import os
import pandas as pd
from osgeo import osr, ogr
import time
start = time.time()
api_url, api_user, api_pass = config.credentials('api')
for band in bands:
requrl = """{}/query/rawChipByLocation?lon={}&lat={}&start_date={}&end_date={}"""
if band is not None:
# band = '_'.join(band)
# band = band.replace(' ', '')
requrl = f"{requrl}&band={band}"
if chipsize is not None:
requrl = f"{requrl}&chipsize={chipsize}"
# Create a valid geometry from the returned JSON withGeometry
geom = ogr.CreateGeometryFromJson(parcel.get('geom')[0])
source = osr.SpatialReference()
source.ImportFromEPSG(parcel.get('srid')[0])
# Assign this projection to the geometry
geom.AssignSpatialReference(source)
target = osr.SpatialReference()
target.ImportFromEPSG(4326)
transform = osr.CoordinateTransformation(source, target)
# And get the lon, lat for its centroid, so that we can center the chips on
# the parcel
centroid = geom.Centroid()
centroid.Transform(transform)
# Use pid for next request
pid = parcel['ogc_fid'][0]
# cropname = parcel['cropname'][0]
# Set up the rawChip request
cen_x, cen_y = str(centroid.GetX()), str(centroid.GetY())
response = requests.get(requrl.format(api_url, cen_y, cen_x,
start_date, end_date, band,
chipsize),
auth=(api_user, api_pass))
# Directly create a pandas DataFrame from the json response
df = pd.read_json(response.content)
os.makedirs(os.path.dirname(filespath), exist_ok=True)
df_file = f'{filespath}{pid}_images_list.{band}.csv'
df.to_csv(df_file, index=True, header=True)
# print(f"The response table is saved to: {df_file}")
# Download the GeoTIFFs that were just created in the user cache
for c in df.chips:
url = f"{api_url}{c}"
res = requests.get(url, stream=True)
outf = f"{filespath}{c.split('/')[-1]}"
# print(f"Downloading {c.split('/')[-1]}")
with open(outf, "wb") as handle:
for chunk in res.iter_content(chunk_size=512):
if chunk: # filter out keep-alive new chunks
handle.write(chunk)
print(
f"Images for band '{band}', for the selected dates are downloaded."
)
# if len(df.index) != 0:
# print(f"All GeoTIFFs for band '{band}' are ",
# f"downloaded in the folder: '{filespath}'")
print("\n------Total time------")
print(
f"Total time required for {len(bands)} bands: {time.time() - start} seconds."
)
