def get_options(debug=False): api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/info""" response = requests.get(requrl.format(api_url), auth=(api_user, api_pass)) if debug: print(requrl.format(api_url), response) return response.content
def background(lon, lat, chipsize=512, extend=512, tms='Google', bg_path='', debug=False): # aoi='undefined', year='', pid='0000', quiet=True): """Download the background image. Examples: background(lon, lat, 512, 512, 'Google', 'temp/test.tif', True) Arguments: lon, lat, longitude and latitude in decimal degrees (float). chipsize, size of the chip in pixels (int). extend, size of the chip in meters (float). tms, tile map server Google or Bing (str). bk_file, the name of the output file (str). """ # Get the api credentials api_url, api_user, api_pass = config.credentials('api') # The url to get the background image requrl = f"lon={lon}&lat={lat}&chipsize={chipsize}&extend={extend}" # print(f"{api_url}/query/backgroundByLocation?{requrl}&tms={tms}&iformat=tif") response = requests.get( f"{api_url}/query/backgroundByLocation?{requrl}&tms={tms}&iformat=tif", auth=(api_user, api_pass)) # print(response) # Try to get the image link from the html response try: img_url = response.content.decode("utf-8") # print(type(img_url), img_url) if img_url == '{}': if debug: print("Image not found...") print( f"{api_url}/query/backgroundByLocation?{requrl}&tms={tms}&iformat=tif", response) return response else: if debug: print( f"{api_url}/query/backgroundByLocation?{requrl}&tms={tms}&iformat=tif", response) res = requests.get(img_url, stream=True) image_name = img_url.split('/')[-1].lower() bg_file = normpath(join(bg_path, image_name)) with open(bg_file, "wb") as handle: for chunk in res.iter_content(chunk_size=1024): if chunk: # filter out keep-alive new chunks handle.write(chunk) return bg_file except AttributeError as err: return err
def wb_fupdate_on_click(b): git_url, git_user, git_pass = config.credentials('git') url = git_url.replace('http://', '') url = url.replace('https://', '') with progress: run_update(['git', '-C', './', 'reset', '--hard', 'HEAD']) run_update([ 'git', '-C', './', 'pull', f'http://{git_user}:{git_pass}@{url}' ])
def pts(area, year, pid, tstype, band=''): if area == 'es_ns': area = f'es' api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/parcelTimeSeries?aoi={}&year={}&pid={}&tstype={}&band={}""" response = requests.get(requrl.format(api_url, area, year, pid, tstype, band), auth=(api_user, api_pass)) return response.content
def wb_update_on_click(b): git_url, git_user, git_pass = config.credentials('git') url = git_url.replace('http://', '') url = url.replace('https://', '') with progress: if git_pass == '': outlog("Please add the password for the git server.") else: run_update([ 'git', '-C', './', 'pull', f'http://{git_user}:{git_pass}@{url}' ], wb_fupdate)
def parcel_ts(aoi, year, pid, tstype='s2', ptype=None, band='', debug=False): api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/parcelTimeSeries?aoi={}&year={}&pid={}&tstype={}""" if ptype not in [None, '']: requrl = f"{requrl}&ptype={ptype}" if band not in [None, '']: requrl = f"{requrl}&band={band}" response = requests.get(requrl.format(api_url, aoi, year, pid, tstype, band), auth=(api_user, api_pass)) if debug: print(requrl.format(api_url, aoi, year, pid, tstype, band), response) return response.content
def pid(area, year, pid, geom=False): if area == 'es_ns': parcels = f'es{year}_nour_subset' else: parcels = f'{area}{year}' api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/parcelById?parcels={}&parcelid={}""" if geom is True: requrl = f"{requrl}&withGeometry=True" response = requests.get(requrl.format(api_url, parcels, pid), auth=(api_user, api_pass)) return response.content
def cbl(lon, lat, start_date, end_date, bands=None, lut=None, chipsize=None): api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/chipsByLocation?lon={}&lat={}&start_date={}&end_date={}""" band = '_'.join(bands) if band is not None: requrl = f"{requrl}&band={band}" if chipsize is not None: requrl = f"{requrl}&chipsize={chipsize}" if lut != '': requrl = f"{requrl}&lut={lut}" # print(requrl.format(api_url, lon, lat, start_date, end_date)) response = requests.get(requrl.format(api_url, lon, lat, start_date, end_date), auth=(api_user, api_pass)) return response
def ppoly(area, year, polygon, geom=False, only_ids=True): if area == 'es_ns': parcels = f'es{year}_nour_subset' else: parcels = f'{area}{year}' api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/parcelsByPolygon?parcels={}&polygon={}""" if geom is True: requrl = f"{requrl}&withGeometry=True" if only_ids is True: requrl = f"{requrl}&only_ids=True" response = requests.get(requrl.format(api_url, parcels, polygon), auth=(api_user, api_pass)) return response.content
def parcel_peers(aoi, year, pid, distance=1000.0, maxPeers=10, ptype=None, debug=False): api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/parcelPeers?aoi={}&year={}&pid={}&distance={}&max={}""" if ptype not in [None, '']: requrl = f"{requrl}&ptype={ptype}" response = requests.get(requrl.format(api_url, aoi, year, pid, distance, maxPeers), auth=(api_user, api_pass)) if debug: print(requrl.format(api_url, aoi, year, pid, distance, maxPeers), response) return response.content
def parcel_by_id(aoi, year, pid, ptype=None, geom=False, wgs84=False, debug=False): api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/parcelByID?aoi={}&year={}&pid={}""" if geom is True: requrl = f"{requrl}&withGeometry=True" if ptype not in [None, '']: requrl = f"{requrl}&ptype={ptype}" if wgs84 is True: requrl = f"{requrl}&wgs84={wgs84}" # print(requrl.format(api_url, aoi, year, pid)) response = requests.get(requrl.format(api_url, aoi, year, pid), auth=(api_user, api_pass)) if debug: print(requrl.format(api_url, aoi, year, pid), response) return response.content
def check(): progress = Output() def outlog(*text): with progress: print(*text) git_url, git_user, git_pass = config.credentials('git') url = git_url.replace('http://', '') url = url.replace('https://', '') command = [ 'git', '-C', './', 'remote', 'show', f'http://{git_user}:{git_pass}@{url}' ] out = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) stdout, stderr = out.communicate() if 'out of date' in stdout.decode(): display(progress) outlog("There is an new version CbM.", "Do you want to update your local CbM repository?") display(btn_update())
def rcbl(parcel, start_date, end_date, bands, chipsize, filespath, debug=False): """Get parcel raw chip images from RESTful API by location""" import os import os.path import pandas as pd from osgeo import osr, ogr import time start = time.time() api_url, api_user, api_pass = config.credentials('api') for band in bands: requrl = """{}/query/rawChipByLocation?lon={}&lat={}&start_date={}&end_date={}""" if band is not None: requrl = f"{requrl}&band={band}" if chipsize is not None: requrl = f"{requrl}&chipsize={chipsize}" # Create a valid geometry from the returned JSON withGeometry geom = ogr.CreateGeometryFromJson(parcel.get('geom')[0]) source = osr.SpatialReference() source.ImportFromEPSG(parcel.get('srid')[0]) # Assign this projection to the geometry geom.AssignSpatialReference(source) target = osr.SpatialReference() target.ImportFromEPSG(4326) transform = osr.CoordinateTransformation(source, target) # And get the lon, lat for its centroid, so that we can center the chips # on the parcel centroid = geom.Centroid() centroid.Transform(transform) # Use pid for next request # pid = parcel['pid'][0] # cropname = parcel['cropname'][0] # Set up the rawChip request cen_x, cen_y = str(centroid.GetX()), str(centroid.GetY()) response = requests.get(requrl.format(api_url, cen_y, cen_x, start_date, end_date, band, chipsize), auth=(api_user, api_pass)) if debug: print( "Request url:", requrl.format(api_url, cen_y, cen_x, start_date, end_date, band, chipsize)) print("Centroid", centroid) print("Response:", response) # Create a pandas DataFrame from the json response df = pd.read_json(response.content) os.makedirs(filespath, exist_ok=True) # Download the GeoTIFFs that were just created in the user cache for c in df.chips: url = f"{api_url}{c}" outf = normpath(join(filespath, c.split('/')[-1])) res = requests.get(url, stream=True) if debug: print(f"Downloading {c.split('/')[-1]}") with open(outf, "wb") as handle: for chunk in res.iter_content(chunk_size=512): if chunk: # filter out keep-alive new chunks handle.write(chunk) # Sore DataFrame to file df_file = normpath(join(filespath, f'images_list.{band}.csv')) if isfile(df_file): df_old = pd.read_csv(df_file, index_col=[0]) df = df.append(df_old, ignore_index=True) df = df.drop_duplicates(subset=['chips']) df = df.sort_values(by="dates") df = df.reset_index(drop=True) df.to_csv(df_file, index=True, header=True) if debug: print(f"Downloaded Images for '{band}'.") print(f"The response table is saved to: {df_file}") if debug: print("\n------Total time------") print(f"Total time required for {len(bands)}", f"bands: {time.time() - start} seconds.")
def get_options(): api_url, api_user, api_pass = config.credentials('api') requrl = """{}/query/options""" response = requests.get(requrl.format(api_url), auth=(api_user, api_pass)) return response.content
def widget_box(): """Update the repository. Args: None Returns: update_widget : A widget for general settings. Raises: Error: Example: """ # User settings user_info = Label("General settings.") values = config.read() user_name = Text(value=values['set']['user'], placeholder='user name', description='User:'******'set']['email'], placeholder='[email protected]', description='email:', disabled=False) user_institution = Text(value=values['set']['institution'], placeholder='EU-', description='Institution:', disabled=False) ms_list = data_options.eu_ms() ms = Dropdown( options=[(ms_list[m], m) for m in ms_list] + [('', '')], value=values['set']['member_state'], description='Member state:', disabled=False, ) wbox_user = VBox([user_info, user_name, user_email, user_institution, ms], layout=Layout(border='1px solid black')) # System settings sys_info = Label("System settings.") paths_info = Label( "Select the personal data folder and the temporary folder.") jupyterlab = Checkbox( value=eval(values['set']['jupyterlab']), description= 'Workin in Jupyter Lab (Uncheck for Voila and classical jupyter environment)', disabled=False, indent=False) def on_jupyterlab_change(change): config.update(['set', 'jupyterlab'], str(jupyterlab.value)) jupyterlab.observe(on_jupyterlab_change, 'value') path_data = Text(value=values['paths']['data'], description='Data path:') path_temp = Text(value=values['paths']['temp'], description='Temp path:') files_info = Label("Select where to store the parcel IDs list file from:") file_pids_poly = Text(value=values['files']['pids_poly'], description='Polygon:') file_pids_dist = Text(value=values['files']['pids_dist'], description='Distance:') plimit_info = Label( "Warning: No more than 25 parcels are tested, unexpected results may occur." ) plimit = BoundedIntText(value=int(values['set']['plimit']), max=100_000_000, min=1, step=1, description='Max parcels that can be downloaded:', disabled=False) wbox_sys = VBox([ sys_info, jupyterlab, plimit_info, plimit, paths_info, path_data, path_temp, files_info, file_pids_poly, file_pids_dist ], layout=Layout(border='1px solid black')) # Git settings git_info = Label( "Git Settings. (To easily get the latest version of notebooks and scripts.)" ) git_url, git_user, git_pass = config.credentials('git') git_url = Text(value=git_url, description='Url:') git_user = Text(value=git_user, description='User name:') git_pass = Password(value=git_pass, placeholder='******', description='Password:'******'1px solid black')) btn_save = Button(description='Save', disabled=False, icon='save') progress = Output() def outlog(*text): with progress: print(*text) @btn_save.on_click def btn_save_on_click(b): progress.clear_output() config.update(['set', 'user'], str(user_name.value)) config.update(['set', 'email'], str(user_email.value)) config.update(['set', 'institution'], str(user_institution.value)) config.update(['set', 'member_state'], str(user_email.value)) config.update(['set', 'plimit'], str(plimit.value)) config.update(['git', 'url'], str(git_url.value)) config.update(['git', 'user'], str(git_user.value)) config.update(['paths', 'data'], str(path_data.value)) config.update(['paths', 'temp'], str(path_temp.value)) config.update(['files', 'pids_poly'], str(file_pids_poly.value)) config.update(['files', 'pids_dist'], str(file_pids_dist.value)) if git_pass.value != '': config.update(['git', 'pass'], str(git_pass.value)) outlog("The new settings are saved.") wbox = VBox([ config.clean_temp(), wbox_user, wbox_sys, wbox_git, HBox([btn_save, update.btn_update()]), progress ]) return wbox
def rcbl(parcel, start_date, end_date, bands, sat, chipsize, filespath): import os import pandas as pd from osgeo import osr, ogr import time start = time.time() api_url, api_user, api_pass = config.credentials('api') for band in bands: requrl = """{}/query/rawChipByLocation?lon={}&lat={}&start_date={}&end_date={}""" if band is not None: # band = '_'.join(band) # band = band.replace(' ', '') requrl = f"{requrl}&band={band}" if chipsize is not None: requrl = f"{requrl}&chipsize={chipsize}" # Create a valid geometry from the returned JSON withGeometry geom = ogr.CreateGeometryFromJson(parcel.get('geom')[0]) source = osr.SpatialReference() source.ImportFromEPSG(parcel.get('srid')[0]) # Assign this projection to the geometry geom.AssignSpatialReference(source) target = osr.SpatialReference() target.ImportFromEPSG(4326) transform = osr.CoordinateTransformation(source, target) # And get the lon, lat for its centroid, so that we can center the chips on # the parcel centroid = geom.Centroid() centroid.Transform(transform) # Use pid for next request pid = parcel['ogc_fid'][0] # cropname = parcel['cropname'][0] # Set up the rawChip request cen_x, cen_y = str(centroid.GetX()), str(centroid.GetY()) response = requests.get(requrl.format(api_url, cen_y, cen_x, start_date, end_date, band, chipsize), auth=(api_user, api_pass)) # Directly create a pandas DataFrame from the json response df = pd.read_json(response.content) os.makedirs(os.path.dirname(filespath), exist_ok=True) df_file = f'{filespath}{pid}_images_list.{band}.csv' df.to_csv(df_file, index=True, header=True) # print(f"The response table is saved to: {df_file}") # Download the GeoTIFFs that were just created in the user cache for c in df.chips: url = f"{api_url}{c}" res = requests.get(url, stream=True) outf = f"{filespath}{c.split('/')[-1]}" # print(f"Downloading {c.split('/')[-1]}") with open(outf, "wb") as handle: for chunk in res.iter_content(chunk_size=512): if chunk: # filter out keep-alive new chunks handle.write(chunk) print( f"Images for band '{band}', for the selected dates are downloaded." ) # if len(df.index) != 0: # print(f"All GeoTIFFs for band '{band}' are ", # f"downloaded in the folder: '{filespath}'") print("\n------Total time------") print( f"Total time required for {len(bands)} bands: {time.time() - start} seconds." )