def get_references(references, data_path, stage_in=True): ciop = cioppy.Cioppy() if len(references) == 1 and stage_in: logging.info('Stage-in product to {}'.format(data_path)) if len(references) > 1 and stage_in: logging.info('Stage-in {} products to {}'.format( len(references), data_path)) retrieved = [] identifier = [] for index, reference in enumerate(references): logging.info('The input reference (#{} of {}) is: {}'.format( index + 1, len(references), reference)) search_params = dict() search_params['do'] = 'terradue' search = ciop.search(end_point=reference, params=search_params, output_fields='enclosure,identifier', model='GeoTime') assert (search) identifier.append(search[0]['identifier']) logging.info('The input reference identifier is: {}'.format( search[0]['identifier'])) if stage_in: logging.info('Retrieve {} from {}'.format(search[0]['identifier'], search[0]['enclosure'])) local_path = ciop.copy(search[0]['enclosure'], data_path) logging.info('Staged {}'.format(local_path)) assert (local_path) retrieved.append(local_path) else: retrieved.append('') return retrieved, identifier
def get_vsi_url(input_reference, username, api_key): enclosure = cioppy.Cioppy().search(input_reference, [], 'enclosure', 'GeoTime', creds='{}:{}'.format( username, api_key))[0]['enclosure'] parsed_url = urlparse(enclosure) vsi_url = '/vsicurl/{}://{}:{}@{}{}'.format( urlparse(enclosure).scheme, username, api_key, urlparse(enclosure).netloc, urlparse(enclosure).path) return vsi_url
def get_metadata(input_references, data_path): ciop = cioppy.Cioppy() if isinstance(input_references, str): search_params = dict() search_params['do'] = 'terradue' products = gp.GeoDataFrame( ciop.search( end_point=input_references, params=search_params, output_fields= 'identifier,self,wkt,startdate,enddate,enclosure,orbitDirection,track,orbitNumber', model='EOP')) else: temp_results = [] for index, self in enumerate(input_references): search_params = dict() search_params['do'] = 'terradue' temp_results.append( ciop.search( end_point=self, params=search_params, output_fields= 'identifier,self,wkt,startdate,enddate,enclosure,orbitDirection,track,orbitNumber', model='EOP')[0]) products = gp.GeoDataFrame(temp_results) products = products.merge(products.apply( lambda row: analyse(row, data_path), axis=1), left_index=True, right_index=True) return products
def get_pipeline_results(pipeline_parameters, search_params): ciop = cioppy.Cioppy() if not 'cat' in search_params: # add cat out key search_params['cat'] = 'out' creds = '{}:{}'.format(pipeline_parameters['username'], pipeline_parameters['api_key']) search = gpd.GeoDataFrame(ciop.search(end_point=pipeline_parameters['end_point'], params=search_params, output_fields='link:results', model='GeoTime', creds=creds)) fields = 'title,identifier,self,enclosure,cat,cc,wkt,updated,startdate,vs:"tileid"' search_result_params = [] df = pd.DataFrame() for index, row in search.iterrows(): end_point = row['link:results'] temp_df = pd.DataFrame.from_dict(ciop.search(end_point=end_point, params=search_result_params, output_fields=fields, model='EOP', creds=creds)) df = df.append(temp_df, ignore_index=True) df = df.merge(df.apply(lambda row: analyse_row(row), axis=1), left_index=True, right_index=True) return df
def get_product_metadata(input_references, username, api_key): temp_searches = [] for index, reference in enumerate(input_references): search_temp = gp.GeoDataFrame(cioppy.Cioppy().search( end_point=reference, params=[], output_fields= 'self,track,enclosure,identifier,wkt,startdate,enddate,platform,cc', model='EOP', creds='{}:{}'.format(username, api_key))) temp_searches.append(search_temp) search = gp.GeoDataFrame(pd.concat(temp_searches, ignore_index=True)) search['geometry'] = search['wkt'].apply(loads) search['cc'] = pd.to_numeric(search['cc']) search['startdate'] = pd.to_datetime(search['startdate']) search['enddate'] = pd.to_datetime(search['enddate']) return search
import lxml.etree as etree import subprocess import tempfile import time #import psutil import os import sys sys.path.append('/opt/anaconda/envs/env_ewf_satcen_03_01_01/snap/.snap/snap-python') import snappy from snappy import GPF import logging import cioppy ciop = cioppy.Cioppy() logging.basicConfig(stream=sys.stderr, level=logging.INFO, format='%(asctime)s %(levelname)-8s %(message)s', datefmt='%Y-%m-%dT%H:%M:%S') from pygments import highlight from pygments.lexers import XmlLexer from pygments.formatters import HtmlFormatter import IPython from IPython.display import HTML def display_xml_nice(xml): formatter = HtmlFormatter() IPython.display.display(HTML('<style type="text/css">{}</style> {}'.format(formatter.get_style_defs('.highlight'), highlight(xml, XmlLexer(), formatter)))) def run_command(command, **kwargs):
def main(input_reference, data_path): os.environ['OTB_MAX_RAM_HINT'] = '4096' ciop = cioppy.Cioppy() temp_results = [] search_params = dict() for index, entry in enumerate(input_reference['value'].split(',')): temp_results.append( ciop.search( end_point=entry, params=search_params, output_fields= 'identifier,self,wkt,startdate,enddate,enclosure,orbitDirection,cc', model='EOP')[0]) sentinel2_search = GeoDataFrame(temp_results) sentinel2_search['startdate'] = pd.to_datetime( sentinel2_search['startdate']) sentinel2_search['enddate'] = pd.to_datetime(sentinel2_search['enddate']) sentinel2_search['wkt'] = sentinel2_search['wkt'].apply(loads) sentinel2_search = sentinel2_search.merge(sentinel2_search.apply( lambda row: analyse(row, data_path['value']), axis=1), left_index=True, right_index=True) composites = [] bands = ['B12', 'B8A', 'B04'] for index, row in sentinel2_search.iterrows(): # cloud mask logging.info('Cloud mask 20%') mask_prb = get_mask_prob(row) output_name = '{}_CLOUD_MASK_20.tif'.format(row['identifier']) cloud_mask(mask_prb, 20, output_name) cog(output_name) metadata(output_name, 'Cloud mask 20% {}'.format(row['identifier']), row) vrt_bands = [] for j, band in enumerate(bands): vrt_bands.append(get_band_path(row, band)) vrt = '{0}.vrt'.format(row['identifier']) ds = gdal.BuildVRT(vrt, vrt_bands, srcNodata=0, xRes=10, yRes=10, separate=True) ds.FlushCache() tif = '{}_ACTIVE_FIRE_UInt16.tif'.format(row['identifier']) logging.info('Convert {} to UInt16'.format(row['identifier'])) metadata(tif, 'RGB UInt16 Composite {}'.format(row['identifier']), row) gdal.Translate(tif, vrt, outputType=gdal.GDT_UInt16) cog(tif) tif = '{0}.tif'.format(row['identifier']) logging.info('Convert {} to byte'.format(row['identifier'])) gdal.Translate(tif, vrt, outputType=gdal.GDT_Byte, scaleParams=[[0, 10000, 0, 255]]) tif_e = '{}_ACTIVE_FIRE.tif'.format(row['identifier']) contrast_enhancement(tif, tif_e) composites.append(tif_e) os.remove(tif) os.remove(vrt) cog(tif_e) metadata(tif_e, 'RGB Composite {}'.format(row['identifier']), row) vrt = '{0}.vrt'.format(row['identifier']) ds = gdal.BuildVRT(vrt, [get_band_path(row, 'SCL')], separate=True) ds.FlushCache() scl_tif = '{0}_SCL.tif'.format(row['identifier']) metadata(scl_tif, 'Scene Classification {}'.format(row['identifier']), row) gdal.Translate(scl_tif, vrt, xRes=10, yRes=10, outputType=gdal.GDT_Byte, resampleAlg=gdal.GRA_Mode) cog(scl_tif) bands = ['B12'] #resampleAlg=gdal.GRA_Mode, for index, row in sentinel2_search.iterrows(): vrt_bands = [] for j, band in enumerate(bands): vrt_bands.append(get_band_path(row, band)) vrt = '{0}.vrt'.format(row['identifier']) ds = gdal.BuildVRT(vrt, vrt_bands, srcNodata=0, xRes=10, yRes=10, separate=True) ds.FlushCache() tif = '{0}.tif'.format(row['identifier']) gdal.Translate(tif, vrt, outputType=gdal.GDT_UInt16) hot_spot_name = '{}_HOT_SPOT.tif'.format(row['identifier']) metadata(hot_spot_name, 'Hot spot {}'.format(row['identifier']), row) logging.info('Hot spot detection for {}'.format(row['identifier'])) hot_spot(tif, scl_tif, hot_spot_name) cog(hot_spot_name) logging.info('Vectorize detected hot spots in {}'.format( row['identifier'])) results_gdf = polygonize(hot_spot_name, row['startdate'], row['identifier']) results_gdf.to_file('{}_HOT_SPOT_VECTOR.geojson'.format( row['identifier']), driver='GeoJSON') metadata('{}_HOT_SPOT_VECTOR.geojson'.format(row['identifier']), 'Hot spot vector {}'.format(row['identifier']), row) os.remove(tif) os.remove(vrt)