def test_footprints_s2(products):
footprints = SentinelAPI.to_geojson(products)
for footprint in footprints['features']:
assert not footprint['geometry'].errors()
with open(FIXTURES_DIR + '/expected_search_footprints_s2.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s2(products):
footprints = SentinelAPI.to_geojson(products)
for footprint in footprints['features']:
assert not footprint['geometry'].errors()
with open(FIXTURES_DIR + '/expected_search_footprints_s2.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s2(products, fixture_path):
footprints = SentinelAPI.to_geojson(products)
for footprint in footprints["features"]:
assert not footprint["geometry"].errors()
with open(fixture_path("expected_search_footprints_s2.geojson")) as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s2(products):
footprints = SentinelAPI.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s2.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s2(products):
footprints = SentinelAPI.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s2.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')),
(datetime(2014, 10, 10), datetime(2014, 12, 31)), producttype="GRD"
)
footprints = api.to_geojson(products)
for footprint in footprints['features']:
assert not footprint['geometry'].errors()
with open(FIXTURES_DIR + '/expected_search_footprints_s1.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson(FIXTURES_DIR + '/map.geojson')),
(datetime(2014, 10, 10), datetime(2014, 12, 31)), producttype="GRD"
)
footprints = api.to_geojson(products)
for footprint in footprints['features']:
assert not footprint['geometry'].errors()
with open(FIXTURES_DIR + '/expected_search_footprints_s1.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
products = api.query(geojson_to_wkt(read_geojson('tests/map.geojson')),
datetime(2014, 10, 10),
datetime(2014, 12, 31),
producttype="GRD")
footprints = api.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s1.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def test_footprints_s1():
api = SentinelAPI(**_api_auth)
products = api.query(
geojson_to_wkt(read_geojson('tests/map.geojson')),
datetime(2014, 10, 10), datetime(2014, 12, 31), producttype="GRD"
)
footprints = api.to_geojson(products)
for footprint in footprints['features']:
validation = geojson.is_valid(footprint['geometry'])
assert validation['valid'] == 'yes', validation['message']
with open('tests/expected_search_footprints_s1.geojson') as geojson_file:
expected_footprints = geojson.loads(geojson_file.read())
# to compare unordered lists (JSON objects) they need to be sorted or changed to sets
assert set(footprints) == set(expected_footprints)
def main(options):
api = SentinelAPI('bartulo', 'ventanuco')
footprint = geojson_to_wkt(read_geojson(options.geojson))
products = api.query( footprint,
date = ( date.today() - timedelta(options.days), date.today() + timedelta(1)),
producttype = 'S2MSI2A',
platformname = 'Sentinel-2')
data = api.to_geojson(products)['features']
question_name = 'capa'
questions = [
inquirer.List(
question_name,
message = 'Imagenes disponibles',
choices = ["Id: %s - Fecha: %s - Cobertura de nubes:%.4s%%" % (i, data[i]['properties']['beginposition'], data[i]['properties']['cloudcoverpercentage']) for i in range(len(data))],
),
]
answers = inquirer.prompt(questions)
index = int(answers[question_name].split('-')[0].split(':')[1].replace(' ', ''))
baseURL = data[index]['properties']['link_alternative']
filename = data[index]['properties']['filename']
api_session = requests.Session()
api_session.auth = ('bartulo', 'ventanuco')
granules = api_session.get("%s/Nodes('%s')/Nodes('GRANULE')/Nodes?$format=json" % (baseURL, filename)).json()
granules_id = granules['d']['results'][0]['Id']
print("%s/Nodes('%s')/Nodes('GRANULE')/Nodes?$format=json" % (baseURL, filename))
bands_10m = api_session.get("%s/Nodes('%s')/Nodes('GRANULE')/Nodes('%s')/Nodes('IMG_DATA')/Nodes('R10m')/Nodes?$format=json" % (baseURL, filename, granules_id)).json()
band8 = bands_10m['d']['results'][4]['__metadata']['media_src']
bandColor = bands_10m['d']['results'][5]['__metadata']['media_src']
bands_20m = api_session.get("%s/Nodes('%s')/Nodes('GRANULE')/Nodes('%s')/Nodes('IMG_DATA')/Nodes('R20m')/Nodes?$format=json" % (baseURL, filename, granules_id)).json()
band12 = bands_20m['d']['results'][8]['__metadata']['media_src']
print("%s/Nodes('%s')/Nodes('GRANULE')/Nodes('%s')/Nodes('IMG_DATA')/Nodes('R20m')/Nodes?$format=json" % (baseURL, filename, granules_id))
# download(band12, 'banda12.jp2', api_session)
# download(band8, 'banda8.jp2', api_session)
# download(bandColor, 'color.jp2', api_session)
banda8 = gdal.Open('banda8.jp2')
banda12 = gdal.Open('banda12.jp2')
b8 = banda8.ReadAsArray()
b12 = banda12.ReadAsArray()
class Downloader:
def __init__(self, str_username, str_password, str_link):
self.api = SentinelAPI(str_username, str_password, str_link)
self.products = None
def search_polygon(self, footprint: object, str_date_start: str,
str_date_end: str, str_platform_name: str, percentage: object):
print('searching')
self.products = self.api.query(footprint,
date=(str_date_start, str_date_end),
platformname=str_platform_name,
cloudcoverpercentage=(percentage[0], percentage[1]))
size = self.api.get_products_size(self.products)
print(f'found {size}GiB of data')
# print(self.products)
def download_zip(self, path):
self.api.download_all(self.products, path, max_attempt, True)
def download_products(self, path, download_file):
if download_file:
self.download_zip(path)
print('downloaded')
df_products = self.api.to_dataframe(self.products)
return df_products
def download_geoproduct(self, path, download_file):
if download_file:
self.download_zip(path)
# print('download Geos')
gdf_products = self.api.to_geodataframe(self.products)
return gdf_products
def download_json(self):
return self.api.to_geojson(self.products)
def download_one(self, key, path):
self.api.download(key, path, True)
def getMetaData(self, year, month, day):
try:
_date = datetime(int(year), int(month), int(day))
api = SentinelAPI('USERNAME', 'PASSWORD',
'https://scihub.copernicus.eu/dhus')
footprint = geojson_to_wkt(read_geojson('goldmine_sa.geojson'))
products = api.query(footprint,
date=(_date.strftime('%Y%m%d'),
(_date +
timedelta(days=1)).strftime('%Y%m%d')),
platformname='Sentinel-1',
producttype='GRD')
fc = api.to_geojson(products)
features = fc['features']
for items in features:
if json.dumps(items['properties']['filename']).replace(
'"', '') == fn:
metaData = items
return metaData
except:
print 'couldnt retreive metadata'
exit()
from sentinelsat import SentinelAPI, read_geojson, geojson_to_wkt
mypath = r'C:\Users\mccoy\PycharmProjects\untitled\gz_2010_us_outline_500k.json'
# products = api.query(footprint,
# producttype='SLC',
# orbitdirection='ASCENDING')
# api.download_all(products)
testAPI = SentinelAPI(user='******', password='******', api_url='https://s5phub.copernicus.eu/dhus')
footprint = geojson_to_wkt(read_geojson(mypath))
#products = testAPI.query(area = footprint, date = "[NOW-20DAYS TO NOW]", platformname='Sentinel-5p')
#Pollution map for the US
products = testAPI.query(area = footprint, date = "[NOW-1DAYS TO NOW]",producttype='L2__NO2___')
# download all results from the search
mypath = testAPI.download_all(products)
downloadedfile = mypath[0][next(iter(mypath[0]))]['path']
dir_path = os.path.dirname(os.path.realpath(__file__))
downloadedfile_full = dir_path + downloadedfile
# GeoJSON FeatureCollection containing footprints and metadata of the scenes
testAPI.to_geojson(products)
# GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries
api.to_geodataframe(products)
#Get all data for the whole world
#products = testAPI.query(date = "[NOW-1DAYS TO NOW]")
print('x')
def apirequest(year="2017"):
full_path = os.getcwd()
os.chdir(Path(full_path).parents[0])
downloadfolder = r"Produtos_Sentinel"
try:
if not os.path.exists(downloadfolder):
os.makedirs(downloadfolder)
except OSError:
print("Erro: Criar Pasta " + downloadfolder)
api = SentinelAPI('pfmacf', '4mcTUobqdf',
'https://scihub.copernicus.eu/dhus')
path = "Dataset" + str(year)
my_dir_path = Path(path)
if not my_dir_path.is_dir():
switcherfunc = {
0: convertshapeepsg,
1: createindividualshp,
2: datatxt,
3: getgeojson
}
for func in switcherfunc:
_func_exec = switcherfunc[func](year)
num_sub_folders = [name for name in os.listdir(path)]
json_folder = check_jsonstats_folder("JsonApis")
index_filename = "LastIndex_SentinelApi" + " " + str(year) + ".json"
jpath = os.path.join(json_folder, index_filename)
jsonstring = "Last Iteration" + " " + str(year)
unique_txt = r"Unique_Products.txt"
open(unique_txt, 'a').close()
contador_vazios = 0
my_file_index = Path(index_filename)
if my_file_index.is_file():
with open(index_filename, "r") as file:
data = file.read()
file_dict = json.loads(data)
index = file_dict[jsonstring]
else:
index = 0
for i in range(index, len(num_sub_folders)):
pathfinal = os.path.join(path, str(i))
pathtxt = os.path.join(pathfinal, "Data.txt")
pathgeojson = os.path.join(pathfinal, "bbox4326.geojson")
data = linecache.getline(pathtxt, 2)
date_fire = data.rstrip('\n').split(" ")
ano = int(date_fire[2].split("-")[0])
mes = int(date_fire[2].split("-")[1])
dia = int(date_fire[2].split("-")[2])
search_date = [ano, mes, dia]
products = get_footprint(api, pathgeojson, search_date, 14)
print('\nNumero de produtos encontrados = ', (len(products)))
print('\nIndex = ', i)
if not len(products):
products = get_footprint(api, pathgeojson, search_date, 30)
if not len(products):
products = get_footprint(api, pathgeojson, search_date, 60)
if not len(products):
filename = "Sem_Produtos.txt"
filepath = os.path.join(pathfinal, filename)
write_to_file(filepath, mode="w", text="Invalido")
contador_vazios += 1
continue
# converter para DataFrame
products_df = api.to_dataframe(products)
products_geojson = api.to_geojson(products)
valid_list = validate_footprints(products_geojson, pathfinal, i, year)
if valid_list:
product_id, product_title = get_final_product(
valid_list, products_df, pathfinal)
product_info = api.get_product_odata(product_id)
file = open(unique_txt, "r+", encoding="utf-8")
line_found = any(product_title in line for line in file)
if not line_found:
write_to_file(unique_txt, mode="a", text=product_title)
open("Products_to_Download.txt", 'a').close()
check_availability_download(product_info, product_title,
product_id, api, downloadfolder)
file.close()
else:
file.close()
else:
filename = "Sem_Produtos.txt"
filepath = os.path.join(pathfinal, filename)
write_to_file(filepath, mode="w", text="Invalido")
contador_vazios += 1
json_dict = {jsonstring: i}
json_dict_exception = {jsonstring: 0}
if i < len(num_sub_folders) - 1:
with open(jpath, 'w') as output:
json.dump(json_dict, output, indent=4)
else:
with open(jpath, 'w') as output:
json.dump(json_dict_exception, output, indent=4)
print("Contagem de incendios sem Produto: ", contador_vazios)
class Downloader(object):
def __init__(self,
username,
password,
satellite,
order_id,
directory=Path('/data/')):
# The connection to ESA scihub
self.api = SentinelAPI(username,
password,
'https://scihub.copernicus.eu/dhus',
timeout=500.00)
# Sentinel-5p currently has its own pre-operations hub
self.api_s5p = SentinelAPI(user='******',
password='******',
api_url='https://s5phub.copernicus.eu/dhus')
# Use the current datetime to name the download order
self.order_id = order_id
# Use ordered dict to store the metadata of the queries products
self.products = OrderedDict()
self.satellite = satellite
self.directory = directory
# if not self.directory.exists(): # Create directory if it does not exist
# os.makedirs(self.directory)
def query(self, footprint, startdate, enddate):
if self.satellite == 's1' or self.satellite == 'all':
self.query_s1(footprint, startdate, enddate)
if self.satellite == 's2' or self.satellite == 'all':
self.query_s2(footprint, startdate, enddate)
if self.satellite == 's3' or self.satellite == 'all':
self.query_s3(footprint, startdate, enddate)
if self.satellite == 's5p' or self.satellite == 'all':
self.query_s5p(footprint, startdate, enddate)
def query_s1(self, footprint, startdate, enddate):
# Define producttypes (here it is Sentinel-1 GRDH products)
producttypes = ['GRD']
# Loop over producttypes and update the query dictionary
# TODO: Fix this inefficient way of querying the relative orbits
print(str(footprint))
if FLAGS.s2_intersection:
for producttype in producttypes:
queried_products = self.api.query(
area=footprint,
date=(startdate, enddate),
platformname='Sentinel-1',
#area_relation='Contains',
producttype=producttype,
sensoroperationalmode='IW',
polarisationmode='VV VH')
self.products.update(queried_products)
self.intersect_products()
elif FLAGS.s1_relative_orbit == [0]:
for producttype in producttypes:
queried_products = self.api.query(
area=footprint,
date=(startdate, enddate),
platformname='Sentinel-1',
#area_relation='Contains',
producttype=producttype,
sensoroperationalmode='IW',
polarisationmode='VV VH')
self.products.update(queried_products)
else:
for producttype in producttypes:
for relative_orbit in FLAGS.s1_relative_orbit:
queried_products = self.api.query(
area=footprint,
date=(startdate, enddate),
platformname='Sentinel-1',
producttype=producttype,
#area_relation='Contains',
sensoroperationalmode='IW',
relativeorbitnumber=relative_orbit)
self.products.update(queried_products)
def query_s2(self, footprint, startdate, enddate):
# Load parameters from FLAGS
max_cloudcoverage = FLAGS.s2_max_cloudcoverage
# Define producttypes (here it is Sentinel-2 L2A products)
producttypes = [
'S2MSI2Ap', 'S2MSI2A'
] # Producttype names differ depending on the year they were published
# Loop over producttypes and update the query dictionary
# TODO: Fix this inefficient way of querying the relative orbits
if FLAGS.s2_relative_orbit == [0]:
for producttype in producttypes:
queried_products = self.api.query(
footprint,
date=(startdate, enddate),
platformname='Sentinel-2',
producttype=producttype,
cloudcoverpercentage=(0, max_cloudcoverage),
order_by='-ingestiondate')
self.only_complete_tile(queried_products)
self.products.update(queried_products)
else:
for producttype in producttypes:
for relative_orbit in FLAGS.s2_relative_orbit:
queried_products = self.api.query(
footprint,
date=(startdate, enddate),
platformname='Sentinel-2',
relativeorbitnumber=relative_orbit,
producttype=producttype,
cloudcoverpercentage=(0, max_cloudcoverage))
self.only_complete_tile(queried_products)
self.products.update(queried_products)
def query_s3(self, footprint, startdate, enddate):
queried_products = self.api.query(footprint,
date=(startdate, enddate),
platformname='Sentinel-3',
producttype='SL_2_LST___',
productlevel='L2')
self.products.update(queried_products)
def query_s5p(self, footprint, startdate, enddate):
kwargs = {}
producttypedescriptions = [
'Ozone', 'Sulphur Dioxide', 'Nitrogen Dioxide', 'Methane',
'Formaldehyde', 'Carbon Monoxide', 'Aerosol Index',
'Aerosol Layer Height', 'Cloud'
]
# producttypedescriptions = ['Ozone']
# Loop over producttypes and update the query dictionary
for producttypedescription in producttypedescriptions:
queried_products = self.api_s5p.query(
footprint,
date=(startdate, enddate),
platformname='Sentinel-5 Precursor',
processinglevel='L2',
producttypedescription=producttypedescription,
**kwargs)
# Remove any 'Suomi-NPP VIIRS Clouds' products which are returned as 'Cloud' (they shouldn't have been)
# https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-5p/products-algorithms
if producttypedescription == 'Cloud':
temp_queried_products = queried_products.copy()
for key in queried_products.keys():
if queried_products[key][
'producttypedescription'] != 'Cloud':
del temp_queried_products[key]
queried_products = temp_queried_products
self.products.update(queried_products)
def print_num_and_size_of_products(self):
logging.info('Number of products = ' + str(len(list(self.products))))
logging.info('Total size [GB] = ' +
str(self.api.get_products_size(self.products)))
# https://sentinelsat.readthedocs.io/en/master/api.html#lta-products
# TODO: Get LTA retrieval to work properly (install of newest sentinelsat version is in dockerfile)
# Retry every 30 min (+10 second buffertime) to request LTA products.
@tenacity.retry(stop=tenacity.stop_after_attempt(200),
wait=tenacity.wait_fixed(1810))
def download_zipfiles(self):
zipfiles_directory = self.directory / 'zipfiles'
if len(self.products) == 0:
logging.info('Unable to find any products for the selected biome')
sys.exit(0)
return
if not zipfiles_directory.exists(
): # Create directory if it does not exist
os.makedirs(zipfiles_directory)
# Get the products to be downloaded. The sample() funcitons permutes the dataframe, such that a new LTA product
# is request at every retry. The optimal solution would have been to rearrange the dataframe by rotating the
# index at every retry, but this is a quick and dirty way to achieve something similar.
# (https://stackoverflow.com/a/34879805/12045808).
products_df = self.queried_products_as_df().sample(frac=1)
# NOTE: The code below is only useful while the Sentinel-5p has a different api than the others. After this has
# been fixed, the code should be reduced to the following single line:
# Download all zipfiles (it automatically checks if zipfiles already exist)
# self.api.download_all(self.products, directory_path=zipfiles_directory) # Download all zipfiles
# But for now, use the following code:
non_s5p_products = products_df[
products_df['platformname'] != 'Sentinel-5 Precursor']
s5p_products = products_df[products_df['platformname'] ==
'Sentinel-5 Precursor']
if len(non_s5p_products):
logging.info("Downloading Sentinel-1/2/3 products")
try:
downloaded, triggered, failed = self.api.download_all(
non_s5p_products.to_dict(into=OrderedDict, orient='index'),
directory_path=zipfiles_directory)
logging.info("Downloaded: " + str(downloaded))
logging.info("Triggered: " + str(triggered))
logging.info("failed: " + str(failed))
except InvalidChecksumError:
logging.info("Error downloading products due to CheckSumError")
except Exception:
logging.info("Error downloading products due to unkown error")
else:
logging.info("No Sentinel-1/2/3 products found in query")
if len(s5p_products):
logging.info("Downloading Sentinel-5p products")
self.api_s5p.download_all(s5p_products.to_dict(into=OrderedDict,
orient='index'),
directory_path=zipfiles_directory)
else:
logging.info("No Sentinel-5p products found in query")
# The Sentinel-5p data has wrongly been given the filetype .zip, but it should be .nc, so make a copy with
# .nc extension. A copy is made instead of renaming so sentinelsat doesn't re-download the file every time
# it is run.
s5p_downloaded_files = zipfiles_directory.glob('S5P*.zip')
logging.debug(
"Renaming downloaded Sentinel-5p files from .zip to .nc (due to bug in SentinelSat)"
)
for file in s5p_downloaded_files:
if not file.with_suffix('.nc').exists():
shutil.copy(str(file), str(file.with_suffix('.nc')))
def queried_products_as_geojson(self):
return self.api.to_geojson(self.products)
def only_complete_tile(self, products):
found_one = False
delete_list = []
for i in products:
local_footprint = products.get(i).get('footprint')
elements = local_footprint.split(',')
if len(elements) == 5 and found_one == False:
found_one = True
continue
else:
delete_list.append(i)
for i in delete_list:
del products[i]
def intersect_products(self):
print('Found ' + str(len(self.products)) + ' products')
S2_geojson_path = (self.directory / 'orders' /
FLAGS.s2_order_id).with_suffix('.geojson')
ground_geojsons = read_geojson(S2_geojson_path)
products_geojsons = self.queried_products_as_geojson()
ground_polygon = ground_geojsons.get('features')[0].get(
'geometry').get('coordinates')
ground_polygon = geometry.Polygon(ground_polygon[0][0])
titles = []
ids = []
for item in products_geojsons.get('features'):
id = item.get('properties').get('id')
item = item.get('properties').get('title')
item = (item[17:25] + item[48:55])
titles.append(item)
ids.append([item, id])
unique = list(set(titles))
unique.sort()
union_list = []
for i, element in enumerate(unique):
local_polygon = Polygon()
for j in range(len(titles)):
if titles[j] == element:
item = products_geojsons.get('features')[j]
item = item.get('geometry').get('coordinates')
item = geometry.Polygon(item[0][0])
item = affinity.scale(item, xfact=1.01, yfact=1.01)
polygons = [item, local_polygon]
local_polygons = unary_union(polygons)
local_polygon = item
union_list.append([local_polygons, element])
found_id = None
for index, element in enumerate(union_list):
wkt = element[0].wkt
if ground_polygon.within(element[0]):
found_id = element[1]
break
for i in ids:
if found_id != i[0]:
del self.products[i[1]]
print('Reduced the products to ' + str(len(self.products)) +
' products')
def queried_products_as_df(self):
return self.api.to_dataframe(self.products)
def save_queried_products(self):
orders_directory = self.directory / 'orders'
if not orders_directory.exists():
os.makedirs(orders_directory)
# Save the queried products to a geojson file (e.g. to be loaded into QGIS)
geojson_path = (self.directory / 'orders' /
self.order_id).with_suffix('.geojson')
with geojson_path.open('w') as geojson_file:
geojson_data = self.api.to_geojson(self.products)
geojson_file.write(str(geojson_data))
# Save the queried products as pandas df in a pkl file (preferred format when working in Python)
df_path = (self.directory / 'orders' /
self.order_id).with_suffix('.pkl')
df = self.api.to_dataframe(self.products)
df.to_pickle(df_path)
def save_queried_products_location(self, path):
path = Path(path)
path = path.parent.absolute()
path = path / 'log'
# Save the queried products to a geojson file (e.g. to be loaded into QGIS)
geojson_path = (path / self.order_id).with_suffix('.geojson')
with geojson_path.open('w') as geojson_file:
geojson_data = self.api.to_geojson(self.products)
geojson_file.write(str(geojson_data))
# Save the queried products as pandas df in a pkl file (preferred format when working in Python)
df_path = (path / self.order_id).with_suffix('.pkl')
df = self.api.to_dataframe(self.products)
df.to_pickle(df_path)
def main(args):
""" Runs dataLayer processing scripts to turn raw dataLayer from (../raw) into
cleaned dataLayer ready to be analyzed (saved in ../processed).
"""
## Talk to Rune about how dataLayer is handle. If it should be part of the "big" project.
## set number_tiles:1764
config = TrainingConfig()
config = update_config(args, config)
logger = logging.getLogger(__name__)
logger.info('making final dataLayer set from raw dataLayer')
userTuple = [['pandagud', 'damp4ever'], ['pandagud2', 'damp4ever'],
['pandagud3', 'damp4ever'], ['au524478', 'Palantir1234']]
current_user = random.choice(userTuple)
api = SentinelAPI(current_user[0], current_user[1],
'https://scihub.copernicus.eu/dhus')
# search by polygon, time, and SciHub query keywords
path = r"C:\Users\panda\Downloads\LC80290292014132LGN00.geojson"
footprint = geojson_to_wkt(read_geojson(path))
products = api.query(area=footprint,
date=('20210101', '20210105'),
platformname='Sentinel-2',
order_by='+ingestiondate',
limit=1)
areas = api.to_geodataframe(products)
geojson = api.to_geojson(products)
api.download_all(products, into=r'C:\Users\panda\Sat_paper\Alfa')
products = api.query(area=footprint,
date=('20210401', '20210430'),
producttype='GRD',
platformname='Sentinel-1',
sensoroperationalmode='IW',
polarisationmode='VV VH',
order_by='ingestiondate')
firstproduct = next(iter(products))
online_product = ''
for i in products:
is_online = api.is_online(products.get(i).get('uuid'))
if is_online:
online_product = i
break
delete_list = []
for i in products:
if i != online_product:
delete_list.append(i)
for i in delete_list:
del products[i]
ground_geojsons = read_geojson(path)
products_geojsons = api.to_geojson(products)
ground_polygon = ground_geojsons.get('features')[0].get('geometry').get(
'coordinates')
ground_polygon = geometry.Polygon(ground_polygon[0][0])
import numpy as np
titles = []
ids = []
for item in products_geojsons.get('features'):
id = item.get('properties').get('id')
item = item.get('properties').get('title')
item = (item[17:25] + item[48:55])
titles.append(item)
ids.append([item, id])
unique = list(set(titles))
union_list = []
for i, element in enumerate(unique):
local_polygon = Polygon()
for j in range(len(titles)):
if titles[j] == element:
item = products_geojsons.get('features')[j]
item = item.get('geometry').get('coordinates')
item = geometry.Polygon(item[0][0])
item = affinity.scale(item, xfact=1.01, yfact=1.01)
polygons = [item, local_polygon]
local_polygons = unary_union(polygons)
local_polygon = item
union_list.append([local_polygons, element])
for index, element in enumerate(union_list):
wkt = element[0].wkt
if ground_polygon.within(element[0]):
found_id = element[1]
break
for i in ids:
if found_id != i[0]:
del products[i[1]]
area_list = []
for index, item in enumerate(products_geojsons.get('features')):
item = item.get('geometry').get('coordinates')
item = geometry.Polygon(item[0][0])
local_intersection = item.intersection(ground_polygon)
local_intersection = [local_intersection.area, index]
area_list.append(local_intersection)
area_list.sort(reverse=True)
for index in range(len(area_list)):
item = products_geojsons.get('features')[area_list[index][1]]
id = item.get('properties').get('id')
item = item.get('geometry').get('coordinates')
item = geometry.Polygon(item[0][0])
if item.intersects(ground_polygon):
local_intersection = ground_polygon.intersection(item)
print(str(ground_polygon.area))
print(str(local_intersection.area))
# ground_polygon = ground_polygon.difference(local_intersection)
ground_polygon = (ground_polygon.symmetric_difference(
local_intersection)).difference(local_intersection)
else:
del products[id]
import datetime
from datetime import timedelta
S2_geojson = read_geojson(path)
start_S1_date = S2_geojson.get('features')[0].get('properties').get(
'ingestiondate')
start_S1_date = start_S1_date.split('T')[0]
start_S1_date = datetime.datetime.strptime(start_S1_date,
'%Y-%m-%d').date()
## New end date for S1
end_S1_date = start_S1_date + timedelta(days=7)
start_S1_date = start_S1_date - timedelta(days=7)
start_S1_date_str = str(start_S1_date).replace('-', '')
end_S1_date_str = str(end_S1_date).replace('-', '')
## COMBINE FOOTPRINT
geom_in_geojson = []
geom_in_geojson.append(
geojson.Feature(geometry=ground_polygon,
properties={"MissingData": "Test"}))
feature_collection = FeatureCollection(geom_in_geojson)
pathToFile = r'C:\Users\panda\Sat_paper\missing.geojson'
with open(pathToFile, 'w') as f:
dump(feature_collection, f)
print("Done")
class Downloader(object):
def __init__(self,
username,
password,
satellite,
order_id,
directory=Path('/data/')):
# The connection to ESA scihub
self.api = SentinelAPI(username, password,
'https://scihub.copernicus.eu/dhus')
# Sentinel-5p currently has its own pre-operations hub
self.api_s5p = SentinelAPI(user='******',
password='******',
api_url='https://s5phub.copernicus.eu/dhus')
# Use the current datetime to name the download order
self.order_id = order_id
# Use ordered dict to store the metadata of the queries products
self.products = OrderedDict()
self.satellite = satellite
self.directory = directory
# if not self.directory.exists(): # Create directory if it does not exist
# os.makedirs(self.directory)
def query(self, footprint, startdate, enddate):
if self.satellite == 's1' or self.satellite == 'all':
self.query_s1(footprint, startdate, enddate)
if self.satellite == 's2' or self.satellite == 'all':
self.query_s2(footprint, startdate, enddate)
if self.satellite == 's3' or self.satellite == 'all':
self.query_s3(footprint, startdate, enddate)
if self.satellite == 's5p' or self.satellite == 'all':
self.query_s5p(footprint, startdate, enddate)
def query_s1(self, footprint, startdate, enddate):
# Define producttypes (here it is Sentinel-1 GRDH products)
producttypes = ['GRD']
# Loop over producttypes and update the query dictionary
# TODO: Fix this inefficient way of querying the relative orbits
if FLAGS.s1_relative_orbit == [0]:
for producttype in producttypes:
queried_products = self.api.query(footprint,
date=(startdate, enddate),
platformname='Sentinel-1',
producttype=producttype,
sensoroperationalmode='IW')
self.products.update(queried_products)
else:
for producttype in producttypes:
for relative_orbit in FLAGS.s1_relative_orbit:
queried_products = self.api.query(
footprint,
date=(startdate, enddate),
platformname='Sentinel-1',
producttype=producttype,
sensoroperationalmode='IW',
relativeorbitnumber=relative_orbit)
self.products.update(queried_products)
def query_s2(self, footprint, startdate, enddate):
# Load parameters from FLAGS
max_cloudcoverage = FLAGS.s2_max_cloudcoverage
# Define producttypes (here it is Sentinel-2 L2A products)
producttypes = [
'S2MSI2Ap', 'S2MSI2A'
] # Producttype names differ depending on the year they were published
# Loop over producttypes and update the query dictionary
# TODO: Fix this inefficient way of querying the relative orbits
if FLAGS.s2_relative_orbit == [0]:
for producttype in producttypes:
queried_products = self.api.query(
footprint,
date=(startdate, enddate),
platformname='Sentinel-2',
producttype=producttype,
cloudcoverpercentage=(0, max_cloudcoverage))
self.products.update(queried_products)
else:
for producttype in producttypes:
for relative_orbit in FLAGS.s2_relative_orbit:
queried_products = self.api.query(
footprint,
date=(startdate, enddate),
platformname='Sentinel-2',
relativeorbitnumber=relative_orbit,
producttype=producttype,
cloudcoverpercentage=(0, max_cloudcoverage))
self.products.update(queried_products)
def query_s3(self, footprint, startdate, enddate):
queried_products = self.api.query(footprint,
date=(startdate, enddate),
platformname='Sentinel-3',
producttype='SL_2_LST___',
productlevel='L2')
self.products.update(queried_products)
def query_s5p(self, footprint, startdate, enddate):
kwargs = {}
producttypedescriptions = [
'Ozone', 'Sulphur Dioxide', 'Nitrogen Dioxide', 'Methane',
'Formaldehyde', 'Carbon Monoxide', 'Aerosol Index',
'Aerosol Layer Height', 'Cloud'
]
# producttypedescriptions = ['Ozone']
# Loop over producttypes and update the query dictionary
for producttypedescription in producttypedescriptions:
queried_products = self.api_s5p.query(
footprint,
date=(startdate, enddate),
platformname='Sentinel-5 Precursor',
processinglevel='L2',
producttypedescription=producttypedescription,
**kwargs)
# Remove any 'Suomi-NPP VIIRS Clouds' products which are returned as 'Cloud' (they shouldn't have been)
# https://sentinel.esa.int/web/sentinel/technical-guides/sentinel-5p/products-algorithms
if producttypedescription == 'Cloud':
temp_queried_products = queried_products.copy()
for key in queried_products.keys():
if queried_products[key][
'producttypedescription'] != 'Cloud':
del temp_queried_products[key]
queried_products = temp_queried_products
self.products.update(queried_products)
def print_num_and_size_of_products(self):
logging.info('Number of products = ' + str(len(list(self.products))))
logging.info('Total size [GB] = ' +
str(self.api.get_products_size(self.products)))
# https://sentinelsat.readthedocs.io/en/master/api.html#lta-products
# TODO: Get LTA retrieval to work properly (install of newest sentinelsat version is in dockerfile)
# Retry every 30 min (+10 second buffertime) to request LTA products.
@tenacity.retry(stop=tenacity.stop_after_attempt(200),
wait=tenacity.wait_fixed(1810))
def download_zipfiles(self):
zipfiles_directory = self.directory / 'zipfiles'
if not zipfiles_directory.exists(
): # Create directory if it does not exist
os.makedirs(zipfiles_directory)
# Get the products to be downloaded. The sample() funcitons permutes the dataframe, such that a new LTA product
# is request at every retry. The optimal solution would have been to rearrange the dataframe by rotating the
# index at every retry, but this is a quick and dirty way to achieve something similar.
# (https://stackoverflow.com/a/34879805/12045808).
products_df = self.queried_products_as_df().sample(frac=1)
# NOTE: The code below is only useful while the Sentinel-5p has a different api than the others. After this has
# been fixed, the code should be reduced to the following single line:
# Download all zipfiles (it automatically checks if zipfiles already exist)
# self.api.download_all(self.products, directory_path=zipfiles_directory) # Download all zipfiles
# But for now, use the following code:
non_s5p_products = products_df[
products_df['platformname'] != 'Sentinel-5 Precursor']
s5p_products = products_df[products_df['platformname'] ==
'Sentinel-5 Precursor']
if len(non_s5p_products):
logging.info("Downloading Sentinel-1/2/3 products")
self.api.download_all(non_s5p_products.to_dict(into=OrderedDict,
orient='index'),
directory_path=zipfiles_directory)
else:
logging.info("No Sentinel-1/2/3 products found in query")
if len(s5p_products):
logging.info("Downloading Sentinel-5p products")
self.api_s5p.download_all(s5p_products.to_dict(into=OrderedDict,
orient='index'),
directory_path=zipfiles_directory)
else:
logging.info("No Sentinel-5p products found in query")
# The Sentinel-5p data has wrongly been given the filetype .zip, but it should be .nc, so make a copy with
# .nc extension. A copy is made instead of renaming so sentinelsat doesn't re-download the file every time
# it is run.
s5p_downloaded_files = zipfiles_directory.glob('S5P*.zip')
logging.debug(
"Renaming downloaded Sentinel-5p files from .zip to .nc (due to bug in SentinelSat)"
)
for file in s5p_downloaded_files:
if not file.with_suffix('.nc').exists():
shutil.copy(str(file), str(file.with_suffix('.nc')))
def queried_products_as_geojson(self):
return self.api.to_geojson(self.products)
def queried_products_as_df(self):
return self.api.to_dataframe(self.products)
def save_queried_products(self):
orders_directory = self.directory / 'orders'
if not orders_directory.exists():
os.makedirs(orders_directory)
# Save the queried products to a geojson file (e.g. to be loaded into QGIS)
geojson_path = (self.directory / 'orders' /
self.order_id).with_suffix('.geojson')
with geojson_path.open('w') as geojson_file:
geojson_data = self.api.to_geojson(self.products)
geojson_file.write(str(geojson_data))
# Save the queried products as pandas df in a pkl file (preferred format when working in Python)
df_path = (self.directory / 'orders' /
self.order_id).with_suffix('.pkl')
df = self.api.to_dataframe(self.products)
df.to_pickle(df_path)
condition = True
while condition:
try:
api = SentinelAPI('USERNAME', 'PASSWORD', 'https://scihub.copernicus.eu/dhus')
footprint = geojson_to_wkt(read_geojson('goldmine_sa.geojson'))
products = api.query(
footprint,
date=(_date.strftime('%Y%m%d'), (_date+timedelta(days=1)).strftime('%Y%m%d')),
platformname='Sentinel-1',
producttype='GRD'
)
except Exception as e:
print('{} for date: {}'.format(e, _date.strftime('%Y-%m-%d')))
logging.error('{} for date: {}'.format(e, _date.strftime('%Y-%m-%d')))
else:
fc = api.to_geojson(products)
features = fc['features']
if len(features):
print(_date)
for feature in features:
properties = feature.properties
# properties
id = properties['id']
identifier = properties['identifier']
title = properties['title']
footprint = str(feature['geometry'])
#slave
acquisitiontype = properties['acquisitiontype']
beginposition = properties['beginposition']
endposition = properties['endposition']
filename = properties['filename']
api = SentinelAPI('fabiolana', 'Albertone_2016')
footprint = geojson_to_wkt(read_geojson('/media/sf_Downloads/__effis/ricerca.geojson'))
products = api.query(footprint,
# date=('20170801', date(2017, 12, 15)),
date=('20151001', '20171205'),
platformname='Sentinel-2',
# producttype='SLC')
#,orbitdirection='ASCENDING') #,
cloudcoverpercentage=(0, 10))
# download all results from the search
# api.download_all(products)
# GeoJSON FeatureCollection containing footprints and metadata of the scenes
api.to_geojson(products)
# inserito in dataframe
df_prods = pd.DataFrame(api.to_geojson(products))
# print df_prods
# GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries
print api.to_geodataframe(products)
# Get basic information about the product: its title, file size, MD5 sum, date, footprint and
# its download url
# api.get_product_odata(<product_id>)
# Get the product's full metadata available on the server
# api.get_product_odata(<product_id>, full=True)
# todo check for processed data len(products) # convert to Pandas DataFrame products_df = api.to_dataframe(products) # sort for most recent and lowest cloud cover products_df_sorted = products_df.sort_values(['ingestiondate', 'cloudcoverpercentage'], ascending=[True, True]) test = products_df_sorted.head(1) test['cloudcoverpercentage'] test['ingestiondate'] set(products_df['processinglevel']) # GeoJSON FeatureCollection containing footprints and metadata of the scenes geojson_products = api.to_geojson(products) # GeoPandas GeoDataFrame with the metadata of the scenes and the footprints as geometries geodata_products = api.to_geodataframe(products) # plot product time vs cloudcover # data = [go.Scatter(x=products_df_sorted.ingestiondate, y=products_df_sorted[['cloudcoverpercentage']])] # py.plotly.iplot(data, filename = 'time-series-simple') # download api.download_all(test.index) # unzip # todo: needs loop to run through the data sets directory_to_extract_to = "unzip" zip = zipfile.ZipFile(str(test.title[0]) + '.zip')
def query_sentinel_data(cfg, save_json=True):
workpath = Path(os.getcwd())
# api = SentinelAPI('puzhao', 'kth10044ESA!', 'https://scihub.copernicus.eu/dhus')
# user, password = "******", "19940911"
api = SentinelAPI(cfg.user, cfg.password,
'https://scihub.copernicus.eu/dhus')
now = datetime.now().strftime("%Y-%m-%dT%H%M%S")
today = datetime.today().strftime("%Y-%m-%d")
if cfg.start_date is None:
cfg.start_date = (datetime.today() +
timedelta(-1)).strftime("%Y-%m-%d")
if cfg.end_date is None:
cfg.end_date = (datetime.today() + timedelta(1)).strftime("%Y-%m-%d")
print("now: ", now)
cfg.update({
"roi_url": cfg.roi_url,
'placename': "British Columbia",
"query_by": "roi", # 'place' has problem
"query_date": today,
"start_date": cfg.start_date,
"end_date": cfg.end_date,
"platformname": cfg.platformname, # Sentinel-2
"producttype": cfg.producttype, # S2MSI1C, S2MSI2A
# 'relativeorbitnumber': 84,
# "orbitdirection": "ASCENDING",
"download_flag": cfg.download_flag,
"download_one": True, # download one by one
"download_all": True, # download all once
})
pprint(cfg)
Sat_Abb_Dict = {'Sentinel-1': 'S1', 'Sentinel-2': 'S2', 'Sentinel-3': 'S3'}
SAT = Sat_Abb_Dict[cfg.platformname]
datafolder = Path(cfg.datafolder)
savePath = datafolder / "data" / f"{SAT}_{cfg.producttype}"
if not os.path.exists(savePath): os.makedirs(savePath)
cfg.update({"sat_folder": str(savePath)})
cfg.download_all = False if cfg.download_one else True
cfg.download_all = cfg.download_all and cfg.download_flag
cfg.download_one = cfg.download_one and cfg.download_flag
if cfg.query_by == "roi":
footprint = geojson_to_wkt(read_geojson(str(workpath / cfg.roi_url)))
roi_name = os.path.split(cfg.roi_url)[-1].split(".")[0]
if cfg.query_by == "place":
footprint = placename_to_wkt(cfg.placename)
roi_name = cfg.placename.replace(" ", "_")
# print(BC)
### DSC rorb = 22
if cfg.platformname == "Sentinel-1":
cfg.checkProperty = "system:index"
cfg.check_eeImgCol = "COPERNICUS/S1_GRD"
products = api.query(
footprint,
date=(cfg.start_date.replace("-",
""), cfg.end_date.replace("-", "")),
platformname=cfg.platformname,
producttype=cfg.producttype,
order_by='+beginposition',
)
else: # S2, S3 ...
cfg.checkProperty = "PRODUCT_ID"
cfg.check_eeImgCol = "COPERNICUS/S2" if 'S2MSI1C' == cfg.producttype else "COPERNICUS/S2_SR"
products = api.query(
footprint,
date=(cfg.start_date.replace("-",
""), cfg.end_date.replace("-", "")),
platformname=cfg.platformname,
producttype=cfg.producttype,
order_by='+beginposition',
cloudcoverpercentage=(0, cfg.cloudcoverpercentage), # for S2 only
)
# print(products['0c05435b-0cd3-45a0-93f4-8c317eb1d558'])
print("\n\n===========> Sentinel Auto-Query <============")
products_df = api.to_dataframe(products)
# print(products_df.keys())
# print(products_df.index)
# pprint(products_df[['sensoroperationalmode', 'orbitdirection', 'relativeorbitnumber']])
products_dict = products_df.transpose().to_dict()
products_list = products_df.index.tolist()
if len(products_list) > 0:
example_dict = products_dict[products_list[0]]
property_list = [
key for key in example_dict.keys()
if is_jsonable(example_dict[key])
]
# pprint(products_dict.keys())
# select property for saving to json
orbit_dict = {'ASCENDING': 'ASC', 'DESCENDING': 'DSC'}
products_QueryInfo = edict()
checkImgCol = ee.ImageCollection(f"{cfg.check_eeImgCol}")
if SAT == "S1":
sentinel_asset = ee.ImageCollection("users/omegazhangpzh/Sentinel1")
if SAT == "S2":
sentinel_asset = ee.ImageCollection("users/omegazhangpzh/Sentinel2")
for product_id in products_dict.keys():
title = products_dict[product_id]['title']
filtered_size = ee.Number(checkImgCol.filter(ee.Filter.eq(cfg.checkProperty, title)).size())\
.add(sentinel_asset.filter(ee.Filter.eq(cfg.checkProperty, title)).size()).getInfo()
flag = filtered_size > 0
print(title, flag)
# flag = False
if not flag: # if this product is not available in GEE
# print(title)
# print(title, flag.getInfo())
products_QueryInfo[title] = {
key: products_dict[product_id][key]
for key in property_list
}
# products_QueryInfo[title]['product_id'] = product_id
orbit_direction = products_dict[product_id]['orbitdirection']
orbit_num = products_dict[product_id]['relativeorbitnumber']
products_QueryInfo[title]['orbit_key'] = orbit_dict[
orbit_direction] + "_" + str(orbit_num)
QueryInfo = edict()
QueryInfo["products"] = products_QueryInfo
QueryInfo["results"] = edict()
QueryInfo["results"]['total_number'] = len(products_QueryInfo.keys())
QueryInfo["results"]['products_list'] = sorted(
list(products_QueryInfo.keys()))
QueryInfo["results"]['orbKey_list'] = list(
set([
products_QueryInfo[product]['orbit_key']
for product in list(products_QueryInfo.keys())
]))
QueryInfo["cfg"] = cfg
# roi_name = os.path.split(cfg.roi_url)[-1].split(".")[0]
jsonPath = datafolder / "outputs" / roi_name
if not os.path.exists(str(jsonPath)):
os.makedirs(jsonPath)
if save_json:
""" save as json """
json_url = jsonPath / f"{SAT}_{cfg.producttype}_{now}.json"
print("\njson_url: " + str(json_url))
with open(str(json_url), 'w') as fp:
json.dump(edict(QueryInfo), fp, ensure_ascii=False, indent=4)
""" save as geojson """
import geojson
with open(jsonPath / f"S1_{cfg.producttype}_{now}.geojson", 'w') as fp:
geojson.dump(api.to_geojson(products), fp, indent=4)
print()
# print(footprint)
print("now: ", now)
print("Total Number of Searched Products:" +
str(len(QueryInfo["results"]['products_list'])))
pprint(QueryInfo["results"]['products_list'])
return QueryInfo
