def find(self, provider, aoi, date_start, date_stop, clouds=None):
session = requests.Session()
session.auth = (self.user, self.pwd)
session.stream = True
acquisition_date = '(beginPosition:[%s TO %s])' % (date_start.strftime(
'%Y-%m-%dT%H:%M:%SZ'), date_stop.strftime('%Y-%m-%dT%H:%M:%SZ'))
poly = Polygon(aoi)
geometry = wkt_dumps(poly)
query_area = ' AND (footprint:"Intersects(%s)")' % geometry
query = ''.join([acquisition_date, query_area])
response = requests.post(self.url, dict(q=query), auth=session.auth)
assert response.status_code == requests.codes.ok, 'Connection to copernicus server went wrong [%d]. Please check %s. \\n%s' % \
(response.status_code, self.url, response.text)
products = response.json()['feed']['entry']
datasets = set()
print products
for p in products:
ds = Catalog_Dataset()
ds.entity_id = p['title']
ds.acq_time = next(x for x in p["date"]
if x["name"] == "beginposition")["content"]
ds.sensor = next(x for x in p["str"]
if x["name"] == "platformname")["content"]
resource_url = next(x for x in p["link"]
if len(x.keys()) == 1)["href"]
if ds.sensor == 'Sentinel-2':
# ds.tile_identifier = r['tile_identifier']
ds.clouds = p['double']['content']
ds.level = next(x for x in p["str"]
if x["name"] == "processinglevel")["content"]
daynight = 'day'
if next(x for x in p["str"] if x["name"] ==
"orbitdirection")["content"] != 'DESCENDING':
daynight = 'night'
ds.daynight = daynight
cop = CopernicusSciHubContainer()
cop.http = resource_url
container = cop.to_dict()
s3 = S3PublicContainer()
if remote_file_exists(SENTINEL_S3_HTTP_ZIP_BASEURL +
ds.entity_id + '.zip'):
s3.http = SENTINEL_S3_HTTP_ZIP_BASEURL + ds.entity_id + '.zip'
if public_key_exists('sentinel-s2-l1c',
'zips/%s.zip' % ds.entity_id):
s3.bucket = SENTINEL_S3_BUCKET
s3.prefix = 'zips/%s.zip' % ds.entity_id
if s3.http != None or s3.bucket != None:
container.update(s3.to_dict())
# print s3.to_dict()
ds.container = container
datasets.add(ds)
return datasets
def sentinel_harvester(in_csv, N, M=1000):
datasets = []
with open(in_csv, 'r') as f:
for counter, line in enumerate(f):
content_list = line.split(' ')
tileinfokey = content_list[-1]
tileinfokey = tileinfokey.rstrip("\n")
quicklookkey = tileinfokey.replace('tileInfo.json', 'preview.jpg')
if counter < N + M and counter >= N:
if public_key_exists(SENTINEL_S3_BUCKET,
tileinfokey) and public_key_exists(
SENTINEL_S3_BUCKET, quicklookkey):
tilenfodict = ujson.loads(
public_get_filestream(SENTINEL_S3_BUCKET, tileinfokey))
productkey = tilenfodict['productPath']
s3 = SentinelS3Container()
s3.bucket = SENTINEL_S3_BUCKET
s3.tile = tilenfodict['path'] + '/'
s3.quicklook = quicklookkey
dataset = Catalog_Dataset()
dataset.entity_id = tilenfodict['productName']
dataset.tile_identifier = '%02d%s%s' % (
tilenfodict['utmZone'], tilenfodict['latitudeBand'],
tilenfodict['gridSquare'])
dataset.clouds = tilenfodict['cloudyPixelPercentage']
dataset.acq_time = dateutil.parser.parse(
tilenfodict['timestamp'])
if public_key_exists(SENTINEL_S3_BUCKET,
productkey + '/metadata.xml'):
s3.product = productkey + '/'
metadatakey = productkey + '/metadata.xml'
metadatadict = xmltodict.parse(
public_get_filestream(SENTINEL_S3_BUCKET,
metadatakey))
dataset.sensor = \
metadatadict['n1:Level-1C_User_Product']['n1:General_Info']['Product_Info']['Datatake'][
'SPACECRAFT_NAME']
dataset.level = metadatadict[
'n1:Level-1C_User_Product']['n1:General_Info'][
'Product_Info']['PROCESSING_LEVEL']
daynight = 'day'
if metadatadict['n1:Level-1C_User_Product'][
'n1:General_Info']['Product_Info']['Datatake'][
'SENSING_ORBIT_DIRECTION'] != 'DESCENDING':
daynight = 'night'
dataset.daynight = daynight
quicklookurl = SENTINEL_S3_HTTP_BASEURL + tilenfodict[
'path'] + '/preview.jpg'
metadataurl = SENTINEL_S3_HTTP_BASEURL + productkey + '/metadata.xml'
container = dict()
if remote_file_exists(quicklookurl):
container['quicklook'] = quicklookurl
if remote_file_exists(metadataurl):
container['metadata'] = metadataurl
if remote_file_exists(SENTINEL_S3_HTTP_ZIP_BASEURL +
dataset.entity_id + '.zip'):
s3.zip = SENTINEL_S3_HTTP_ZIP_BASEURL + dataset.entity_id + '.zip'
if s3.zip != None or s3.bucket != None:
container.update(s3.to_dict())
dataset.resources = container
datasets.append(dataset)
print counter, 'processed...', N
return datasets
def testPublicExistsFalse(self):
self.assertFalse(public_key_exists('sentinel-s2-l1c', 'tiles/44/W/PS/2016/8/15/1/preview'))
def generate_s2_tile_information(tile_path):
tileinfokey = os.path.join(tile_path, 'tileInfo.json')
quicklookkey = os.path.join(tile_path, 'preview.jp2')
if public_key_exists(SENTINEL_S3_BUCKET,
tileinfokey) and public_key_exists(
SENTINEL_S3_BUCKET, quicklookkey):
tilenfodict = ujson.loads(
public_get_filestream(SENTINEL_S3_BUCKET, tileinfokey))
productkey = tilenfodict['productPath']
s3 = SentinelS3Container()
s3.bucket = SENTINEL_S3_BUCKET
s3.tile = tilenfodict['path'] + '/'
s3.quicklook = quicklookkey
dataset = Catalog_Dataset()
dataset.entity_id = tilenfodict['productName']
dataset.tile_identifier = '%02d%s%s' % (tilenfodict['utmZone'],
tilenfodict['latitudeBand'],
tilenfodict['gridSquare'])
dataset.clouds = tilenfodict['cloudyPixelPercentage']
dataset.acq_time = dateutil.parser.parse(tilenfodict['timestamp'])
if public_key_exists(SENTINEL_S3_BUCKET, productkey + '/metadata.xml'):
s3.product = productkey + '/'
metadatakey = productkey + '/metadata.xml'
metadatadict = xmltodict.parse(
public_get_filestream(SENTINEL_S3_BUCKET, metadatakey))
dataset.sensor = \
metadatadict['n1:Level-1C_User_Product']['n1:General_Info']['Product_Info']['Datatake'][
'SPACECRAFT_NAME']
dataset.level = metadatadict['n1:Level-1C_User_Product'][
'n1:General_Info']['Product_Info']['PROCESSING_LEVEL']
daynight = 'day'
if metadatadict['n1:Level-1C_User_Product']['n1:General_Info'][
'Product_Info']['Datatake'][
'SENSING_ORBIT_DIRECTION'] != 'DESCENDING':
daynight = 'night'
dataset.daynight = daynight
quicklookurl = SENTINEL_S3_HTTP_BASEURL + tilenfodict[
'path'] + '/preview.jp2'
metadataurl = SENTINEL_S3_HTTP_BASEURL + productkey + '/metadata.xml'
container = dict()
if remote_file_exists(quicklookurl):
container['quicklook'] = quicklookurl
if remote_file_exists(metadataurl):
container['metadata'] = metadataurl
if remote_file_exists(SENTINEL_S3_HTTP_ZIP_BASEURL +
dataset.entity_id + '.zip'):
s3.zip = SENTINEL_S3_HTTP_ZIP_BASEURL + dataset.entity_id + '.zip'
if s3.zip is not None or s3.bucket is not None:
container.update(s3.to_dict())
dataset.resources = container
return dataset
else:
logger.warn(
"No quicklook and/or tileinfo metadata file in bucket found: %s" %
tile_path)
return None