def start_process(self, params):
cube_id = get_cube_id(params['datacube'], 'MED')
tiles = params['tiles'].split(',')
start_date = datetime.strptime(params['start_date'],
'%Y-%m-%d').strftime('%Y-%m-%d')
end_date = datetime.strptime(params['end_date'], '%Y-%m-%d').strftime('%Y-%m-%d') \
if params.get('end_date') else datetime.now().strftime('%Y-%m-%d')
# verify cube info
cube_infos = Collection.query().filter(
Collection.id == cube_id).first()
if not cube_infos:
return 'Cube not found!', 404
# get bands list
bands = Band.query().filter(
Band.collection_id == get_cube_id(params['datacube'])).all()
bands_list = [band.name for band in bands]
# items => old mosaic
# orchestrate
self.score['items'] = orchestrate(params['datacube'], cube_infos,
tiles, start_date, end_date)
# prepare merge
prepare_merge(self, params['datacube'],
params['collections'].split(','), bands_list,
cube_infos.bands_quicklook, bands[0].resolution_x,
bands[0].resolution_y, bands[0].fill,
cube_infos.raster_size_schemas.raster_size_x,
cube_infos.raster_size_schemas.raster_size_y,
cube_infos.raster_size_schemas.chunk_size_x,
cube_infos.grs_schema.crs)
return 'Succesfully', 201
def load_items(fixture_path: str):
"""Load default items and assets to database.
Args:
fixture_path - Path relative to fixtures. i.e 'data/items.json'
"""
items = json_parser(resource_string(__name__, fixture_path))
with db.session.begin_nested():
for item in items:
assets = item.pop('assets')
c_item = CollectionItem(**item)
c_item.save(commit=False)
c_tile = CollectionTile()
c_tile.collection_id = c_item.collection_id
c_tile.grs_schema_id = c_item.grs_schema_id
c_tile.tile_id = c_item.tile_id
c_tile.save(commit=False)
for asset in assets:
b = Band.query().filter(
Band.collection_id == c_item.collection_id,
Band.name == asset.pop('band')).one()
a = Asset(**asset)
a.band = b
a.save(commit=False)
def load_collections(fixture_path: str):
"""Load default collections to database.
Args:
fixture_path - Path relative to fixtures. i.e 'data/tiles.json'
"""
collections = json_parser(resource_string(__name__, fixture_path))
with db.session.begin_nested():
for collection in collections:
bands = collection.pop('bands')
c = Collection(**collection)
c.save(commit=False)
for band in bands:
b = Band(**band)
b.collection = c
b.save(commit=False)
def publish(self, activity):
print('==> start PUBLISH')
services = self.services
activity['mystart'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
warped_cube = '_'.join(activity['datacube'].split('_')[0:2])
# Generate quicklooks for CUBES (MEDIAN, STACK ...)
qlbands = activity['quicklook'].split(',')
for function in ['MED', 'STK']:
cube_id = get_cube_id(activity['datacube'], function)
general_scene_id = '{}_{}_{}_{}'.format(
cube_id, activity['tileid'], activity['start'], activity['end'])
qlfiles = []
for band in qlbands:
qlfiles.append(services.prefix + activity['blended'][band][function + 'file'])
pngname = generateQLook(general_scene_id, qlfiles)
dirname_ql = activity['dirname'].replace(
'{}/'.format(warped_cube), '{}/'.format(cube_id))
if pngname is None:
print('publish - Error generateQLook for {}'.format(general_scene_id))
return False
s3pngname = os.path.join(dirname_ql, '{}_{}'.format(activity['start'], activity['end']), os.path.basename(pngname))
services.upload_file_S3(pngname, s3pngname, {'ACL': 'public-read'})
os.remove(pngname)
# Generate quicklooks for all ARD scenes (WARPED)
for datedataset in activity['scenes']:
scene = activity['scenes'][datedataset]
cube_id = get_cube_id(activity['datacube'])
general_scene_id = '{}_{}_{}'.format(
cube_id, activity['tileid'], str(scene['date'])[0:10])
qlfiles = []
for band in qlbands:
filename = os.path.join(services.prefix + activity['dirname'], str(scene['date'])[0:10], scene['ARDfiles'][band])
qlfiles.append(filename)
pngname = generateQLook(general_scene_id, qlfiles)
if pngname is None:
print('publish - Error generateQLook for {}'.format(general_scene_id))
return False
s3pngname = os.path.join(activity['dirname'], str(scene['date'])[0:10], os.path.basename(pngname))
services.upload_file_S3(pngname, s3pngname, {'ACL': 'public-read'})
os.remove(pngname)
# register collection_items and assets in DB (MEDIAN, STACK ...)
for function in ['MED', 'STK']:
cube_id = '{}_{}'.format(activity['datacube'], function)
cube = Collection.query().filter(
Collection.id == cube_id
).first()
if not cube:
print('cube {} not found!'.format(cube_id))
continue
general_scene_id = '{}_{}_{}_{}'.format(
cube_id, activity['tileid'], activity['start'], activity['end'])
# delete collection_items and assets if exists
assets = Asset.query().filter(
Asset.collection_item_id == general_scene_id
).all()
for asset in assets:
db.session().delete(asset)
db.session().commit()
coll_item = CollectionItem.query().filter(
CollectionItem.id == general_scene_id
).first()
if coll_item:
db.session().delete(coll_item)
db.session().commit()
# insert 'collection_item'
range_date = '{}_{}'.format(activity['start'], activity['end'])
png_name = '{}.png'.format(general_scene_id)
dirname_ql = activity['dirname'].replace(
'{}/'.format(warped_cube), '{}/'.format(cube_id))
s3_pngname = os.path.join(dirname_ql, range_date, png_name)
CollectionItem(
id=general_scene_id,
collection_id=cube_id,
grs_schema_id=cube.grs_schema_id,
tile_id=activity['tileid'],
item_date=activity['start'],
composite_start=activity['start'],
composite_end=activity['end'],
quicklook='{}/{}'.format(BUCKET_NAME, s3_pngname),
cloud_cover=activity['cloudratio'],
scene_type=function,
compressed_file=None
).save()
# insert 'assets'
bands_by_cube = Band.query().filter(
Band.collection_id == cube_id
).all()
for band in activity['bands']:
if band == 'quality':
continue
band_id = list(filter(lambda b: str(b.common_name) == band, bands_by_cube))
if not band_id:
print('band {} not found!'.format(band))
continue
Asset(
collection_id=cube_id,
band_id=band_id[0].id,
grs_schema_id=cube.grs_schema_id,
tile_id=activity['tileid'],
collection_item_id=general_scene_id,
url='{}/{}'.format(BUCKET_NAME, activity['blended'][band][function + 'file']),
source=None,
raster_size_x=activity['raster_size_x'],
raster_size_y=activity['raster_size_y'],
raster_size_t=1,
chunk_size_x=activity['chunk_size_x'],
chunk_size_y=activity['chunk_size_y'],
chunk_size_t=1
).save()
# Register all ARD scenes - WARPED Collection
for datedataset in activity['scenes']:
scene = activity['scenes'][datedataset]
cube_id = get_cube_id(activity['datacube'])
cube = Collection.query().filter(
Collection.id == cube_id
).first()
if not cube:
print('cube {} not found!'.format(cube_id))
continue
general_scene_id = '{}_{}_{}'.format(
cube_id, activity['tileid'], str(scene['date'])[0:10])
# delete 'assets' and 'collection_items' if exists
assets = Asset.query().filter(
Asset.collection_item_id == general_scene_id
).all()
for asset in assets:
db.session().delete(asset)
db.session().commit()
coll_item = CollectionItem.query().filter(
CollectionItem.id == general_scene_id
).first()
if coll_item:
db.session().delete(coll_item)
db.session().commit()
# insert 'collection_item'
pngname = '{}.png'.format(general_scene_id)
s3pngname = os.path.join(activity['dirname'], str(scene['date'])[0:10], pngname)
CollectionItem(
id=general_scene_id,
collection_id=cube_id,
grs_schema_id=cube.grs_schema_id,
tile_id=activity['tileid'],
item_date=scene['date'],
composite_start=scene['date'],
composite_end=scene['date'],
quicklook='{}/{}'.format(BUCKET_NAME, s3pngname),
cloud_cover=int(scene['cloudratio']),
scene_type='WARPED',
compressed_file=None
).save()
# insert 'assets'
bands_by_cube = Band.query().filter(
Band.collection_id == cube_id
).all()
for band in activity['bands']:
if band not in scene['ARDfiles']:
print('publish - problem - band {} not in scene[files]'.format(band))
continue
band_id = list(filter(lambda b: str(b.common_name) == band, bands_by_cube))
if not band_id:
print('band {} not found!'.format(band))
continue
raster_size_x = scene['raster_size_x'] if scene.get('raster_size_x') else activity.get('raster_size_x')
raster_size_y = scene['raster_size_y'] if scene.get('raster_size_y') else activity.get('raster_size_y')
block_size = scene['block_size'] if scene.get('block_size') else activity.get('block_size')
Asset(
collection_id=cube_id,
band_id=band_id[0].id,
grs_schema_id=cube.grs_schema_id,
tile_id=activity['tileid'],
collection_item_id=general_scene_id,
url='{}/{}'.format(BUCKET_NAME, os.path.join(activity['dirname'], str(scene['date'])[0:10], scene['ARDfiles'][band])),
source=None,
raster_size_x=raster_size_x,
raster_size_y=raster_size_y,
raster_size_t=1,
chunk_size_x=block_size,
chunk_size_y=block_size,
chunk_size_t=1
).save()
# Update status and end time in DynamoDB
activity['myend'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
activity['mystatus'] = 'DONE'
services.put_item_kinesis(activity)
refresh_materialized_view(db.session, AssetMV.__table__)
return True
def create_cube(self, params):
params['composite_function_list'] = ['IDENTITY', 'STK', 'MED']
# generate cubes metadata
cubes_db = Collection.query().filter().all()
cubes = []
cubes_serealized = []
for composite_function in params['composite_function_list']:
c_function_id = composite_function.upper()
raster_size_id = '{}-{}'.format(params['grs'],
int(params['resolution']))
cube_id = get_cube_id(params['datacube'], c_function_id)
# add cube
if not list(filter(lambda x: x.id == cube_id, cubes)) and not list(
filter(lambda x: x.id == cube_id, cubes_db)):
cube = Collection(
id=cube_id,
temporal_composition_schema_id=params['temporal_schema']
if c_function_id.upper() != 'IDENTITY' else 'Anull',
raster_size_schema_id=raster_size_id,
composite_function_schema_id=c_function_id,
grs_schema_id=params['grs'],
description=params['description'],
radiometric_processing=None,
geometry_processing=None,
sensor=None,
is_cube=True,
oauth_scope=params.get('oauth_scope', None),
license=params['license'],
bands_quicklook=','.join(params['bands_quicklook']),
metadata=params['metadata'])
cubes.append(cube)
cubes_serealized.append(Serializer.serialize(cube))
BaseModel.save_all(cubes)
bands = []
for cube in cubes:
# save bands
for band in params['bands']:
band = band.strip()
if (band == 'cnc' and cube.composite_function_schema_id == 'IDENTITY') or \
(band =='quality' and cube.composite_function_schema_id != 'IDENTITY'):
continue
is_not_cloud = band != 'quality' and band != 'cnc'
bands.append(
Band(name=band,
collection_id=cube.id,
min=0 if is_not_cloud else 0,
max=10000 if is_not_cloud else 255,
fill=-9999 if is_not_cloud else 0,
scale=0.0001 if is_not_cloud else 1,
data_type='int16' if is_not_cloud else 'Uint16',
common_name=band,
resolution_x=params['resolution'],
resolution_y=params['resolution'],
resolution_unit='m',
description='',
mime_type='image/tiff'))
BaseModel.save_all(bands)
return cubes_serealized, 201
def orchestrate(self):
"""Orchestrate datacube defintion and prepare temporal resolutions."""
self.datacube = Collection.query().filter(
Collection.id == self.params['datacube']).one()
temporal_schema = self.datacube.temporal_composition_schema.temporal_schema
temporal_step = self.datacube.temporal_composition_schema.temporal_composite_t
# Create tiles
self.create_tiles(self.params['tiles'], self.datacube)
cube_start_date = self.params['start_date']
dstart = self.params['start_date']
dend = self.params['end_date']
if cube_start_date is None:
cube_start_date = dstart.strftime('%Y-%m-%d')
cube_end_date = dend.strftime('%Y-%m-%d')
periodlist = decode_periods(temporal_schema, cube_start_date,
cube_end_date, int(temporal_step))
where = [Tile.grs_schema_id == self.datacube.grs_schema_id]
if self.params.get('tiles'):
where.append(Tile.id.in_(self.params['tiles']))
self.tiles = Tile.query().filter(*where).all()
self.bands = Band.query().filter(
Band.collection_id == self.warped_datacube.id).all()
number_cols = int(self.datacube.raster_size_schemas.raster_size_x)
number_rows = int(self.datacube.raster_size_schemas.raster_size_y)
for tile in self.tiles:
self.mosaics[tile.id] = dict(periods=dict())
for datekey in sorted(periodlist):
requested_period = periodlist[datekey]
for periodkey in requested_period:
_, startdate, enddate = periodkey.split('_')
if dstart is not None and startdate < dstart.strftime(
'%Y-%m-%d'):
continue
if dend is not None and enddate > dend.strftime(
'%Y-%m-%d'):
continue
self.mosaics[tile.id]['periods'][periodkey] = {}
self.mosaics[
tile.id]['periods'][periodkey]['start'] = startdate
self.mosaics[
tile.id]['periods'][periodkey]['end'] = enddate
self.mosaics[
tile.id]['periods'][periodkey]['cols'] = number_cols
self.mosaics[
tile.id]['periods'][periodkey]['rows'] = number_rows
self.mosaics[tile.id]['periods'][periodkey][
'dirname'] = '{}/{}/{}-{}/'.format(
self.datacube.id, tile.id, startdate, enddate)
def publish_merge(bands, datacube, dataset, tile_id, period, date, scenes):
item_id = '{}_{}_{}'.format(datacube.id, tile_id, date)
quick_look_name = '{}_{}_{}'.format(datacube.id, tile_id, date)
quick_look_file = os.path.join(
Config.DATA_DIR, 'Repository/Warped/{}/{}/{}/{}'.format(
datacube.id.replace('_WARPED', ''), tile_id, period,
quick_look_name))
cube_bands = Band.query().filter(Band.collection_id == datacube.id).all()
raster_size_schemas = datacube.raster_size_schemas
ql_files = []
for band in bands:
ql_files.append(scenes['ARDfiles'][band])
quick_look_file = generate_quick_look(quick_look_file, ql_files)
Asset.query().filter(Asset.collection_item_id == item_id).delete()
CollectionItem.query().filter(CollectionItem.id == item_id).delete()
with db.session.begin_nested():
CollectionItem(id=item_id,
collection_id=datacube.id,
grs_schema_id=datacube.grs_schema_id,
tile_id=tile_id,
item_date=date,
composite_start=date,
composite_end=period.split('_')[-1],
quicklook=quick_look_file.replace(Config.DATA_DIR, ''),
cloud_cover=scenes.get('cloudratio', 0),
scene_type='WARPED',
compressed_file=None).save(commit=False)
for band in scenes['ARDfiles']:
band_model = next(
filter(lambda b: b.common_name == band, cube_bands))
# Band does not exists on model
if not band_model:
logging.warning(
'Band {} of {} does not exist on database'.format(
band, datacube.id))
continue
asset_relative_path = scenes['ARDfiles'][band].replace(
Config.DATA_DIR, '')
Asset(collection_id=datacube.id,
band_id=band_model.id,
grs_schema_id=datacube.grs_schema_id,
tile_id=tile_id,
collection_item_id=item_id,
url='{}'.format(asset_relative_path),
source=None,
raster_size_x=raster_size_schemas.raster_size_x,
raster_size_y=raster_size_schemas.raster_size_y,
raster_size_t=1,
chunk_size_x=raster_size_schemas.chunk_size_x,
chunk_size_y=raster_size_schemas.chunk_size_y,
chunk_size_t=1).save(commit=False)
db.session.commit()
return quick_look_file
def publish_datacube(cube, bands, datacube, tile_id, period, scenes,
cloudratio):
start_date, end_date = period.split('_')
cube_bands = Band.query().filter(Band.collection_id == cube.id).all()
raster_size_schemas = cube.raster_size_schemas
for composite_function in ['MED', 'STK']:
item_datacube = '{}_{}'.format("_".join(cube.id.split('_')[:-1]),
composite_function)
item_id = '{}_{}_{}'.format(item_datacube, tile_id, period)
_datacube = build_datacube_name(datacube, composite_function)
quick_look_name = '{}_{}_{}'.format(_datacube, tile_id, period)
quick_look_relpath = 'Repository/Mosaic/{}/{}/{}/{}'.format(
_datacube, tile_id, period, quick_look_name)
quick_look_file = os.path.join(Config.DATA_DIR, quick_look_relpath)
ql_files = []
for band in bands:
ql_files.append(scenes[band][composite_function])
quick_look_file = generate_quick_look(quick_look_file, ql_files)
Asset.query().filter(Asset.collection_item_id == item_id).delete()
CollectionItem.query().filter(CollectionItem.id == item_id).delete()
with db.session.begin_nested():
CollectionItem(id=item_id,
collection_id=item_datacube,
grs_schema_id=cube.grs_schema_id,
tile_id=tile_id,
item_date=start_date,
composite_start=start_date,
composite_end=end_date,
quicklook=quick_look_file.replace(
Config.DATA_DIR, ''),
cloud_cover=cloudratio,
scene_type=composite_function,
compressed_file=None).save(commit=False)
for band in scenes:
if band == 'quality':
continue
band_model = next(
filter(lambda b: b.common_name == band, cube_bands))
# Band does not exists on model
if not band_model:
logging.warning(
'Band {} of {} does not exist on database'.format(
band, cube.id))
continue
asset_relative_path = scenes[band][composite_function].replace(
Config.DATA_DIR, '')
Asset(collection_id=item_datacube,
band_id=band_model.id,
grs_schema_id=cube.grs_schema_id,
tile_id=tile_id,
collection_item_id=item_id,
url='{}'.format(asset_relative_path),
source=None,
raster_size_x=raster_size_schemas.raster_size_x,
raster_size_y=raster_size_schemas.raster_size_y,
raster_size_t=1,
chunk_size_x=raster_size_schemas.chunk_size_x,
chunk_size_y=raster_size_schemas.chunk_size_y,
chunk_size_t=1).save(commit=False)
db.session.commit()
return quick_look_file
def orchestrate(self):
self.datacube = Collection.query().filter(
Collection.id == self.params['datacube']).one()
temporal_schema = self.datacube.temporal_composition_schema.temporal_schema
temporal_step = self.datacube.temporal_composition_schema.temporal_composite_t
# Create tiles
self.create_tiles(self.params['tiles'], self.datacube)
# TODO: Check in STAC for cube item
# datacube_stac = stac_cli.collection(self.datacube.id)
collections_items = CollectionItem.query().filter(
CollectionItem.collection_id == self.datacube.id,
CollectionItem.grs_schema_id ==
self.datacube.grs_schema_id).order_by(
CollectionItem.composite_start).all()
cube_start_date = self.params['start_date']
if list(
filter(lambda c_i: c_i.tile_id == self.params['tiles'][0],
collections_items)):
cube_start_date = collections_items[0].composite_start
dstart = self.params['start_date']
dend = self.params['end_date']
if cube_start_date is None:
cube_start_date = dstart.strftime('%Y-%m-%d')
cube_end_date = dend.strftime('%Y-%m-%d')
periodlist = decode_periods(temporal_schema, cube_start_date,
cube_end_date, int(temporal_step))
where = [Tile.grs_schema_id == self.datacube.grs_schema_id]
if self.params.get('tiles'):
where.append(Tile.id.in_(self.params['tiles']))
self.tiles = Tile.query().filter(*where).all()
self.bands = Band.query().filter(
Band.collection_id == self.datacube.id).all()
number_cols = self.datacube.raster_size_schemas.raster_size_x
number_rows = self.datacube.raster_size_schemas.raster_size_y
for tile in self.tiles:
self.mosaics[tile.id] = dict(periods=dict())
for datekey in sorted(periodlist):
requested_period = periodlist[datekey]
for periodkey in requested_period:
_, startdate, enddate = periodkey.split('_')
if dstart is not None and startdate < dstart.strftime(
'%Y-%m-%d'):
continue
if dend is not None and enddate > dend.strftime(
'%Y-%m-%d'):
continue
self.mosaics[tile.id]['periods'][periodkey] = {}
self.mosaics[
tile.id]['periods'][periodkey]['start'] = startdate
self.mosaics[
tile.id]['periods'][periodkey]['end'] = enddate
self.mosaics[
tile.id]['periods'][periodkey]['cols'] = number_cols
self.mosaics[
tile.id]['periods'][periodkey]['rows'] = number_rows
self.mosaics[tile.id]['periods'][periodkey][
'dirname'] = '{}/{}/{}-{}/'.format(
self.datacube.id, tile.id, startdate, enddate)
def create(cls, params: dict):
"""Create and persist datacube on database."""
params['composite_function_list'] = ['IDENTITY', 'STK', 'MED']
# generate cubes metadata
cubes_db = Collection.query().filter().all()
cubes = []
cubes_serealized = []
for composite_function in params['composite_function_list']:
c_function_id = composite_function.upper()
cube_id = get_cube_id(params['datacube'], c_function_id)
raster_size_id = '{}-{}'.format(params['grs'],
int(params['resolution']))
temporal_composition = params[
'temporal_schema'] if c_function_id.upper(
) != 'IDENTITY' else 'Anull'
# add cube
if not list(filter(lambda x: x.id == cube_id, cubes)) and not list(
filter(lambda x: x.id == cube_id, cubes_db)):
cube = Collection(
id=cube_id,
temporal_composition_schema_id=temporal_composition,
raster_size_schema_id=raster_size_id,
composite_function_schema_id=c_function_id,
grs_schema_id=params['grs'],
description=params['description'],
radiometric_processing=None,
geometry_processing=None,
sensor=None,
is_cube=True,
oauth_scope=params.get('oauth_scope', None),
bands_quicklook=','.join(params['bands_quicklook']),
license=params.get('license'))
cubes.append(cube)
cubes_serealized.append(CollectionForm().dump(cube))
BaseModel.save_all(cubes)
bands = []
for cube in cubes:
fragments = get_cube_parts(cube.id)
# A IDENTITY data cube is composed by CollectionName and Resolution (LC8_30, S2_10)
is_identity = len(fragments) == 2
# save bands
for band in params['bands']:
# Skip creation of band CNC for IDENTITY data cube
# or band quality for composite data cube
if (band == 'cnc' and is_identity) or (band == 'quality'
and not is_identity):
continue
is_not_cloud = band != 'quality' and band != 'cnc'
band = band.strip()
bands.append(
Band(name=band,
collection_id=cube.id,
min=0 if is_not_cloud else 0,
max=10000 if is_not_cloud else 255,
fill=-9999 if is_not_cloud else 0,
scale=0.0001 if is_not_cloud else 1,
data_type='int16' if is_not_cloud else 'Uint16',
common_name=band,
resolution_x=params['resolution'],
resolution_y=params['resolution'],
resolution_unit='m',
description='',
mime_type='image/tiff'))
BaseModel.save_all(bands)
return cubes_serealized, 201