def test_refresh_materialized_view(self, session, Article, User, ArticleMV): article = Article(name='Some article', author=User(name='Some user')) session.add(article) session.commit() refresh_materialized_view(session, 'article_mv') materialized = session.query(ArticleMV).first() assert materialized.name == 'Some article' assert materialized.author_name == 'Some user'
def publish(blends): """Execute publish task and catalog datacube result. Args: activity - Datacube Activity Model """ logging.warning('Executing publish') cube = Collection.query().filter( Collection.id == blends[0]['datacube']).first() warped_datacube = blends[0]['warped_datacube'] tile_id = blends[0]['tile_id'] period = blends[0]['period'] cloudratio = blends[0]['cloudratio'] # Retrieve which bands to generate quick look quick_look_bands = cube.bands_quicklook.split(',') merges = dict() blend_files = dict() for blend_result in blends: blend_files[blend_result['band']] = blend_result['blends'] if blend_result.get('cloud_count_file'): blend_files['cnc'] = dict(MED=blend_result['cloud_count_file'], STK=blend_result['cloud_count_file']) for merge_date, definition in blend_result['scenes'].items(): merges.setdefault( merge_date, dict(dataset=definition['dataset'], cloudratio=definition['cloudratio'], ARDfiles=dict())) merges[merge_date]['ARDfiles'].update(definition['ARDfiles']) # Generate quick looks for cube scenes publish_datacube(cube, quick_look_bands, cube.id, tile_id, period, blend_files, cloudratio) # Generate quick looks of irregular cube wcube = Collection.query().filter(Collection.id == warped_datacube).first() for merge_date, definition in merges.items(): date = merge_date.replace(definition['dataset'], '') publish_merge(quick_look_bands, wcube, definition['dataset'], tile_id, period, date, definition) try: refresh_materialized_view(db.session, AssetMV.__table__) db.session.commit() logging.info('View refreshed.') except: db.session.rollback()
def refresh_assets_view(refresh_on_aws=True): """Update the Brazil Data Cube Assets View.""" if not Config.ENABLE_REFRESH_VIEW: logging.info('Skipping refresh view.') return refresh_materialized_view(db.session, AssetMV.__table__) commit(db) if refresh_on_aws: refresh_materialized_view(db_aws.session, AssetMV.__table__) commit(db) logging.info('View refreshed.')
def publish(self, activity): print('==> start PUBLISH') services = self.services activity['mystart'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') warped_cube = '_'.join(activity['datacube'].split('_')[0:2]) # Generate quicklooks for CUBES (MEDIAN, STACK ...) qlbands = activity['quicklook'].split(',') for function in ['MED', 'STK']: cube_id = get_cube_id(activity['datacube'], function) general_scene_id = '{}_{}_{}_{}'.format( cube_id, activity['tileid'], activity['start'], activity['end']) qlfiles = [] for band in qlbands: qlfiles.append(services.prefix + activity['blended'][band][function + 'file']) pngname = generateQLook(general_scene_id, qlfiles) dirname_ql = activity['dirname'].replace( '{}/'.format(warped_cube), '{}/'.format(cube_id)) if pngname is None: print('publish - Error generateQLook for {}'.format(general_scene_id)) return False s3pngname = os.path.join(dirname_ql, '{}_{}'.format(activity['start'], activity['end']), os.path.basename(pngname)) services.upload_file_S3(pngname, s3pngname, {'ACL': 'public-read'}) os.remove(pngname) # Generate quicklooks for all ARD scenes (WARPED) for datedataset in activity['scenes']: scene = activity['scenes'][datedataset] cube_id = get_cube_id(activity['datacube']) general_scene_id = '{}_{}_{}'.format( cube_id, activity['tileid'], str(scene['date'])[0:10]) qlfiles = [] for band in qlbands: filename = os.path.join(services.prefix + activity['dirname'], str(scene['date'])[0:10], scene['ARDfiles'][band]) qlfiles.append(filename) pngname = generateQLook(general_scene_id, qlfiles) if pngname is None: print('publish - Error generateQLook for {}'.format(general_scene_id)) return False s3pngname = os.path.join(activity['dirname'], str(scene['date'])[0:10], os.path.basename(pngname)) services.upload_file_S3(pngname, s3pngname, {'ACL': 'public-read'}) os.remove(pngname) # register collection_items and assets in DB (MEDIAN, STACK ...) for function in ['MED', 'STK']: cube_id = '{}_{}'.format(activity['datacube'], function) cube = Collection.query().filter( Collection.id == cube_id ).first() if not cube: print('cube {} not found!'.format(cube_id)) continue general_scene_id = '{}_{}_{}_{}'.format( cube_id, activity['tileid'], activity['start'], activity['end']) # delete collection_items and assets if exists assets = Asset.query().filter( Asset.collection_item_id == general_scene_id ).all() for asset in assets: db.session().delete(asset) db.session().commit() coll_item = CollectionItem.query().filter( CollectionItem.id == general_scene_id ).first() if coll_item: db.session().delete(coll_item) db.session().commit() # insert 'collection_item' range_date = '{}_{}'.format(activity['start'], activity['end']) png_name = '{}.png'.format(general_scene_id) dirname_ql = activity['dirname'].replace( '{}/'.format(warped_cube), '{}/'.format(cube_id)) s3_pngname = os.path.join(dirname_ql, range_date, png_name) CollectionItem( id=general_scene_id, collection_id=cube_id, grs_schema_id=cube.grs_schema_id, tile_id=activity['tileid'], item_date=activity['start'], composite_start=activity['start'], composite_end=activity['end'], quicklook='{}/{}'.format(BUCKET_NAME, s3_pngname), cloud_cover=activity['cloudratio'], scene_type=function, compressed_file=None ).save() # insert 'assets' bands_by_cube = Band.query().filter( Band.collection_id == cube_id ).all() for band in activity['bands']: if band == 'quality': continue band_id = list(filter(lambda b: str(b.common_name) == band, bands_by_cube)) if not band_id: print('band {} not found!'.format(band)) continue Asset( collection_id=cube_id, band_id=band_id[0].id, grs_schema_id=cube.grs_schema_id, tile_id=activity['tileid'], collection_item_id=general_scene_id, url='{}/{}'.format(BUCKET_NAME, activity['blended'][band][function + 'file']), source=None, raster_size_x=activity['raster_size_x'], raster_size_y=activity['raster_size_y'], raster_size_t=1, chunk_size_x=activity['chunk_size_x'], chunk_size_y=activity['chunk_size_y'], chunk_size_t=1 ).save() # Register all ARD scenes - WARPED Collection for datedataset in activity['scenes']: scene = activity['scenes'][datedataset] cube_id = get_cube_id(activity['datacube']) cube = Collection.query().filter( Collection.id == cube_id ).first() if not cube: print('cube {} not found!'.format(cube_id)) continue general_scene_id = '{}_{}_{}'.format( cube_id, activity['tileid'], str(scene['date'])[0:10]) # delete 'assets' and 'collection_items' if exists assets = Asset.query().filter( Asset.collection_item_id == general_scene_id ).all() for asset in assets: db.session().delete(asset) db.session().commit() coll_item = CollectionItem.query().filter( CollectionItem.id == general_scene_id ).first() if coll_item: db.session().delete(coll_item) db.session().commit() # insert 'collection_item' pngname = '{}.png'.format(general_scene_id) s3pngname = os.path.join(activity['dirname'], str(scene['date'])[0:10], pngname) CollectionItem( id=general_scene_id, collection_id=cube_id, grs_schema_id=cube.grs_schema_id, tile_id=activity['tileid'], item_date=scene['date'], composite_start=scene['date'], composite_end=scene['date'], quicklook='{}/{}'.format(BUCKET_NAME, s3pngname), cloud_cover=int(scene['cloudratio']), scene_type='WARPED', compressed_file=None ).save() # insert 'assets' bands_by_cube = Band.query().filter( Band.collection_id == cube_id ).all() for band in activity['bands']: if band not in scene['ARDfiles']: print('publish - problem - band {} not in scene[files]'.format(band)) continue band_id = list(filter(lambda b: str(b.common_name) == band, bands_by_cube)) if not band_id: print('band {} not found!'.format(band)) continue raster_size_x = scene['raster_size_x'] if scene.get('raster_size_x') else activity.get('raster_size_x') raster_size_y = scene['raster_size_y'] if scene.get('raster_size_y') else activity.get('raster_size_y') block_size = scene['block_size'] if scene.get('block_size') else activity.get('block_size') Asset( collection_id=cube_id, band_id=band_id[0].id, grs_schema_id=cube.grs_schema_id, tile_id=activity['tileid'], collection_item_id=general_scene_id, url='{}/{}'.format(BUCKET_NAME, os.path.join(activity['dirname'], str(scene['date'])[0:10], scene['ARDfiles'][band])), source=None, raster_size_x=raster_size_x, raster_size_y=raster_size_y, raster_size_t=1, chunk_size_x=block_size, chunk_size_y=block_size, chunk_size_t=1 ).save() # Update status and end time in DynamoDB activity['myend'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') activity['mystatus'] = 'DONE' services.put_item_kinesis(activity) refresh_materialized_view(db.session, AssetMV.__table__) return True
[ Article.id, Article.name, User.id.label('author_id'), User.name.label('author_name') ], from_obj=(Article.__table__.join( User, Article.author_id == User.id))), metadata=Base.metadata) Base.metadata.drop_all(engine) Base.metadata.create_all(engine) # Test session = Session() article = Article(name='Some article', author=User(name='Some user')) session.add(article) session.commit() refresh_materialized_view(session, 'article_mv') materialized = session.query(ArticleMV).first() assert materialized.name == 'Some article' assert materialized.author_name == 'Some user' article = Article(name='Some article', author=User(name='Some user')) session.add(article) session.commit() row = session.query(ArticleView).first() assert row.name == 'Some article' assert row.author_name == 'Some user'
def refresh_otu_sample_otu_mv(session, mv): refresh_materialized_view(session, mv)
def refresh_lastest_man_day_id(): """刷新最新工数ID的View :return: """ refresh_materialized_view(db.session, 'lastest_man_day_id', concurrently=True)