def create_ingest_definition_op(*args, **kwargs):
"""Create ingest definition and upload to S3"""
logger.info('Beginning to create ingest definition...')
xcom_client = kwargs['task_instance']
conf = kwargs['dag_run'].conf
scene_dict = conf.get('scene')
xcom_client.xcom_push(key='ingest_scene_id', value=scene_dict['id'])
scene = Scene.from_id(scene_dict['id'])
if scene.ingestStatus != IngestStatus.TOBEINGESTED:
raise Exception('Scene is no longer waiting to be ingested, error error')
if scene.datasource != datasource_id:
raise Exception('Unable to import scene %s, only able to import Landsat 8 scenes', scene.id)
scene.ingestStatus = IngestStatus.INGESTING
logger.info('Updating scene status to ingesting')
scene.update()
logger.info('Successfully updated scene status')
logger.info('Creating ingest definition')
ingest_definition = create_landsat8_ingest(scene)
ingest_definition.put_in_s3()
logger.info('Successfully created and pushed ingest definition')
# Store values for later tasks
xcom_client.xcom_push(key='ingest_def_uri', value=ingest_definition.s3_uri)
xcom_client.xcom_push(key='ingest_def_id', value=ingest_definition.id)
def set_ingest_status_failure_op(*args, **kwargs):
"""Set ingest status on failure"""
xcom_client = kwargs['task_instance']
scene_id = xcom_client.xcom_pull(key='ingest_scene_id', task_ids=None)
logger.info("Setting scene (%s) ingested status to failed", scene_id)
scene = Scene.from_id(scene_id)
scene.ingestStatus = IngestStatus.FAILED
scene.update()
logger.info("Finished setting scene (%s) ingest status (%s)", scene_id, IngestStatus.FAILED)
def set_ingest_status_success_op(*args, **kwargs):
"""Set scene ingest status on success"""
xcom_client = kwargs['task_instance']
scene_id = xcom_client.xcom_pull(key='ingest_scene_id', task_ids=None)
logger.info("Setting scene (%s) ingested status to success", scene_id)
scene = Scene.from_id(scene_id)
scene.ingestStatus = IngestStatus.INGESTED
layer_s3_bucket = os.getenv('TILE_SERVER_BUCKET')
s3_output_location = 's3://{}/layers'.format(layer_s3_bucket)
scene.ingestLocation = s3_output_location
scene.update()
logger.info("Finished setting scene (%s) ingest status (%s)", scene_id, IngestStatus.INGESTED)