product=_params['products'][0],
version=_steps['reduccion']['version'],
queue=_steps['reduccion']['queue'],
dag=dag,
task_id="joined",
delete_partial_results=_steps['reduccion']['del_prev_result'],
params=_steps['reduccion']['params'],
)
else:
reduccion = mascara_0
medianas = dag_utils.IdentityMap(
reduccion,
product=_params['products'][0],
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas",
queue=_steps['medianas']['queue'],
dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
workflow = medianas
if queue_utils.get_tiles(_params['lat'], _params['lon']) > 1:
mosaico = dag_utils.OneReduce(
workflow,
task_id="mosaic",
algorithm=_steps['mosaico']['algorithm'],
product=_params['products'][0],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
_params['products'][1]))
reducer = dag_utils.reduceByTile(consulta_ls7 + consulta_ls8,
algorithm="joiner-reduce",
version="1.0",
queue='airflow_medium',
dag=dag,
taxprefix="joined",
params={'bands': _params['bands']})
medianas = dag_utils.IdentityMap(reducer,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
taxprefix="medianas_",
queue='airflow_small',
dag=dag,
params={
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid'],
})
mosaico = dag_utils.OneReduce(medianas,
algorithm="joiner",
version="1.0",
queue='airflow_medium',
dag=dag,
taxprefix="mosaic")
kmeans = dag_utils.IdentityMap(mosaico,
algorithm="k-means-wf",
algorithm=_steps['ndvi']['algorithm'],
version=_steps['ndvi']['version'],
params=_steps['ndvi']['params'],
queue=_steps['ndvi']['queue'],
delete_partial_results=_steps['ndvi']['del_prev_result'],
dag=dag,
task_id="ndvi",
to_tiff=not (_params['genera_mosaico'] and
queue_utils.get_tiles(_params['lat'], _params['lon']) > 1))
bosque = dag_utils.IdentityMap(
ndvi,
algorithm=_steps['bosque']['algorithm'],
product=_params['products'][0],
version=_steps['bosque']['version'],
params=_steps['bosque']['params'],
queue=_steps['bosque']['queue'],
delete_partial_results=_steps['ndvi']['del_prev_result'],
dag=dag,
task_id="bosque",
to_tiff=not (_params['genera_mosaico'] and
queue_utils.get_tiles(_params['lat'], _params['lon']) > 1))
workflow = bosque
if _params['genera_mosaico'] and queue_utils.get_tiles(_params['lat'],
_params['lon']) > 1:
mosaico = dag_utils.OneReduce(
workflow,
task_id="mosaic",
algorithm=_steps['mosaico']['algorithm'],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
reduccion = dag_utils.reduceByTile(
wofs,
algorithm=_steps['reduccion']['algorithm'],
version=_steps['reduccion']['version'],
queue=_steps['reduccion']['queue'],
dag=dag,
task_id="joined",
delete_partial_results=_steps['reduccion']['del_prev_result'],
params=_steps['reduccion']['params'],
)
serie_tiempo = dag_utils.IdentityMap(
reduccion,
algorithm=_steps['serie_tiempo']['algorithm'],
version=_steps['serie_tiempo']['version'],
task_id="wofs_serie_tiempo",
queue=_steps['serie_tiempo']['queue'],
delete_partial_results=_steps['serie_tiempo']['del_prev_result'],
dag=dag,
to_tiff=not (_params['genera_mosaico'] and
queue_utils.get_tiles(_params['lat'], _params['lon']) > 1))
workflow = serie_tiempo
if _params['genera_mosaico'] and queue_utils.get_tiles(_params['lat'],
_params['lon']) > 1:
mosaico = dag_utils.OneReduce(
workflow,
task_id="mosaic",
algorithm=_steps['mosaico']['algorithm'],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
delete_partial_results=_steps['mosaico']['del_prev_result'],
product=_params['products'][0],
params={
'normalized':_params['normalized'],
'bands':_params['bands'],
'minValid': _params['minValid']
},queue=_queues['mascara-landsat'],dag=dag, task_id="consulta_analysis_{}_".format(_params['products'][0])
)
medians_baseline = dag_utils.IdentityMap(
consulta_baseline,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
task_id="medianas_baseline_",
queue=_queues['compuesto-temporal-medianas-wf'],
dag=dag,
params={
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid'],
})
medians_analysis = dag_utils.IdentityMap(
consulta_analysis,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
task_id="medianas_analysis_",
queue=_queues['compuesto-temporal-medianas-wf'],
dag=dag,
params={
'normalized': _params['normalized'],
reduccion = dag_utils.reduceByTile(
wofs,
algorithm=_steps['reduccion']['algorithm'],
version=_steps['reduccion']['version'],
queue=_steps['reduccion']['queue'],
dag=dag,
task_id="joined",
delete_partial_results=_steps['reduccion']['del_prev_result'],
params=_steps['reduccion']['params'],
)
serie_tiempo = dag_utils.IdentityMap(
reduccion,
algorithm=_steps['serie_tiempo']['algorithm'],
version=_steps['serie_tiempo']['version'],
task_id="wofs_serie_tiempo",
queue=_steps['serie_tiempo']['queue'],
delete_partial_results=_steps['serie_tiempo']['del_prev_result'],
dag=dag)
workflow = serie_tiempo
if _params['genera_mosaico']:
mosaico = dag_utils.OneReduce(
workflow,
task_id="mosaic",
algorithm=_steps['mosaico']['algorithm'],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
delete_partial_results=_steps['mosaico']['del_prev_result'],
trigger_rule=TriggerRule.NONE_FAILED,
dag=dag)
lat=_params['lat'],
lon=_params['lon'],
time_ranges=_params['time_ranges'],
algorithm=_steps['consulta']['algorithm'],
version=_steps['consulta']['version'],
product=_params['products'][1],
params=_steps['consulta']['params'],
queue=_steps['consulta']['queue'],
dag=dag,
task_id="consulta_referencia_" + _params['products'][1])
medianas = dag_utils.IdentityMap(
mascara_ls8,
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas",
queue=_steps['medianas']['queue'],
dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
mosaico = dag_utils.OneReduce(
medianas,
task_id="mosaico_medianas",
algorithm=_steps['mosaico']['algorithm'],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
delete_partial_results=_steps['mosaico']['del_prev_result'],
trigger_rule=TriggerRule.NONE_FAILED,
dag=dag)
time_ranges=_params['time_ranges'][0],
algorithm=_steps['mascara']['algorithm'],
version=_steps['mascara']['version'],
product=_params['products'][0],
params=_steps['mascara']['params'],
queue=_steps['mascara']['queue'],
dag=dag,
task_id="mascara" +
_params['products'][0]['name'])
greenest_pixel = dag_utils.IdentityMap(
mascara,
algorithm=_steps['greenest_pixel']['algorithm'],
product=_params['products'][0],
version=_steps['greenest_pixel']['version'],
task_id="greenest_pixel",
queue=_steps['greenest_pixel']['queue'],
dag=dag,
delete_partial_results=_steps['greenest_pixel']['del_prev_result'],
to_tiff=not (_params['genera_mosaico'] and
queue_utils.get_tiles(_params['lat'], _params['lon']) > 1))
if _params['genera_mosaico'] and queue_utils.get_tiles(_params['lat'],
_params['lon']) > 1:
mosaico = dag_utils.OneReduce(
greenest_pixel,
task_id="mosaico",
algorithm=_steps['mosaico']['algorithm'],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
delete_partial_results=_steps['mosaico']['del_prev_result'],
algorithm=_steps['reduccion']['algorithm'],
version=_steps['reduccion']['version'],
queue=_steps['reduccion']['queue'],
dag=dag,
task_id="joined",
delete_partial_results=_steps['reduccion']['del_prev_result'],
params=_steps['reduccion']['params'],
)
else:
reduccion = mascara_0
medianas = dag_utils.IdentityMap(
reduccion,
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas",
queue=_steps['medianas']['queue'],
dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
evi = dag_utils.IdentityMap(
medianas,
algorithm=_steps['evi']['algorithm'],
version=_steps['evi']['version'],
queue=_steps['evi']['queue'],
delete_partial_results=_steps['evi']['del_prev_result'],
dag=dag,
task_id="evi",
to_tiff=not _params['genera_mosaico'])
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid'],
},
dag=dag,
taxprefix="wofs_")
joins = dag_utils.reduceByTile(wofs_classification,
algorithm="joiner-reduce-wofs",
version="1.0",
dag=dag,
taxprefix="joined")
time_series = dag_utils.IdentityMap(joins,
algorithm="wofs-time-series-wf",
version="1.0",
taxprefix="wofs_time_series_",
dag=dag)
if _params['mosaic']:
task_id = 'mosaic'
algorithm = 'joiner'
else:
task_id = 'print_context'
algorithm = 'test-reduce'
join = CDColReduceOperator(task_id=task_id,
algorithm=algorithm,
version='1.0',
dag=dag)
'bands': _params['bands'],
'minValid': _params['minValid'],
},
dag=dag,
taxprefix="maskedLS7_")
joins = dag_utils.reduceByTile(maskedLS7 + maskedLS8,
algorithm="joiner-reduce",
version="1.0",
dag=dag,
taxprefix="joined")
medians = dag_utils.IdentityMap(joins,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
taxprefix="medianas_",
dag=dag,
params={
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid'],
})
ndvi = dag_utils.IdentityMap(medians,
algorithm="ndvi-wf",
version="1.0",
dag=dag,
taxprefix="ndvi")
mosaic = CDColReduceOperator(task_id='print_context',
algorithm='joiner',
version='1.0',
dag=dag)
product=_params['products'][1],
params={'bands': _params['bands']},
queue=_queues['mascara-landsat'], dag=dag, task_id="masked_{}"+_params['products'][1]
)
full_query = dag_utils.reduceByTile(masked0 + masked1, algorithm="joiner-reduce", version="1.0", queue=_queues['joiner-reduce'], dag=dag, task_id="joined" , params={'bands': _params['bands']})
else:
full_query = masked0
medians=dag_utils.IdentityMap(
full_query,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
task_id="medianas",
queue=_queues['compuesto-temporal-medianas-wf'],
dag=dag,
params={
'normalized':_params['normalized'],
'bands':_params['bands'],
'minValid': _params['minValid']
},
)
ndvi=dag_utils.IdentityMap(medians, algorithm="ndvi-wf", version="1.0", queue=_queues['ndvi-wf'], dag=dag, task_id="ndvi")
bosque=dag_utils.IdentityMap(
ndvi,
algorithm="bosque-no-bosque-wf",
version="1.0",
params={
'ndvi_threshold': _params['ndvi_threshold'],
'vegetation_rate':_params['vegetation_rate'],
'slice_size':_params['slice_size']
dag = DAG(
dag_id=args["execID"], default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=120))
mascara = dag_utils.queryMapByTile(lat=_params['lat'], lon=_params['lon'],
time_ranges=_params['time_ranges'][0],
algorithm=_steps['mascara']['algorithm'],
version=_steps['mascara']['version'],
product=_params['products'][0],
params=_steps['mascara']['params'],
queue=_steps['mascara']['queue'], dag=dag,
task_id="mascara" + _params['products'][0]['name'])
medianas = dag_utils.IdentityMap(mascara,algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
params=_steps['medianas']['params'],
queue=_steps['medianas']['queue'], dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
task_id="medianas")
greenest_pixel = dag_utils.IdentityMap(medianas, algorithm=_steps['greenest_pixel']['algorithm'],
version=_steps['greenest_pixel']['version'],
task_id="greenest_pixel",
queue=_steps['greenest_pixel']['queue'], dag=dag,
delete_partial_results=_steps['greenest_pixel']['del_prev_result'])
if queue_utils.get_tiles(_params['lat'], _params['lon']) > 1:
mosaico = dag_utils.OneReduce(greenest_pixel, task_id="mosaico",
algorithm=_steps['mosaico']['algorithm'],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
delete_partial_results=_steps['mosaico']['del_prev_result'],
product=data['Product'],
params={
'normalized': data['Normalized'],
'bands': LS7_JsonBands,
'minValid': data['Min_valid'],
},
dag=dag,
taxprefix="maskedLS7_")
if 'LS8' in data['Product']:
LS8_medians = dag_utils.IdentityMap(
maskedLS8,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
taxprefix="medianas_",
dag=dag,
params={
'normalized': data['Product'],
'bands': LS8_JsonBands,
'minValid': data['Min_valid'],
})
if 'LS7' in data['Product']:
LS7_medians = dag_utils.IdentityMap(
maskedLS7,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
taxprefix="medianas_",
dag=dag,
params={
'normalized': data['Product'],
product=_params['products'][0],
version=_steps['reduccion']['version'],
queue=_steps['reduccion']['queue'],
dag=dag,
task_id="joined",
delete_partial_results=_steps['reduccion']['del_prev_result'],
params=_steps['reduccion']['params'],
)
else:
reduccion = mascara_0
medianas = dag_utils.IdentityMap(
reduccion,
product=_params['products'][0],
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas",
queue=_steps['medianas']['queue'],
dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
ndvi = dag_utils.IdentityMap(
medianas,
algorithm=_steps['ndvi']['algorithm'],
version=_steps['ndvi']['version'],
queue=_steps['ndvi']['queue'],
delete_partial_results=_steps['ndvi']['del_prev_result'],
dag=dag,
task_id="ndvi",
to_tiff=not (_params['genera_mosaico'] and
queue_utils.get_tiles(_params['lat'], _params['lon']) > 1))
queue=_steps['mascara']['queue'], dag=dag,
task_id="mascara_p1_" + _params['products'][0])
mascara_periodo_2 = dag_utils.queryMapByTile(lat=_params['lat'], lon=_params['lon'],
time_ranges=_params['time_ranges'][1],
algorithm=_steps['mascara']['algorithm'],
version=_steps['mascara']['version'],
product=_params['products'][0],
params=_steps['mascara']['params'],
queue=_steps['mascara']['queue'], dag=dag,
task_id="mascara_p2_" + _params['products'][0])
medianas_periodo_1 = dag_utils.IdentityMap(
mascara_periodo_1,
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas_p1_",
queue=_steps['medianas']['queue'], dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
medianas_periodo_2 = dag_utils.IdentityMap(
mascara_periodo_2,
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas_p2_",
queue=_steps['medianas']['queue'], dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
medianas_periodo_1
medianas_periodo_2
full_query = dag_utils.reduceByTile(masked0 + masked1,
algorithm="joiner-reduce",
version="1.0",
queue=_queues['joiner-reduce'],
dag=dag,
task_id="joined",
params={'bands': _params['bands']})
else:
full_query = masked0
medians = dag_utils.IdentityMap(
full_query,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
task_id="medianas",
queue=_queues['compuesto-temporal-medianas-wf'],
dag=dag,
params={
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid'],
})
ndvi = dag_utils.IdentityMap(medians,
algorithm="ndvi-wf",
version="1.0",
queue=_queues['ndvi-wf'],
dag=dag,
task_id="ndvi")
delete_partial_results = PythonOperator(
task_id='delete_partial_results',
algorithm=_steps['reduccion']['algorithm'],
version=_steps['reduccion']['version'],
queue=_steps['reduccion']['queue'],
dag=dag,
task_id="joined",
delete_partial_results=_steps['reduccion']['del_prev_result'],
params=_steps['reduccion']['params'],
)
else:
reduccion = mascara_0
medianas = dag_utils.IdentityMap(
reduccion,
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas",
queue=_steps['medianas']['queue'],
dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
workflow = medianas
if queue_utils.get_tiles(_params['lat'], _params['lon']) > 1:
mosaico = dag_utils.OneReduce(
workflow,
task_id="mosaic",
algorithm=_steps['mosaico']['algorithm'],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
delete_partial_results=_steps['mosaico']['del_prev_result'],
trigger_rule=TriggerRule.NONE_FAILED,
queue=_steps['reduccion']['queue'],
product=_params['products'][2],
dag=dag, task_id="joined",
delete_partial_results=_steps['reduccion']['del_prev_result'],
params=_steps['reduccion']['params'],
)
else:
reduccion_lansat = mascara_0
medianas = dag_utils.IdentityMap(
reduccion_lansat,
product=_params['products'][2],
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas",
queue=_steps['medianas']['queue'],
dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params']
)
mascara_dem_mosaic = dag_utils.queryMapByTile(lat=_params['lat'], lon=_params['lon'],
time_ranges=_params['time_ranges'][0],
algorithm=_steps['consulta']['algorithm'],
version=_steps['consulta']['version'],
product=_params['products'][0],
params=_steps['consulta']['params'],
queue=_steps['consulta']['queue'], dag=dag,
task_id="consulta_referencia_" + _params['products'][0]['name'])
'bands': _params['bands'],
'minValid': _params['minValid']
},
queue='airflow_small',dag=dag , taxprefix="masked_{}_".format(_params['products'][1])
)
full_query = dag_utils.reduceByTile(masked0 + masked1, algorithm="joiner-reduce", version="1.0", queue='airflow_small', dag=dag, taxprefix="joined", params={'bands': _params['bands']},)
else:
full_query = masked0
medians = dag_utils.IdentityMap(
full_query,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
taxprefix="medianas_",
queue='airflow_small',dag=dag,
params={
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid'],
})
if _params['mosaic']:
task_id = 'mosaic'
algorithm = 'joiner'
queue = 'airflow_small'
else:
task_id = 'print_context'
algorithm = 'test-reduce'
product=_params['products'][0],
queue=_queues['wofs-wf'],
months=6,
dag=dag,
task_id="wofs")
reducer = dag_utils.reduceByTile(wofs_classification,
algorithm="joiner",
version="1.0",
queue=_queues['joiner-reduce-wofs'],
dag=dag,
task_id="joined")
time_series = dag_utils.IdentityMap(reducer,
algorithm="wofs-time-series-wf",
version="1.0",
task_id="wofs_time_series",
queue=_queues['wofs-time-series-wf'],
dag=dag)
delete_partial_results = PythonOperator(
task_id='delete_partial_results',
provide_context=True,
python_callable=other_utils.delete_partial_results,
queue='airflow_small',
op_kwargs={
'algorithms': {
'wofs-wf': "1.0",
'joiner-reduce-wofs': "1.0",
},
'execID': args['execID']
},
lon=_params['lon'],
time_ranges=_params['time_ranges'][1],
algorithm="mascara-landsat",
version="1.0",
product=_params['products'][0],
params={'bands': _params['bands']},
queue=_queues['mascara-landsat'],
dag=dag,
task_id="masked_p_2_" +
_params['products'][0])
medians1 = dag_utils.IdentityMap(
period1,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
task_id="medianas_p_1",
queue=_queues['compuesto-temporal-medianas-wf'],
dag=dag,
params={
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid'],
})
medians2 = dag_utils.IdentityMap(
period2,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
task_id="medianas_p_2",
queue=_queues['compuesto-temporal-medianas-wf'],
dag=dag,
params={
'normalized': _params['normalized'],