def reduceByTile(upstream, algorithm, version, queue, dag, task_id, delete_partial_results=False, to_tiff=False, params={}, **kwargs):
reducers = {}
trans = str.maketrans({"(": None, ")": None, " ": None, ",": "_"})
for prev in upstream:
key = "{}_{}".format(prev.lat, prev.lon)
if key not in reducers:
reducers[key] = CDColReduceOperator(
task_id="{}_{}".format(task_id, key.translate(trans)),
algorithm=algorithm,
version=version,
params=params,
lat=prev.lat,
lon=prev.lon,
queue=queue,
to_tiff=to_tiff,
dag=dag, **kwargs)
prev >> reducers[key]
if delete_partial_results:
delete = PythonOperator(task_id="del_"+prev.task_id,
provide_context=True,
python_callable=other_utils.delete_partial_result,
queue='util',
op_kwargs={'algorithm': prev.algorithm, 'version':prev.version, 'execID': prev.execID, 'task_id':prev.task_id},
dag=dag)
delete << reducers[key]
return reducers.values()
def OneReduce(upstream, algorithm, version, queue, dag, task_id, delete_partial_results=False,params={}, to_tiff=False, trigger_rule=None, **kwargs):
reduce = CDColReduceOperator(
task_id="{}_{}_{}".format(task_id, "all", "all"),
algorithm=algorithm,
version=version,
params=params,
queue=queue,
trigger_rule=trigger_rule,
to_tiff=to_tiff,
dag=dag, **kwargs)
upstream >> reduce
if delete_partial_results:
tasks = []
for prev in upstream:
delete = PythonOperator(task_id="del_" + prev.task_id,
provide_context=True,
python_callable=other_utils.delete_partial_result,
queue='util',
op_kwargs={'algorithm': prev.algorithm, 'version': prev.version,
'execID': prev.execID, 'task_id': prev.task_id},
dag=dag)
tasks.append(delete)
tasks << reduce
return [reduce]
queue='airflow_small',
op_kwargs={
'algorithms': {
'wofs-wf': "1.0",
'joiner-reduce-wofs': "1.0",
},
'execID': args['execID']
},
dag=dag)
workflow = time_series
if _params['mosaic']:
mosaic = CDColReduceOperator(task_id="mosaic",
algorithm="joiner",
version="1.0",
queue=_queues['joiner'],
trigger_rule=TriggerRule.NONE_FAILED,
dag=dag)
# if _params['normalized']:
# normalization = CDColFromFileOperator(task_id="normalization", algorithm="normalization-wf", version="1.0", queue=_queues['normalization'])
workflow = [workflow >> mosaic]
if _params['generate-geotiff']:
workflow = dag_utils.BashMap(workflow,
task_id="generate-geotiff",
algorithm="generate-geotiff",
version="1.0",
queue=_queues['joiner'],
dag=dag)
workflow >> delete_partial_results
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas",
queue=_steps['medianas']['queue'],
dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
mosaico = dag_utils.OneReduce(
medianas,
task_id="mosaico_medianas",
algorithm=_steps['mosaico']['algorithm'],
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
delete_partial_results=_steps['mosaico']['del_prev_result'],
trigger_rule=TriggerRule.NONE_FAILED,
dag=dag)
normalizacion = CDColReduceOperator(
algorithm=_steps['normalizacion']['algorithm'],
version=_steps['normalizacion']['version'],
queue=_steps['normalizacion']['queue'],
params=_steps['normalizacion']['params'],
delete_partial_results=_steps['normalizacion']['del_prev_result'],
dag=dag,
task_id="normalizacion",
to_tiff=True)
mosaico >> normalizacion
mascara_ls7_mosaic >> normalizacion
mosaico,
algorithm=_steps['entrenamiento']['algorithm'],
version=_steps['entrenamiento']['version'],
task_id="entrenamiento",
queue=_steps['entrenamiento']['queue'],
dag=dag,
delete_partial_results=_steps['entrenamiento']['del_prev_result'],
params=_steps['entrenamiento']['params']
)
clasificador = CDColReduceOperator(
task_id="clasificador_generico",
algorithm=_steps['clasificador']['algorithm'],
version=_steps['clasificador']['version'],
queue=_steps['clasificador']['queue'],
dag=dag,
lat=_params['lat'],
lon=_params['lon'],
params=_steps['clasificador']['params'],
delete_partial_results=_steps['clasificador']['del_prev_result'],
)
mascara_fnf = CDColReduceOperator(algorithm=_steps['mascara_fnf']['algorithm'],
version=_steps['mascara_fnf']['version'],
queue=_steps['mascara_fnf']['queue'],
params=_steps['mascara_fnf']['params'],
delete_partial_results=_steps['mascara_fnf']['del_prev_result'],
dag=dag, task_id="clasificacion_final", to_tiff=True)
entrenamiento>>clasificador
)
full_query = dag_utils.reduceByTile(masked0 + masked1, algorithm="joiner-reduce", version="1.0", queue='airflow_small', dag=dag, taxprefix="joined", params={'bands': _params['bands']},)
else:
full_query = masked0
medians = dag_utils.IdentityMap(
full_query,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
taxprefix="medianas_",
queue='airflow_small',dag=dag,
params={
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid'],
})
if _params['mosaic']:
task_id = 'mosaic'
algorithm = 'joiner'
queue = 'airflow_small'
else:
task_id = 'print_context'
algorithm = 'test-reduce'
queue = 'airflow_small'
join = CDColReduceOperator(task_id=task_id,algorithm=algorithm,version='1.0', queue=queue, dag=dag)
map(lambda b: b >> join, medians)
version=_steps['mosaico']['version'],
queue=_steps['mosaico']['queue'],
delete_partial_results=_steps['mosaico']['del_prev_result'],
trigger_rule=TriggerRule.NONE_FAILED,
dag=dag)
workflow = mosaico
entrenamiento = dag_utils.IdentityMap(
workflow,
algorithm=_steps['entrenamiento']['algorithm'],
version=_steps['entrenamiento']['version'],
task_id="entrenamiento",
queue=_steps['entrenamiento']['queue'],
dag=dag,
delete_partial_results=False,
params=_steps['entrenamiento']['params'])
clasificador = CDColReduceOperator(
task_id="clasificador_generico",
algorithm=_steps['clasificador']['algorithm'],
version=_steps['clasificador']['version'],
queue=_steps['clasificador']['queue'],
dag=dag,
lat=_params['lat'],
lon=_params['lon'],
params=_steps['clasificador']['params'],
to_tiff=True)
entrenamiento >> clasificador
workflow >> clasificador
task_id="medianas_2013")
medianas_2014 = CDColFromFileOperator(
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
lat=(10, 11),
lon=(-75, -74),
product="LS7_ETM_LEDAPS",
time_ranges=("2014-01-01", "2014-12-31"),
params={
'bands': ["blue", "green", "red", "nir", "swir1", "swir2"],
'minValid': 1,
},
queue='airflow_small',
dag=dag,
task_id="medianas_2014")
pca = CDColReduceOperator(
algorithm="deteccion-cambios-pca-wf",
version="1.0",
queue='airflow_small',
params={
'bands': ["blue", "green", "red", "nir", "swir1", "swir2"],
},
dag=dag,
task_id="pca",
)
consulta_2013 >> medianas_2013 >> pca
consulta_2014 >> medianas_2014 >> pca
joins,
algorithm="compuesto-temporal-medianas-wf",
version="1.0",
taxprefix="medianas_",
dag=dag,
params={
'normalized': _params['normalized'],
'bands': _params['bands'],
'minValid': _params['minValid']
},
)
ndvi=dag_utils.IdentityMap(medians, algorithm="ndvi-wf", version="1.0", dag=dag, taxprefix="ndvi")
bosque=dag_utils.IdentityMap(
ndvi,
algorithm="bosque-no-bosque-wf",
version="1.0",
params={
'ndvi_threshold': _params['ndvi_threshold'],
'vegetation_rate': _params['vegetation_rate'],
'slice_size': _params['slice_size']
},
dag=dag, taxprefix="bosque",
)
mosaic=CDColReduceOperator(
task_id='print_context',
algorithm='joiner',
version='1.0',
dag=dag
)
map(lambda b: b>>mosaic,bosque)
dag = DAG(
dag_id='bosque-no-bosque-test-reduce', default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=15))
mediansComposites = queryMapByTile( lat=(2,4),
lon=(-69,-67),time_ranges=[("2013-01-01","2013-12-31")],
algorithm="compuesto-temporal-de-medianas",version="1.0",
params={
'normalized':True,
'bands':["blue","green","red","nir", "swir1","swir2"],
'minValid':1,
},
dag=dag, taxprefix="med")
ndvis = IdentityMap(mediansComposites, algorithm="ndvi-wf",version="1.0",dag=dag, taxprefix="ndvi")
bosque = IdentityMap(ndvis,algorithm="bosque-no-bosque-wf",version="1.0",
params={
'ndvi_threshold':0.7,
'vegetation_rate':0.3,
'slice_size':3
},
dag=dag, taxprefix="bosque")
reduce= CDColReduceOperator(
task_id='print_context',
algorithm='test-reduce',
version='1.0',
dag=dag)
map(lambda b: b>>reduce,bosque)
