def queryMapByTileByMonths(lat, lon, time_ranges, queue, dag, algorithm, version, params={}, months=12, to_tiff=False, task_id="med", alg_folder=common.ALGORITHMS_FOLDER,
**kwargs):
tasks = []
for LAT in range(*lat):
for LON in range(*lon):
start = datetime.strptime(time_ranges[0], '%Y-%m-%d')
end = datetime.strptime(time_ranges[1], '%Y-%m-%d')
while start <= end:
tasks.append(CDColQueryOperator(algorithm=algorithm,version=version,
lat=(LAT, LAT + 1),lon=(LON, LON + 1),
time_ranges=(start.strftime('%d-%m-%Y'), (
start + relativedelta(months=months - 1,
day=end.day)).strftime('%d-%m-%Y')),
params=params,
queue=queue,
to_tiff=to_tiff,
alg_folder=alg_folder,
dag=dag, task_id="{}{}{}_{}_{}".format(task_id, str(LAT), str(LON),
start.strftime('%d-%m-%Y'), (
start + relativedelta(
months=months - 1,
day=end.day)).strftime(
'%d-%m-%Y'), **kwargs)))
start += relativedelta(months=months)
return tasks
def queryMapByTile(lat,lon,time_ranges,dag, algorithm,version,params={},taxprefix="med"):
return [CDColQueryOperator(
algorithm="compuesto-temporal-de-medianas",version="1.0",
lat=(LAT,LAT+1),
lon=(LON,LON+1),
time_ranges=time_ranges,
params=params,
dag=dag, task_id="{}{}{}".format(taxprefix,str(LAT),str(LON))) for LAT in range(*lat) for LON in range(*lon)]
def queryMapByTile(lat, lon, time_ranges, queue, dag, algorithm, version, params={}, to_tiff=False, task_id="med", alg_folder=common.ALGORITHMS_FOLDER, **kwargs):
return [CDColQueryOperator(
algorithm=algorithm, version=version,
lat=(LAT, LAT + 1),
lon=(LON, LON + 1),
time_ranges=time_ranges,
params=params,
queue=queue,
to_tiff=to_tiff,
alg_folder=alg_folder,
dag=dag, task_id="{}{}{}".format(task_id, str(LAT), str(LON)), **kwargs) for LAT in range(*lat) for LON in
range(*lon)]
def queryMapByTileByYear(lat, lon, time_ranges, queue, dag, algorithm, version, params={}, to_tiff=False, task_id="med", alg_folder=common.ALGORITHMS_FOLDER, **kwargs):
logging.info("queryMapByTileByYear Latitude: {}, Longitude: {}, Time Range: {}".format(lat,lon,time_ranges))
lat_min, lat_max = lat
lon_min, lon_max = lon
# Time ranges is a tuple
time_min, time_max = time_ranges
time_min = datetime.strptime(time_min, '%Y-%m-%d')
time_max = datetime.strptime(time_max, '%Y-%m-%d')
tasks = []
# range exclude the last number, example range(1,5) = [1,2,3,4]
for latitude in range(lat_min,lat_max):
for longitude in range(lon_min,lon_max):
for time_year in range(time_min.year,time_max.year + 1):
time_range_min = "{}-{}-{}".format(time_year,time_min.strftime('%m'),time_min.strftime('%d'))
time_range_max = "{}-{}-{}".format(time_year,time_max.strftime('%m'),time_max.strftime('%d'))
logging.info("queryMapByTileByYear Time Range {} and {}".format(time_range_min,time_range_max))
# Task id can not be more than 250 charecters
task_ident = "{}{}{}_{}".format(task_id, latitude, longitude, time_range_min + "_" + time_range_max)
logging.info("queryMapByTileByYear Task id len: {}-{}".format( len(task_id),task_id ) )
task = CDColQueryOperator(
dag=dag,
task_id=task_ident,
queue=queue,
algorithm=algorithm,
version=version,
lat=(latitude, latitude + 1),
lon=(longitude, longitude + 1),
time_ranges=(time_range_min, time_range_max),
to_tiff=to_tiff,
alg_folder=alg_folder,
**kwargs
)
tasks.append(task)
return tasks
lon=_params['lon'],
time_ranges=_params['time_ranges'],
algorithm=_steps['mascara']['algorithm'],
version=_steps['mascara']['version'],
product=_params['products'][0],
params=_steps['mascara']['params'],
queue=_steps['mascara']['queue'],
dag=dag,
task_id="consulta_cubo_" + _params['products'][0])
mascara_ls7_mosaic = CDColQueryOperator(
lat=_params['lat'],
lon=_params['lon'],
time_ranges=_params['time_ranges'],
algorithm=_steps['consulta']['algorithm'],
version=_steps['consulta']['version'],
product=_params['products'][1],
params=_steps['consulta']['params'],
queue=_steps['consulta']['queue'],
dag=dag,
task_id="consulta_referencia_" + _params['products'][1])
medianas = dag_utils.IdentityMap(
mascara_ls8,
algorithm=_steps['medianas']['algorithm'],
version=_steps['medianas']['version'],
task_id="medianas",
queue=_steps['medianas']['queue'],
dag=dag,
delete_partial_results=_steps['medianas']['del_prev_result'],
params=_steps['medianas']['params'])
'time_ranges': ("2017-01-01", "2017-12-31"),
'bands': ["blue", "green", "red", "nir", "swir1", "swir2"],
'products': ["LS8_OLI_LASRC"],
}
args = {
'owner': 'mp.mancipe10',
'start_date': airflow.utils.dates.days_ago(2),
'execID': "mp.mancipe10-paso-1-consulta",
'product': _params['products'][0]
}
dag = DAG(dag_id='mp.mancipe10-paso-1-consulta',
default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=120))
consulta = CDColQueryOperator(algorithm="mascara-landsat",
version="1.0",
lat=_params['lat'],
lon=_params['lon'],
product=_params['products'][0],
time_ranges=_params['time_ranges'],
params={
'bands': _params['bands'],
},
queue='airflow_small',
dag=dag,
task_id="query_")
consulta
'execID': "mp.mancipe10_pca_paso_2_medianas",
'product': "LS7_ETM_LEDAPS"
}
dag = DAG(dag_id=args['execID'],
default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=120))
consulta_2013 = CDColQueryOperator(
algorithm="mascara-landsat",
version="1.0",
lat=(10, 11),
lon=(-75, -74),
product="LS7_ETM_LEDAPS",
time_ranges=("2013-01-01", "2013-12-31"),
params={
'bands': ["blue", "green", "red", "nir", "swir1", "swir2"],
},
queue='airflow_small',
dag=dag,
task_id="consulta_p_2013")
consulta_2014 = CDColQueryOperator(
algorithm="mascara-landsat",
version="1.0",
lat=(10, 11),
lon=(-75, -74),
product="LS7_ETM_LEDAPS",
time_ranges=("2014-01-01", "2014-12-31"),
params={
'bands': ["blue", "green", "red", "nir", "swir1", "swir2"],
'start_date': airflow.utils.dates.days_ago(2),
'execID': _params['execID'],
'product': _params['products'][0]
}
dag = DAG(dag_id=args["execID"],
default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=120))
consulta_periodo_1 = CDColQueryOperator(
lat=_params['lat'],
lon=_params['lon'],
time_ranges=_params['time_ranges'][0],
algorithm=_steps['consulta']['algorithm'],
version=_steps['consulta']['version'],
product=_params['products'][0],
params=_steps['consulta']['params'],
queue=_steps['consulta']['queue'],
dag=dag,
task_id="consulta_p1_" + _params['products'][0]['name'])
consulta_periodo_2 = CDColQueryOperator(
lat=_params['lat'],
lon=_params['lon'],
time_ranges=_params['time_ranges'][1],
algorithm=_steps['consulta']['algorithm'],
version=_steps['consulta']['version'],
product=_params['products'][0],
params=_steps['consulta']['params'],
queue=_steps['consulta']['queue'],
# coding=utf8
import airflow
from airflow.models import DAG
from airflow.operators import CDColQueryOperator, CDColFromFileOperator
from datetime import timedelta
from pprint import pprint
args = {
'owner': 'mp.mancipe10',
'start_date': airflow.utils.dates.days_ago(2),
'execID':"mp.mancipe10_wofs_paso_1_clasificacion_wofs",
'product':"LS7_ETM_LEDAPS"
}
dag = DAG(
dag_id=args['execID'], default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=120))
clasificacion = CDColQueryOperator(algorithm="wofs-wf",
version="1.0",
lat=(10,11),
lon=(-75,-74),
product="LS7_ETM_LEDAPS",
time_ranges=("2013-01-01", "2013-12-31"),
queue='airflow_small',dag=dag, task_id="wofs_clasificacion")
clasificacion
'del_prev_result':
_params['elimina_resultados_anteriores'],
}
}
args = {
'owner': _params['owner'],
'start_date': airflow.utils.dates.days_ago(2),
'execID': _params['execID'],
'product': _params['products'][0]
}
dag = DAG(dag_id=args["execID"],
default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=120))
kmeans = CDColQueryOperator(task_id="k_means",
time_ranges=_params['time_ranges'][0],
product=_params['products'][0],
algorithm=_steps['k_means']['algorithm'],
version=_steps['k_means']['version'],
queue=_steps['k_means']['queue'],
dag=dag,
lat=_params['lat'],
lon=_params['lon'],
params=_steps['k_means']['params'],
to_tiff=True)
kmeans
args = {
'owner': 'airflow',
'start_date': airflow.utils.dates.days_ago(2),
'execID': "wf1",
'product': "LS8_OLI_LASRC",
}
dag = DAG(dag_id='test_cdcol',
default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=15))
mediansComposite = CDColQueryOperator(
algorithm="compuesto-temporal-de-medianas",
version="1.0",
lat=(2, 3),
lon=(-69, -68),
time_ranges=[("2013-01-01", "2013-12-31")],
params={
'normalized': True,
'bands': ["blue", "green", "red", "nir", "swir1", "swir2"],
'minValid': 1,
},
dag=dag,
task_id="med1")
ndvi = CDColFromFileOperator(algorithm="ndvi-wf",
version="1.0",
dag=dag,
task_id="ndvi1")
mediansComposite >> ndvi
'start_date': airflow.utils.dates.days_ago(2),
'execID': "mp.mancipe10_wofs_paso_2_series_de_tiempo",
'product': "LS7_ETM_LEDAPS"
}
dag = DAG(dag_id=args['execID'],
default_args=args,
schedule_interval=None,
dagrun_timeout=timedelta(minutes=120))
clasificacion = CDColQueryOperator(
algorithm="wofs-wf",
version="1.0",
lat=(10, 11),
lon=(-75, -74),
product="LS7_ETM_LEDAPS",
time_ranges=("2013-01-01", "2013-12-31"),
params={
'bands': ["blue", "green", "red", "nir", "swir1", "swir2"],
},
queue='airflow_small',
dag=dag,
task_id="wofs_clasificacion")
series_de_tiempo = CDColFromFileOperator(algorithm="wofs-time-series-wf",
version="1.0",
task_id="wofs_series_de_tiempo",
queue='airflow_small',
dag=dag)
clasificacion >> series_de_tiempo
mascara_dem_mosaic = dag_utils.queryMapByTile(lat=_params['lat'], lon=_params['lon'],
time_ranges=('2013-01-01', '2013-12-31'),
algorithm=_steps['consulta']['algorithm'],
version=_steps['consulta']['version'],
product=_params['products'][0],
params=_steps['consulta']['params'],
delete_partial_results=_steps['consulta']['del_prev_result'],
queue=_steps['consulta']['queue'], dag=dag,
task_id="consulta_referencia_" + _params['products'][0]['name'])
mascara_fnf_mosaic = CDColQueryOperator(lat=_params['lat'], lon=_params['lon'],
time_ranges=_params['time_ranges'][1],
algorithm=_steps['consulta']['algorithm'],
version=_steps['consulta']['version'],
product=_params['products'][1],
params=_steps['consulta']['params'],
delete_partial_results=_steps['consulta']['del_prev_result'],
queue=_steps['consulta']['queue'],
dag=dag,
task_id="consulta_referencia_" + _params['products'][1]['name'])
reduccion = dag_utils.reduceByTile(
mascara_ls7_mosaic + mascara_dem_mosaic,
product=_params['products'][2],
algorithm=_steps['reduccion']['algorithm'],
version=_steps['reduccion']['version'],
queue=_steps['reduccion']['queue'],
dag=dag, task_id="joined",
delete_partial_results=_steps['reduccion']['del_prev_result'],
params=_steps['reduccion']['params'],
)
lat=_params['lat'],
lon=_params['lon']),
'params': {
'modelos': _params['modelos'] + args["execID"]
},
'del_prev_result':
_params['elimina_resultados_anteriores'],
}
}
mosaico = CDColQueryOperator(lat=_params['lat'],
lon=_params['lon'],
time_ranges=_params['time_ranges'][0],
algorithm=_steps['consulta']['algorithm'],
version=_steps['consulta']['version'],
product=_params['products'][0],
params=_steps['consulta']['params'],
queue=_steps['consulta']['queue'],
dag=dag,
task_id="consulta_" +
_params['products'][0]['name'])
entrenamiento = CDColQueryOperator(
algorithm=_steps['entrenamiento']['algorithm'],
version=_steps['entrenamiento']['version'],
task_id="entrenamiento",
queue=_steps['entrenamiento']['queue'],
dag=dag,
delete_partial_results=False,
params=_steps['entrenamiento']['params'])