def build_pipeline(name, transitions, state=100):
assert transitions[-1] is None
ppp = PostProcessingPipeline(name)
initial = True
for transition in transitions:
# State counter
state_start = 'S{0}'.format(str(state).zfill(4))
state += 100
state_end = 'S{0}'.format(str(state).zfill(4))
if not transition:
# End pipeline
s = State(name=state_start, transition=None)
else:
# Set transition
t = Transition(name=transition, destination=state_end, get_args=get_args)
# Set state transition
s = State(name=state_start, transition=t, initial=initial)
# Set initial to False after first transition
initial = False
# Add transition
ppp.add(s)
return ppp
# @copyright Copyright (c)2009 Centre National de la Recherche Scientifique CNRS. All Rights Reserved��
# @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE)
##################################
"""
Contains whole variable pipeline definition for tCDF.
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'CDF_INT'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='spatial_interpolation', destination='S2200', get_args=get_args)
#t2 = Transition(name='qc_qa', destination='S2300', get_args=get_args)
t2 = Transition(name='copy', destination='S2300', get_args=get_args)
t3 = Transition(name='latest', destination='S2400', get_args=get_args)
s1 = State(name='S2100', transition=t1, initial=True)
s2 = State(name='S2200', transition=t2)
s3 = State(name='S2300', transition=t3)
s4 = State(name='S2400', transition=None)
ppp.add(s1, s2, s3, s4)
##################################
"""
Contains variable only pipeline definition for tCDF.
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'CDF_VARIABLE'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='spatial_interpolation', destination='S2200', get_args=get_args)
t2 = Transition(name='correction_cdft', destination='S2300', get_args=get_args)
t3 = Transition(name='time_axis_normalization', destination='S2400', get_args=get_args)
t4 = Transition(name='standardization', destination='S2500', get_args=get_args)
t5 = Transition(name='qc', destination='S2600', get_args=get_args)
s1 = State(name='S2100', transition=t1, initial=True)
s2 = State(name='S2200', transition=t2)
s3 = State(name='S2300', transition=t2)
s4 = State(name='S2400', transition=t3)
s5 = State(name='S2500', transition=t4)
s6 = State(name='S2600', transition=None)
##################################
# @program synda
# @description climate models data transfer program
# @copyright Copyright (c)2009 Centre National de la Recherche Scientifique CNRS. All Rights Reserved��
# @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE)
##################################
"""
Contains dataset only pipeline definition for tCDF
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'CDF_DATASET'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='latest', destination='S2600', get_args=get_args)
t2 = Transition(name='latest', destination='S2700', get_args=get_args)
s1 = State(name='S2500', transition=t1, initial=True)
s2 = State(name='S2600', transition=t2)
s3 = State(name='S2700', transition=None)
ppp.add(s1, s2, s3)
# -*- coding: ISO-8859-1 -*-
##################################
# @program synda
# @description climate models data transfer program
# @copyright Copyright (c)2009 Centre National de la Recherche Scientifique CNRS. All Rights Reserved��
# @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE)
##################################
"""
Contains whole variable pipeline definition for tCDF.
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'CDF_COR_DATASET'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='latest', destination='S3600', get_args=get_args)
s1 = State(name='S3500', transition=t1, initial=True)
s2 = State(name='S3600', transition=None)
ppp.add(s1, s2)
src_path=sppipelineutils.build_mirror_path(kw)
dest_path=sppipelineutils.build_process_path(kw)
return {'project':kw.project,'src_variable_path':src_path,'dest_variable_path':dest_path}
def f3(kw):
set_variable_path_type(kw)
path=sppipelineutils.build_user_path('main',kw)
return {'project':kw.project,'variable_path':path}
def f4(kw):
set_variable_path_type(kw)
src_path=sppipelineutils.build_process_path(kw)
dest_path=sppipelineutils.build_user_path('main',kw)
return {'project':kw.project,'src_variable_path':src_path,'dest_variable_path':dest_path}
def f5(kw):
set_variable_path_type(kw)
path=sppipelineutils.build_user_path('main',kw)
return {'project':kw.project,'dataset_path':path}
name='P001'
ppp=PostProcessingPipeline(name)
t0=Transition(name='is_duplicate',destination={0:'S0300',1:'S0100'})
t1=Transition(name='time_axis_normalization',destination='S0200',get_args=f1)
t2=Transition(name='latest',destination='S0300',get_args=f5)
s0=State(name='S0090',transition=t0,initial=True)
s1=State(name='S0100',transition=t1)
s2=State(name='S0200',transition=t2)
s3=State(name='S0300',transition=None)
ppp.add(s0,s1,s2,s3)
"""Contains 'variable complete' pipeline definition."""
import sppipelineutils
from sppostprocessingutils import PostProcessingPipeline,State,Transition
def get_pipeline():
return ppp
def set_variable_path_type(kw):
assert kw.variable!='' # additional check just in case
kw.path_type='variable'
# init.
name='IPSL_001'
ppp=PostProcessingPipeline(name)
def f1(kw):
set_variable_path_type(kw)
path=sppipelineutils.build_process_path(kw)
return {'project':kw.project,'variable_path':path}
def f2(kw):
set_variable_path_type(kw)
src_path=sppipelineutils.build_mirror_path(kw)
dest_path=sppipelineutils.build_process_path(kw)
return {'project':kw.project,'src_variable_path':src_path,'dest_variable_path':dest_path}
def f3(kw):
set_variable_path_type(kw)
path=sppipelineutils.build_user_path('main',kw)
return {'project':kw.project,'variable_path':path}
def f4(kw):
# @description climate models data transfer program
# @copyright Copyright “(c)2009 Centre National de la Recherche Scientifique CNRS.
# All Rights Reserved”
# @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE)
##################################
"""Contains P001 pipeline definition."""
import sppipelineutils
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
# init.
name = 'P001'
ppp = PostProcessingPipeline(name)
t0 = Transition(name='is_duplicate', destination={0: 'S0300', 1: 'S0100'})
t1 = Transition(name='time_axis_normalization', destination='S0200')
t2 = Transition(name='latest', destination='S0300')
s0 = State(name='S0090', transition=t0, initial=True)
s1 = State(name='S0100', transition=t1)
s2 = State(name='S0200', transition=t2)
s3 = State(name='S0300', transition=None)
ppp.add(s0, s1, s2, s3)
# -*- coding: ISO-8859-1 -*-
##################################
# @program synda
# @description climate models data transfer program
# @copyright Copyright (c)2009 Centre National de la Recherche Scientifique CNRS. All Rights Reserved��
# @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE)
##################################
"""
Contains whole variable pipeline definition for tCDF.
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'CDF_INT_DATASET'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='latest', destination='S2400', get_args=get_args)
s1 = State(name='S2300', transition=t1, initial=True)
s2 = State(name='S2400', transition=None)
ppp.add(s1, s2)
##################################
"""
Contains variable only pipeline definition for tCDF.
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'CDF_VARIABLE'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='spatial_interpolation',
destination='S2200',
get_args=get_args)
t2 = Transition(name='correction_cdft', destination='S2300', get_args=get_args)
t3 = Transition(name='time_axis_normalization',
destination='S2400',
get_args=get_args)
t4 = Transition(name='standardization', destination='S2500', get_args=get_args)
t5 = Transition(name='qc', destination='S2600', get_args=get_args)
s1 = State(name='S2100', transition=t1, initial=True)
s2 = State(name='S2200', transition=t2)
s3 = State(name='S2300', transition=t2)
s4 = State(name='S2400', transition=t3)
##################################
"""
Contains whole variable pipeline definition for tCDF.
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'CDF_COR_VARIABLE'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='correction_cdft', destination='S3200', get_args=get_args)
t2 = Transition(name='standardization', destination='S3300', get_args=get_args)
t3 = Transition(name='time_axis_normalization', destination='S3400', get_args=get_args)
#t4 = Transition(name='qc_qa', destination='S3500', get_args=get_args)
t4 = Transition(name='copy', destination='S3500', get_args=get_args)
s1 = State(name='S3100', transition=t1, initial=True)
s2 = State(name='S3200', transition=t2)
s3 = State(name='S3300', transition=t3)
s4 = State(name='S3400', transition=t4)
s5 = State(name='S3500', transition=None)
ppp.add(s1, s2, s3, s4, s5)
##################################
"""
Contains whole variable default pipeline definition.
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'IPSL'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='suppression_variable', destination='S0200', get_args=get_args)
t2 = Transition(name='coalesce', destination='S0300', get_args=get_args)
t3 = Transition(name='overlap', destination='S0400', get_args=get_args)
t4 = Transition(name='time_axis_normalization', destination='S0500', get_args=get_args)
t5 = Transition(name='suppression_variable', destination='S0600', get_args=get_args)
t6 = Transition(name='copy', destination='S0700', get_args=get_args)
t7 = Transition(name='cdscan', destination='S0800', get_args=get_args)
t8 = Transition(name='latest', destination='S0900', get_args=get_args)
t9 = Transition(name='mapfile', destination='S1000', get_args=get_args)
t10 = Transition(name='publication', destination='S1100', get_args=get_args)
s1 = State(name='S0100', transition=t1, initial=True)
s2 = State(name='S0200', transition=t2)
s3 = State(name='S0300', transition=t3)
##################################
"""
Contains whole variable default pipeline definition.
"""
from sppipelineutils import get_args
from sppostprocessingutils import PostProcessingPipeline, State, Transition
def get_pipeline():
return ppp
name = 'IPSL'
ppp = PostProcessingPipeline(name)
t1 = Transition(name='suppression_variable',
destination='S0200',
get_args=get_args)
t2 = Transition(name='coalesce', destination='S0300', get_args=get_args)
t3 = Transition(name='overlap', destination='S0400', get_args=get_args)
t4 = Transition(name='time_axis_normalization',
destination='S0500',
get_args=get_args)
t5 = Transition(name='suppression_variable',
destination='S0600',
get_args=get_args)
t6 = Transition(name='copy', destination='S0700', get_args=get_args)
t7 = Transition(name='cdscan', destination='S0800', get_args=get_args)
t8 = Transition(name='latest', destination='S0900', get_args=get_args)
import copy
from sppostprocessingutils import PostProcessingPipeline,State,Transition
def get_pipeline(name):
return copy.deepcopy(pipelines[name])
# init.
pipelines={}
# 'variable complete' pipeline definition
name='CMIP5_001'
ppp=PostProcessingPipeline(name)
ppp.project='CMIP5'
t1=Transition(name='suppression_variable',destination='S0200',workdir='process')
t2=Transition(name='coalesce',destination='S0300')
t3=Transition(name='overlap',destination='S0400',workdir='process')
t4=Transition(name='time_axis_normalization',destination='S0500',workdir='process')
t5=Transition(name='cdscan',destination='S0600',workdir='process')
t6=Transition(name='suppression_variable',destination='S0700',workdir='merge')
t7=Transition(name='copy',destination='S0800',workdir='process')
s1=State(name='S0100',transition=t1,initial=True)
s2=State(name='S0200',transition=t2)
s3=State(name='S0300',transition=t3)
s4=State(name='S0400',transition=t4)
s5=State(name='S0500',transition=t5)