def build_pipeline(name, transitions, state=100): assert transitions[-1] is None ppp = PostProcessingPipeline(name) initial = True for transition in transitions: # State counter state_start = 'S{0}'.format(str(state).zfill(4)) state += 100 state_end = 'S{0}'.format(str(state).zfill(4)) if not transition: # End pipeline s = State(name=state_start, transition=None) else: # Set transition t = Transition(name=transition, destination=state_end, get_args=get_args) # Set state transition s = State(name=state_start, transition=t, initial=initial) # Set initial to False after first transition initial = False # Add transition ppp.add(s) return ppp
# @copyright Copyright (c)2009 Centre National de la Recherche Scientifique CNRS. All Rights Reserved�� # @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE) ################################## """ Contains whole variable pipeline definition for tCDF. """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'CDF_INT' ppp = PostProcessingPipeline(name) t1 = Transition(name='spatial_interpolation', destination='S2200', get_args=get_args) #t2 = Transition(name='qc_qa', destination='S2300', get_args=get_args) t2 = Transition(name='copy', destination='S2300', get_args=get_args) t3 = Transition(name='latest', destination='S2400', get_args=get_args) s1 = State(name='S2100', transition=t1, initial=True) s2 = State(name='S2200', transition=t2) s3 = State(name='S2300', transition=t3) s4 = State(name='S2400', transition=None) ppp.add(s1, s2, s3, s4)
################################## """ Contains variable only pipeline definition for tCDF. """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'CDF_VARIABLE' ppp = PostProcessingPipeline(name) t1 = Transition(name='spatial_interpolation', destination='S2200', get_args=get_args) t2 = Transition(name='correction_cdft', destination='S2300', get_args=get_args) t3 = Transition(name='time_axis_normalization', destination='S2400', get_args=get_args) t4 = Transition(name='standardization', destination='S2500', get_args=get_args) t5 = Transition(name='qc', destination='S2600', get_args=get_args) s1 = State(name='S2100', transition=t1, initial=True) s2 = State(name='S2200', transition=t2) s3 = State(name='S2300', transition=t2) s4 = State(name='S2400', transition=t3) s5 = State(name='S2500', transition=t4) s6 = State(name='S2600', transition=None)
################################## # @program synda # @description climate models data transfer program # @copyright Copyright (c)2009 Centre National de la Recherche Scientifique CNRS. All Rights Reserved�� # @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE) ################################## """ Contains dataset only pipeline definition for tCDF """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'CDF_DATASET' ppp = PostProcessingPipeline(name) t1 = Transition(name='latest', destination='S2600', get_args=get_args) t2 = Transition(name='latest', destination='S2700', get_args=get_args) s1 = State(name='S2500', transition=t1, initial=True) s2 = State(name='S2600', transition=t2) s3 = State(name='S2700', transition=None) ppp.add(s1, s2, s3)
# -*- coding: ISO-8859-1 -*- ################################## # @program synda # @description climate models data transfer program # @copyright Copyright (c)2009 Centre National de la Recherche Scientifique CNRS. All Rights Reserved�� # @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE) ################################## """ Contains whole variable pipeline definition for tCDF. """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'CDF_COR_DATASET' ppp = PostProcessingPipeline(name) t1 = Transition(name='latest', destination='S3600', get_args=get_args) s1 = State(name='S3500', transition=t1, initial=True) s2 = State(name='S3600', transition=None) ppp.add(s1, s2)
src_path=sppipelineutils.build_mirror_path(kw) dest_path=sppipelineutils.build_process_path(kw) return {'project':kw.project,'src_variable_path':src_path,'dest_variable_path':dest_path} def f3(kw): set_variable_path_type(kw) path=sppipelineutils.build_user_path('main',kw) return {'project':kw.project,'variable_path':path} def f4(kw): set_variable_path_type(kw) src_path=sppipelineutils.build_process_path(kw) dest_path=sppipelineutils.build_user_path('main',kw) return {'project':kw.project,'src_variable_path':src_path,'dest_variable_path':dest_path} def f5(kw): set_variable_path_type(kw) path=sppipelineutils.build_user_path('main',kw) return {'project':kw.project,'dataset_path':path} name='P001' ppp=PostProcessingPipeline(name) t0=Transition(name='is_duplicate',destination={0:'S0300',1:'S0100'}) t1=Transition(name='time_axis_normalization',destination='S0200',get_args=f1) t2=Transition(name='latest',destination='S0300',get_args=f5) s0=State(name='S0090',transition=t0,initial=True) s1=State(name='S0100',transition=t1) s2=State(name='S0200',transition=t2) s3=State(name='S0300',transition=None) ppp.add(s0,s1,s2,s3)
"""Contains 'variable complete' pipeline definition.""" import sppipelineutils from sppostprocessingutils import PostProcessingPipeline,State,Transition def get_pipeline(): return ppp def set_variable_path_type(kw): assert kw.variable!='' # additional check just in case kw.path_type='variable' # init. name='IPSL_001' ppp=PostProcessingPipeline(name) def f1(kw): set_variable_path_type(kw) path=sppipelineutils.build_process_path(kw) return {'project':kw.project,'variable_path':path} def f2(kw): set_variable_path_type(kw) src_path=sppipelineutils.build_mirror_path(kw) dest_path=sppipelineutils.build_process_path(kw) return {'project':kw.project,'src_variable_path':src_path,'dest_variable_path':dest_path} def f3(kw): set_variable_path_type(kw) path=sppipelineutils.build_user_path('main',kw) return {'project':kw.project,'variable_path':path} def f4(kw):
# @description climate models data transfer program # @copyright Copyright “(c)2009 Centre National de la Recherche Scientifique CNRS. # All Rights Reserved” # @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE) ################################## """Contains P001 pipeline definition.""" import sppipelineutils from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp # init. name = 'P001' ppp = PostProcessingPipeline(name) t0 = Transition(name='is_duplicate', destination={0: 'S0300', 1: 'S0100'}) t1 = Transition(name='time_axis_normalization', destination='S0200') t2 = Transition(name='latest', destination='S0300') s0 = State(name='S0090', transition=t0, initial=True) s1 = State(name='S0100', transition=t1) s2 = State(name='S0200', transition=t2) s3 = State(name='S0300', transition=None) ppp.add(s0, s1, s2, s3)
# -*- coding: ISO-8859-1 -*- ################################## # @program synda # @description climate models data transfer program # @copyright Copyright (c)2009 Centre National de la Recherche Scientifique CNRS. All Rights Reserved�� # @license CeCILL (https://raw.githubusercontent.com/Prodiguer/synda/master/sdt/doc/LICENSE) ################################## """ Contains whole variable pipeline definition for tCDF. """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'CDF_INT_DATASET' ppp = PostProcessingPipeline(name) t1 = Transition(name='latest', destination='S2400', get_args=get_args) s1 = State(name='S2300', transition=t1, initial=True) s2 = State(name='S2400', transition=None) ppp.add(s1, s2)
################################## """ Contains variable only pipeline definition for tCDF. """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'CDF_VARIABLE' ppp = PostProcessingPipeline(name) t1 = Transition(name='spatial_interpolation', destination='S2200', get_args=get_args) t2 = Transition(name='correction_cdft', destination='S2300', get_args=get_args) t3 = Transition(name='time_axis_normalization', destination='S2400', get_args=get_args) t4 = Transition(name='standardization', destination='S2500', get_args=get_args) t5 = Transition(name='qc', destination='S2600', get_args=get_args) s1 = State(name='S2100', transition=t1, initial=True) s2 = State(name='S2200', transition=t2) s3 = State(name='S2300', transition=t2) s4 = State(name='S2400', transition=t3)
################################## """ Contains whole variable pipeline definition for tCDF. """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'CDF_COR_VARIABLE' ppp = PostProcessingPipeline(name) t1 = Transition(name='correction_cdft', destination='S3200', get_args=get_args) t2 = Transition(name='standardization', destination='S3300', get_args=get_args) t3 = Transition(name='time_axis_normalization', destination='S3400', get_args=get_args) #t4 = Transition(name='qc_qa', destination='S3500', get_args=get_args) t4 = Transition(name='copy', destination='S3500', get_args=get_args) s1 = State(name='S3100', transition=t1, initial=True) s2 = State(name='S3200', transition=t2) s3 = State(name='S3300', transition=t3) s4 = State(name='S3400', transition=t4) s5 = State(name='S3500', transition=None) ppp.add(s1, s2, s3, s4, s5)
################################## """ Contains whole variable default pipeline definition. """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'IPSL' ppp = PostProcessingPipeline(name) t1 = Transition(name='suppression_variable', destination='S0200', get_args=get_args) t2 = Transition(name='coalesce', destination='S0300', get_args=get_args) t3 = Transition(name='overlap', destination='S0400', get_args=get_args) t4 = Transition(name='time_axis_normalization', destination='S0500', get_args=get_args) t5 = Transition(name='suppression_variable', destination='S0600', get_args=get_args) t6 = Transition(name='copy', destination='S0700', get_args=get_args) t7 = Transition(name='cdscan', destination='S0800', get_args=get_args) t8 = Transition(name='latest', destination='S0900', get_args=get_args) t9 = Transition(name='mapfile', destination='S1000', get_args=get_args) t10 = Transition(name='publication', destination='S1100', get_args=get_args) s1 = State(name='S0100', transition=t1, initial=True) s2 = State(name='S0200', transition=t2) s3 = State(name='S0300', transition=t3)
################################## """ Contains whole variable default pipeline definition. """ from sppipelineutils import get_args from sppostprocessingutils import PostProcessingPipeline, State, Transition def get_pipeline(): return ppp name = 'IPSL' ppp = PostProcessingPipeline(name) t1 = Transition(name='suppression_variable', destination='S0200', get_args=get_args) t2 = Transition(name='coalesce', destination='S0300', get_args=get_args) t3 = Transition(name='overlap', destination='S0400', get_args=get_args) t4 = Transition(name='time_axis_normalization', destination='S0500', get_args=get_args) t5 = Transition(name='suppression_variable', destination='S0600', get_args=get_args) t6 = Transition(name='copy', destination='S0700', get_args=get_args) t7 = Transition(name='cdscan', destination='S0800', get_args=get_args) t8 = Transition(name='latest', destination='S0900', get_args=get_args)
import copy from sppostprocessingutils import PostProcessingPipeline,State,Transition def get_pipeline(name): return copy.deepcopy(pipelines[name]) # init. pipelines={} # 'variable complete' pipeline definition name='CMIP5_001' ppp=PostProcessingPipeline(name) ppp.project='CMIP5' t1=Transition(name='suppression_variable',destination='S0200',workdir='process') t2=Transition(name='coalesce',destination='S0300') t3=Transition(name='overlap',destination='S0400',workdir='process') t4=Transition(name='time_axis_normalization',destination='S0500',workdir='process') t5=Transition(name='cdscan',destination='S0600',workdir='process') t6=Transition(name='suppression_variable',destination='S0700',workdir='merge') t7=Transition(name='copy',destination='S0800',workdir='process') s1=State(name='S0100',transition=t1,initial=True) s2=State(name='S0200',transition=t2) s3=State(name='S0300',transition=t3) s4=State(name='S0400',transition=t4) s5=State(name='S0500',transition=t5)