def __init__(self, *args, **kwargs): self.__class__ = type(str(self.__class__), (self.__class__, SimpleFunctionPE), {}) SimpleFunctionPE.__init__(self, *args, **kwargs) # name=self.name ProvenancePE.__init__(self, self.name, *args, **kwargs)
def testSplitMerge(): ''' Creates the split/merge graph with 4 nodes. ''' graph = WorkflowGraph() split = splitPE(3) mult1 = SimpleFunctionPE(mult) mult2 = SimpleFunctionPE(mult) mult3 = SimpleFunctionPE(mult) merge = mergePE(3) test = fwritePE() graph.connect(split, 'output0', mult1, 'input') graph.connect(split, 'output1', mult2, 'input') graph.connect(split, 'output2', mult3, 'input') graph.connect(mult1, 'output', merge, 'input0') graph.connect(mult2, 'output', merge, 'input1') graph.connect(mult3, 'output', merge, 'input2') graph.connect(merge, 'output', test, 'input') return graph
'multipart/mixed', 'metadata': download_report } return { '_d4p_prov': prov, '_d4p_data': [ os.environ['STAGED_DATA'] + "/" + data['mseed_path'], os.environ['STAGED_DATA'] + "/" + data['stationxml_path'] ] } waveformr = SimpleFunctionPE(waveform_reader) xmlr = SimpleFunctionPE(stationxml_reader) downloadPE = SimpleFunctionPE(download_data) processes = [ waveform_reader, (plot_stream, { "source": "waveform_reader", "output_dir": "./output-images", "tag": "observed-image" }) ] # processes.append((fn, params)) chain = create_iterative_chain(processes, FunctionPE_class=SimpleFunctionPE)
for x in self.indexmap1: # Writes the output and specify additional dependencies. # Specifically, it refer to the object read from the list used to produce # the cross product self.write('output', (val, self.indexmap1[x]), metadata={'res':str((val, self.indexmap1[x]))},dep=[str('index1'+str(x))]) # Instantiates the Workflow Components sc = Source() sc.name='PE_source' divf=Div() divf.name='PE_div' crossp = CrossProd() squaref=SimpleFunctionPE(square,{}) #Uncomment this line to associate this PE to the mycluster provenance-cluster squaref=SimpleFunctionPE(square,{'prov_cluster':'mycluster'}) #Initialise and compose the workflow graph graph = WorkflowGraph() graph.connect(sc,'output',squaref,'input') graph.connect(sc,'output',crossp,'input1') graph.connect(squaref,'output',crossp,'input2') graph.connect(crossp,'output',divf,'input') #Declare workflow inputs: input_data = {"PE_source": [{"input": [10]}]}
spec = fft(data, nfft) spec_ampl = sqrt(abs(multiply(spec, conjugate(spec)))) spec /= spec_ampl #Do we need to do some smoothing here? ret = real(ifft(spec, nfft)[:n]) st[trace].data = ret return st # Now it's time to create the graph for preprocessing the traces. from dispel4py.workflow_graph import WorkflowGraph streamProducer = SimpleFunctionPE(stream_producer) streamProducer.name = "streamProducer" sta = SimpleFunctionPE(readstats) preprocess_trace = create_iterative_chain([(decimate, { 'sps': 4 }), detrend, demean, (filter, { 'freqmin': 0.01, 'freqmax': 1., 'corners': 4, 'zerophase': False }), spectralwhitening]) graph = WorkflowGraph() graph.connect(streamProducer, 'output', preprocess_trace, 'input')
sc1 = Collector() sc1.name = 'collector' sc2 = Match() sc2.name = 'match' sc3 = Writer("+") sc3.name = 'writer' sc4 = Visualiser() sc4.name = 'vizu' read=SimpleFunctionPE(reader) comb=SimpleFunctionPE(combine,{"operator":"+"}) #processes=[readn,multn] #chain = create_iterative_chain(processes, FunctionPE_class=SimpleFunctionPE) #Initialise the graph graph = WorkflowGraph() #Common way of composing the graph graph.connect(sc1,'output_R',read,'input') graph.connect(read,'output', sc2,'input') graph.connect(sc2,'output_X',comb,'input') graph.connect(comb,'output',sc3,'var') graph.connect(sc1,'output_W',sc3,'file')
def testSimpleFunction(): def add(a, b): return a+b simp = SimpleFunctionPE(add, {'b': 2}) tools.eq_({'output': 3}, simp.process({'input': 1}))
for station in r["data"]: download_report.append({"provider": r["client"], "provider_url": URL_MAPPINGS[r["client"]], "station": "%s.%s" % (station.network, station.station)}) prov={'location':["file://"+socket.gethostname()+"/"+os.environ['STAGED_DATA']+"/"+data['mseed_path'],"file://"+socket.gethostname()+"/"+os.environ['STAGED_DATA']+"/"+data['stationxml_path']], 'format':'multipart/mixed', 'metadata':download_report} return {'_d4p_prov':prov,'_d4p_data':[os.environ['STAGED_DATA']+"/"+data['mseed_path'],os.environ['STAGED_DATA']+"/"+data['stationxml_path']]} waveformr = SimpleFunctionPE(waveform_reader) xmlr = SimpleFunctionPE(stationxml_reader) downloadPE = SimpleFunctionPE(download_data) processes=[waveform_reader,(plot_stream,{"source":"waveform_reader","output_dir": "./output-images","tag": "observed-image"})] #processes.append((fn, params)) chain = create_iterative_chain(processes, FunctionPE_class=SimpleFunctionPE) watcher = WatchDirectory(0) watcher_xml = WatchDirectory(1) downloadPE.name = "downloadPE" graph = WorkflowGraph() graph.add(downloadPE) graph.connect(downloadPE, 'output', watcher, "input")
# the output stream and the trace statistics (metadata). return {'output': st, 'output_stats': st[0].stats} # This fuctions prints the name of an station from the stats. def name_station(stats): station_name = stats['station'] print('Station: ' + station_name) return stats from dispel4py.workflow_graph import WorkflowGraph nameStation = SimpleFunctionPE(name_station) streamProducer = StreamAndStatsProducer() streamProducer.name = "streamProducer" readStats = ReadStats() graph = WorkflowGraph() graph.connect(streamProducer, 'output', readStats, 'input') # The following line can be only performed if streamProducer is # a StreamAndStatsProducer object graph.connect(streamProducer, 'output_stats', nameStation, 'input') def detrend(st): st.detrend('simple') return st
IterativePE.__init__(self) self.index = index def _process(self, inputs): directory = inputs print("DIRECOTRY:%s " % str(directory)) for dir_entry in os.listdir(directory[self.index]): dir_entry_path = os.path.join(directory[self.index], dir_entry) if os.path.isfile(dir_entry_path): self.write('output', dir_entry_path) downloadPE = SimpleFunctionPE(download_data, { "add_end": 600, "add_start": 300 }) #fm downloadPE.name = "downloadPE" watcher = WatchDirectory(0) watcher_xml = WatchDirectory(1) waveformr = SimpleFunctionPE(waveform_reader) xmlr = SimpleFunctionPE(stationxml_reader) processes = [ waveform_reader, (plot_stream, { "source": "waveform_reader", "output_dir": "./output-images", "tag": "observed-image" }) ]
from obspy.signal.cross_correlation import xcorr import numpy def xcorrelation(data, maxlag): st1 = data[1] st2 = data[2] tr1 = st1[0].data tr2 = st2[0].data tr1 = tr1/numpy.linalg.norm(tr1) tr2 = tr2/numpy.linalg.norm(tr2) return xcorr(tr1, tr2, maxlag, full_xcorr=True)[2] from dispel4py.workflow_graph import WorkflowGraph streamProducer1 = SimpleFunctionPE(stream_producer) streamProducer1.name = "SP1" streamProducer2 = SimpleFunctionPE(stream_producer) streamProducer2.name = "SP2" stats1 = SimpleFunctionPE(readstats) stats2 = SimpleFunctionPE(readstats) match_traces=MatchPE() xcorrelation_traces= SimpleFunctionPE(xcorrelation, {'maxlag':1000}) pipeline = [ (decimate, {'sps':4}), detrend, demean, (filter, {'freqmin':0.01, 'freqmax':1., 'corners':4, 'zerophase':False}), spectralwhitening, readstats]
################################### collector = Collector() collector.name = 'collector' match = Match() match.name = 'match' #writer = Writer("-") writer = Writer() writer.name = 'writer' sc4 = Visualiser() sc4.name = 'vizu' read = SimpleFunctionPE(reader) #read=NetCDFReader() #comb=SimpleFunctionPE(combine,{"operator":"-"}) comb = SimpleFunctionPE(combine) #processes=[readn,multn] #chain = create_iterative_chain(processes, FunctionPE_class=SimpleFunctionPE) #Initialise the graph graph = WorkflowGraph() #Common way of composing the graph graph.connect(collector, 'output_R', read, 'input') graph.connect(read, 'output', match, 'input') graph.connect(match, 'output_X', comb, 'input')
import numpy def xcorrelation(data, maxlag): st1 = data[1] st2 = data[2] tr1 = st1[0].data tr2 = st2[0].data tr1 = tr1 / numpy.linalg.norm(tr1) tr2 = tr2 / numpy.linalg.norm(tr2) return xcorr(tr1, tr2, maxlag, full_xcorr=True)[2] from dispel4py.workflow_graph import WorkflowGraph streamProducer = SimpleFunctionPE(stream_producer) stats1 = SimpleFunctionPE(readstats) stats2 = SimpleFunctionPE(readstats) match_traces = MatchPE() xcorrelation_traces = SimpleFunctionPE(xcorrelation, {'maxlag': 1000}) pipeline = [(decimate, { 'sps': 4 }), detrend, demean, (filter, { 'freqmin': 0.01, 'freqmax': 1., 'corners': 4, 'zerophase': False }), spectralwhitening, readstats] preprocess_trace_1 = create_iterative_chain(pipeline)