def update(self, jsconfig): statusText = 'Unidentified Error Saving Project' # close any existing workflow if self.Workflow is not None: try: self.Workflow.close() except: pass # set the new updated config from the UI try: self._config = json.loads(jsconfig) except: statusText = 'This Project Configuration is Bad' else: # Check for valid input component (in_status, inputs) = self.config_has_valid_input() (out_status, outputs) = self.config_has_valid_output() if in_status is False: statusText = 'Unable To Save Configuration<br><br>No Valid Adapter Specified<br>.' # Check for valid output component elif out_status is False: statusText = 'Unable To Save Configuration<br><br>No Valid Publisher Specified<br>.' else: # translate the current config into a usable DAG result = self.translate() if result is False: statusText = 'Error Translating Supplied Configuration' # check the connectivity of the graph elif not self.is_connected(inputs, outputs): statusText = 'Unable To Save Project<br><br>Found Broken Path Between Adapter and Publisher.<br>' else: # Build the new workflow try: try: # get the core count from the config cores = int(config['cores']) # has to be at least 1 if cores < 1: cores = 1 except: log.warning('Could not get core count from config.') traceback.print_exc() log.warning('Defaulting to core count of 1') cores = 1 #log.info('Core count: %s' % cores) self.Workflow = Dataflow(self.Graph, cores) except: statusText = 'Error Constructing Workflow' else: statusText = 'Project Successfully Saved' # Save config here... fp = None try: fp = open('projects/default.txt', 'w') fp.write(jsconfig) except: log.error('Unable to save configuration') else: log.info('Configuration successfully saved') finally: if fp is not None: fp.close() return statusText
class DataFlowGraph(object): def __init__(self): self._workflow = None self._config = None self._graph = None # load plugins self.plugins = init_plugins() self.plugin_registry = {} # load each type here... try: self._filters = self.plugins['FILTER'].keys() except: self._filters = [] else: self.plugin_registry.update(self.plugins['FILTER']) try: self._transformers = self.plugins['TRANSFORMER'].keys() except: self._transformers = [] else: self.plugin_registry.update(self.plugins['TRANSFORMER']) try: self._operators = self.plugins['OPERATOR'].keys() except: self._operators = [] else: self.plugin_registry.update(self.plugins['OPERATOR']) try: self._extractors = self.plugins['EXTRACTOR'].keys() except: self._extractors = [] else: self.plugin_registry.update(self.plugins['EXTRACTOR']) try: self._input_adapters = self.plugins['ADAPTER'].keys() except: self._input_adapters = [] else: self.plugin_registry.update(self.plugins['ADAPTER']) try: self._output_adapters = self.plugins['PUBLISHER'].keys() except: self._output_adapter = [] else: self.plugin_registry.update(self.plugins['PUBLISHER']) self._registered_instances = {} fp = None try: fp = open('projects/default.txt', 'r') jsconfig = fp.read() config = json.loads(jsconfig) except: pass else: self.loadConfig(config) finally: if fp is not None: fp.close() def newInstance(self, className): try: className = className.replace(' ', '') cls = self.plugin_registry[className] this_instance = cls() except: # need to complain about invalid className log.error('Failed to create component %s - Invalid Classname' % className) log.debug('Plugin Registry Dump:', self.plugin_registry) return None else: key = str(hash(this_instance)) self._registered_instances[key] = this_instance return key def removeInstance(self, key): try: self._registered_instances[key] except: log.error('Failed to delete component %s - Instance not found' % key) log.debug('Registered Instances: %s' % self._registered_instances) else: self._registered_instances.pop(key) def Inputs(self, key): #return self._registered_instances[key].Inputs # needs error handling here... return self._registered_instances[key].get_in_ports() def Outputs(self, key): #return self._registered_instances[key].Outputs return self._registered_instances[key].get_out_ports() def Instance(self, key): return self._registered_instances[key] def update(self, jsconfig): statusText = 'Unidentified Error Saving Project' # close any existing workflow if self.Workflow is not None: try: self.Workflow.close() except: pass # set the new updated config from the UI try: self._config = json.loads(jsconfig) except: statusText = 'This Project Configuration is Bad' else: # Check for valid input component (in_status, inputs) = self.config_has_valid_input() (out_status, outputs) = self.config_has_valid_output() if in_status is False: statusText = 'Unable To Save Configuration<br><br>No Valid Adapter Specified<br>.' # Check for valid output component elif out_status is False: statusText = 'Unable To Save Configuration<br><br>No Valid Publisher Specified<br>.' else: # translate the current config into a usable DAG result = self.translate() if result is False: statusText = 'Error Translating Supplied Configuration' # check the connectivity of the graph elif not self.is_connected(inputs, outputs): statusText = 'Unable To Save Project<br><br>Found Broken Path Between Adapter and Publisher.<br>' else: # Build the new workflow try: try: # get the core count from the config cores = int(config['cores']) # has to be at least 1 if cores < 1: cores = 1 except: log.warning('Could not get core count from config.') traceback.print_exc() log.warning('Defaulting to core count of 1') cores = 1 #log.info('Core count: %s' % cores) self.Workflow = Dataflow(self.Graph, cores) except: statusText = 'Error Constructing Workflow' else: statusText = 'Project Successfully Saved' # Save config here... fp = None try: fp = open('projects/default.txt', 'w') fp.write(jsconfig) except: log.error('Unable to save configuration') else: log.info('Configuration successfully saved') finally: if fp is not None: fp.close() return statusText def is_connected(self, starts, ends): for start in starts: for end in ends: connected = self.find_path(self.Graph, self._registered_instances[start], self._registered_instances[end]) if connected is None: return False return True def find_path(self, graph, start, end, path=[]): path = path + [start] if start == end: return path if not graph.has_key(start): return None for node in graph[start]: if node not in path: newpath = self.find_path(graph, node, end, path) if newpath: return newpath return None def _get_workflow(self): return self._workflow def _set_workflow(self, wf): self._workflow = wf def _get_config(self): return self._config def _set_config(self, config): self._config = config def _get_graph(self): return self._graph def _set_graph(self, graph): self._graph = graph def config_has_valid_input(self): valid_input = False valid_inputs = [] for container in self.Config['containers']: if container['type'] == 'Adapters': valid_input = True valid_inputs.append(container['cid']) return (valid_input, valid_inputs) def config_has_valid_output(self): valid_output = False valid_outputs = [] for container in self.Config['containers']: if container['type'] == 'Publishers': valid_output = True valid_outputs.append(container['cid']) return (valid_output, valid_outputs) def translate(self): status = None G = {} for entry in self.Config['wires']: try: source_container_id = entry['src']['moduleId'] target_container_id = entry['tgt']['moduleId'] input = entry['tgt']['termid'] output = entry['src']['termid'] source_key = self.Config['containers'][source_container_id]['cid'] target_key = self.Config['containers'][target_container_id]['cid'] source = self._registered_instances[source_key] target = self._registered_instances[target_key] except: status = False else: if G.has_key(source): current_children = G[source] current_children[target] = (output, input) G[source] = current_children else: G[source] = {target:(output, input)} status = True # set the graph self.Graph = G return status def send(self, doc): response = {} try: if self.Workflow is not None: self.Workflow.send(doc) response['status'] = 'success' else: log.error('No workflow defined') response['status'] = 'failure' response['error'] = 'No Active Workflow Defined' except: response['status'] = 'failure' response['error'] = 'Unexpected Error Running Project' return response def _get_filters(self): self._filters.sort() return self._filters def _get_operators(self): self._operators.sort() return self._operators def _get_extractors(self): self._extractors.sort() return self._extractors def _get_input_adapters(self): self._input_adapters.sort() return self._input_adapters def _get_output_adapters(self): self._output_adapters.sort() return self._output_adapters def _get_transformers(self): self._transformers.sort() return self._transformers def getComponentConfig(self, id): try: params = self._registered_instances[id].get_parameters() except: log.error('Could not get component with id %s' % id) params = 'null' log.debug('Component Parameters: %s' % params) return params def setParam(self, id, param, value): try: self._registered_instances[id].set_parameter(param, value) log.debug('Setting Parameters: %s = %s' % (param, value)) except: log.error('Unable to set paramaters: %s' % (param, value)) # class properties Config = property(_get_config, _set_config) Graph = property(_get_graph, _set_graph) Workflow = property(_get_workflow, _set_workflow) Filters = property(_get_filters) Transformers = property(_get_transformers) Extractors = property(_get_extractors) InputAdapters = property(_get_input_adapters) OutputAdapters = property(_get_output_adapters) Operators = property(_get_operators) def loadConfig(self, config): for mod in config['containers']: this_mod = mod['filterName'] id = self.newInstance(this_mod) mod['cid'] = id try: for key, val in mod['params'].items(): self.setParam(id, key, val[0]) except: pass self.update(json.dumps(config))
def run(self): while True: for line in self.receive_all('in'): if self.regex.match(line): self.send('out', line) self.yield_ctrl() class Printer(Component): __metatype__ = 'PUBLISHER' def __init__(self): Component.__init__(self) def run(self): while True: for data in self.receive_all('in'): print data self.yield_ctrl() tail = Tail(open('/var/log/syslog', 'r')) grep = Grep('.*pants.*') printer = Printer() pipe = Dataflow({tail: {grep: ('out', 'in')}, grep: {printer: ('out', 'in')}}) while True: pipe.send(None) time.sleep(0.1)
from pypes.filters import ConsoleOutputWriter class HelloWorld(Component): __metatype__ = 'ADAPTER' def __init__(self): Component.__init__(self) def run(self): while True: for data in self.receive_all('in'): message = 'Hello %s' % data self.send('out', message) self.yield_ctrl() hello = HelloWorld() # our custom component printer = ConsoleOutputWriter() # writes to console (STDOUT) Network = { hello: {printer:('out','in')} } if __name__ == '__main__': # create a new data flow p = Dataflow(Network) # send some data through the data flow for name in ['Tom', 'Dick', 'Harry']: p.send(name) # shut down the data flow p.close()
while True: for line in self.receive_all('in'): if self.regex.match(line): self.send('out', line) self.yield_ctrl() class Printer(Component): __metatype__ = 'PUBLISHER' def __init__(self): Component.__init__(self) def run(self): while True: for data in self.receive_all('in'): print(data) self.yield_ctrl() tail = Tail(open('/var/log/syslog', 'r')) grep = Grep('.*pants.*') printer = Printer() pipe = Dataflow({ tail: {grep:('out','in')}, grep: {printer:('out', 'in')} }) while True: pipe.send(None) time.sleep(0.1)
| reader |-----| splitter | |__________| |__________|\ \ __________ __________ __________ __________ __________ \ | | | | | | | | | | \| grepper2 |-----| sorter |-----| cutter |-----| uniq |-----| printer2 | |__________| |__________| |__________| |__________| |__________| """ # The "network" defined formally as a dataflow graph Network = { reader: {splitter:('out','in')}, splitter: {grepper1:('out','in'), grepper2:('out2','in')}, grepper1: {printer1:('out','in')}, grepper2: { sorter:('out','in')}, sorter: { cutter:('out','in')}, cutter: { uniq:('out','in')}, uniq: {printer2:('out','in')} } # run our example if __name__ == '__main__': # create a new data flow p = Dataflow(Network) # send a document through the data flow p.send('cups.log') # shut down the data flow p.close()
}, splitter: { grepper1: ('out', 'in'), grepper2: ('out2', 'in') }, grepper1: { printer1: ('out', 'in') }, grepper2: { sorter: ('out', 'in') }, sorter: { cutter: ('out', 'in') }, cutter: { uniq: ('out', 'in') }, uniq: { printer2: ('out', 'in') } } # run our example if __name__ == '__main__': # create a new data flow p = Dataflow(Network) # send a document through the data flow p.send('cups.log') # shut down the data flow p.close()
doubler2: ("out", "in") }, } return graph if __name__ == '__main__': quadrupler = pypes.component.HigherOrderComponent( quadrupler_network_factory()) quadrupler.__metatype__ = "ADAPTER" printer = Printer() doubler = Doubler() doubler.__metatype__ = "ADAPTER" network = { quadrupler: { printer: ("out", "in"), }, } #network = { #doubler: { #printer: ("out", "in"), #}, #} from pypes.pipeline import Dataflow pipeline = Dataflow(network) for number in range(1, 5): pipeline.send(number) pipeline.close()
class DataFlowGraph(object): def __init__(self): self._workflow = None self._config = None self._graph = None # load plugins self.plugins = init_plugins() self.plugin_registry = {} # load each type here... try: self._filters = self.plugins['FILTER'].keys() except: self._filters = [] else: self.plugin_registry.update(self.plugins['FILTER']) try: self._transformers = self.plugins['TRANSFORMER'].keys() except: self._transformers = [] else: self.plugin_registry.update(self.plugins['TRANSFORMER']) try: self._operators = self.plugins['OPERATOR'].keys() except: self._operators = [] else: self.plugin_registry.update(self.plugins['OPERATOR']) try: self._extractors = self.plugins['EXTRACTOR'].keys() except: self._extractors = [] else: self.plugin_registry.update(self.plugins['EXTRACTOR']) try: self._input_adapters = self.plugins['ADAPTER'].keys() except: self._input_adapters = [] else: self.plugin_registry.update(self.plugins['ADAPTER']) try: self._output_adapters = self.plugins['PUBLISHER'].keys() except: self._output_adapter = [] else: self.plugin_registry.update(self.plugins['PUBLISHER']) self._registered_instances = {} fp = None try: fp = open('projects/default.txt', 'r') jsconfig = fp.read() config = json.loads(jsconfig) except: pass else: self.loadConfig(config) finally: if fp is not None: fp.close() def newInstance(self, className): try: className = className.replace(' ', '') cls = self.plugin_registry[className] this_instance = cls() except: # need to complain about invalid className log.error('Failed to create component %s - Invalid Classname' % className) log.debug('Plugin Registry Dump:', self.plugin_registry) return None else: key = str(hash(this_instance)) self._registered_instances[key] = this_instance return key def removeInstance(self, key): try: self._registered_instances[key] except: log.error('Failed to delete component %s - Instance not found' % key) log.debug('Registered Instances: %s' % self._registered_instances) else: self._registered_instances.pop(key) def Inputs(self, key): #return self._registered_instances[key].Inputs # needs error handling here... return self._registered_instances[key].get_in_ports() def Outputs(self, key): #return self._registered_instances[key].Outputs return self._registered_instances[key].get_out_ports() def Instance(self, key): return self._registered_instances[key] def update(self, jsconfig): statusText = 'Unidentified Error Saving Project' # close any existing workflow if self.Workflow is not None: try: self.Workflow.close() except: pass # set the new updated config from the UI try: self._config = json.loads(jsconfig) except: statusText = 'This Project Configuration is Bad' else: # Check for valid input component (in_status, inputs) = self.config_has_valid_input() (out_status, outputs) = self.config_has_valid_output() if in_status is False: statusText = 'Unable To Save Configuration<br><br>No Valid Adapter Specified<br>.' # Check for valid output component elif out_status is False: statusText = 'Unable To Save Configuration<br><br>No Valid Publisher Specified<br>.' else: # translate the current config into a usable DAG result = self.translate() if result is False: statusText = 'Error Translating Supplied Configuration' # check the connectivity of the graph elif not self.is_connected(inputs, outputs): statusText = 'Unable To Save Project<br><br>Found Broken Path Between Adapter and Publisher.<br>' else: # Build the new workflow try: try: # get the core count from the config cores = int(config['cores']) # has to be at least 1 if cores < 1: cores = 1 except: log.warning( 'Could not get core count from config.') traceback.print_exc() log.warning('Defaulting to core count of 1') cores = 1 #log.info('Core count: %s' % cores) self.Workflow = Dataflow(self.Graph, cores) except: statusText = 'Error Constructing Workflow' else: statusText = 'Project Successfully Saved' # Save config here... fp = None try: fp = open('projects/default.txt', 'w') fp.write(jsconfig) except: log.error('Unable to save configuration') else: log.info('Configuration successfully saved') finally: if fp is not None: fp.close() return statusText def is_connected(self, starts, ends): for start in starts: for end in ends: connected = self.find_path(self.Graph, self._registered_instances[start], self._registered_instances[end]) if connected is None: return False return True def find_path(self, graph, start, end, path=[]): path = path + [start] if start == end: return path if not graph.has_key(start): return None for node in graph[start]: if node not in path: newpath = self.find_path(graph, node, end, path) if newpath: return newpath return None def _get_workflow(self): return self._workflow def _set_workflow(self, wf): self._workflow = wf def _get_config(self): return self._config def _set_config(self, config): self._config = config def _get_graph(self): return self._graph def _set_graph(self, graph): self._graph = graph def config_has_valid_input(self): valid_input = False valid_inputs = [] for container in self.Config['containers']: if container['type'] == 'Adapters': valid_input = True valid_inputs.append(container['cid']) return (valid_input, valid_inputs) def config_has_valid_output(self): valid_output = False valid_outputs = [] for container in self.Config['containers']: if container['type'] == 'Publishers': valid_output = True valid_outputs.append(container['cid']) return (valid_output, valid_outputs) def translate(self): status = None G = {} for entry in self.Config['wires']: try: source_container_id = entry['src']['moduleId'] target_container_id = entry['tgt']['moduleId'] input = entry['tgt']['termid'] output = entry['src']['termid'] source_key = self.Config['containers'][source_container_id][ 'cid'] target_key = self.Config['containers'][target_container_id][ 'cid'] source = self._registered_instances[source_key] target = self._registered_instances[target_key] except: status = False else: if G.has_key(source): current_children = G[source] current_children[target] = (output, input) G[source] = current_children else: G[source] = {target: (output, input)} status = True # set the graph self.Graph = G return status def send(self, doc): response = {} try: if self.Workflow is not None: self.Workflow.send(doc) response['status'] = 'success' else: log.error('No workflow defined') response['status'] = 'failure' response['error'] = 'No Active Workflow Defined' except: response['status'] = 'failure' response['error'] = 'Unexpected Error Running Project' return response def _get_filters(self): self._filters.sort() return self._filters def _get_operators(self): self._operators.sort() return self._operators def _get_extractors(self): self._extractors.sort() return self._extractors def _get_input_adapters(self): self._input_adapters.sort() return self._input_adapters def _get_output_adapters(self): self._output_adapters.sort() return self._output_adapters def _get_transformers(self): self._transformers.sort() return self._transformers def getComponentConfig(self, id): try: params = self._registered_instances[id].get_parameters() except: log.error('Could not get component with id %s' % id) params = 'null' log.debug('Component Parameters: %s' % params) return params def setParam(self, id, param, value): try: self._registered_instances[id].set_parameter(param, value) log.debug('Setting Parameters: %s = %s' % (param, value)) except: log.error('Unable to set paramaters: %s' % (param, value)) # class properties Config = property(_get_config, _set_config) Graph = property(_get_graph, _set_graph) Workflow = property(_get_workflow, _set_workflow) Filters = property(_get_filters) Transformers = property(_get_transformers) Extractors = property(_get_extractors) InputAdapters = property(_get_input_adapters) OutputAdapters = property(_get_output_adapters) Operators = property(_get_operators) def loadConfig(self, config): for mod in config['containers']: this_mod = mod['filterName'] id = self.newInstance(this_mod) mod['cid'] = id try: for key, val in mod['params'].items(): self.setParam(id, key, val[0]) except: pass self.update(json.dumps(config))
def update(self, jsconfig): statusText = 'Unidentified Error Saving Project' # close any existing workflow if self.Workflow is not None: try: self.Workflow.close() except: pass # set the new updated config from the UI try: self._config = json.loads(jsconfig) except: statusText = 'This Project Configuration is Bad' else: # Check for valid input component (in_status, inputs) = self.config_has_valid_input() (out_status, outputs) = self.config_has_valid_output() if in_status is False: statusText = 'Unable To Save Configuration<br><br>No Valid Adapter Specified<br>.' # Check for valid output component elif out_status is False: statusText = 'Unable To Save Configuration<br><br>No Valid Publisher Specified<br>.' else: # translate the current config into a usable DAG result = self.translate() if result is False: statusText = 'Error Translating Supplied Configuration' # check the connectivity of the graph elif not self.is_connected(inputs, outputs): statusText = 'Unable To Save Project<br><br>Found Broken Path Between Adapter and Publisher.<br>' else: # Build the new workflow try: try: # get the core count from the config cores = int(config['cores']) # has to be at least 1 if cores < 1: cores = 1 except: log.warning( 'Could not get core count from config.') traceback.print_exc() log.warning('Defaulting to core count of 1') cores = 1 #log.info('Core count: %s' % cores) self.Workflow = Dataflow(self.Graph, cores) except: statusText = 'Error Constructing Workflow' else: statusText = 'Project Successfully Saved' # Save config here... fp = None try: fp = open('projects/default.txt', 'w') fp.write(jsconfig) except: log.error('Unable to save configuration') else: log.info('Configuration successfully saved') finally: if fp is not None: fp.close() return statusText