def createDefinition(self): atomicDefinition = DEFINITION_ECHO nestDefinition = DefinitionModule.getNewNestDefinition() parameter = ParameterModule.DataParameter( id='item to echo', portDirection=ParameterModule.PORT_DIRECTION_INPUT) nestDefinition.addParameter(parameter) node = nestDefinition.createNode(id='node') node.definitionToReference(atomicDefinition) node.name('echo') nestDefinition._connectParameters( nestDefinition, 'item to echo', node, 'item to echo' ) compositeDefinition = DefinitionModule.getNewNestDefinition() parameter = ParameterModule.DataParameter( id='item to echo', portDirection=ParameterModule.PORT_DIRECTION_INPUT) compositeDefinition.addParameter(parameter) node = compositeDefinition.createNode(id='nest') node.definitionToReference(nestDefinition) compositeDefinition._connectParameters( compositeDefinition, 'item to echo', node, 'item to echo' ) return compositeDefinition
def generateBootstrapLoaderPomset(self): """ generates the composite pomset used to load all the library pomsets """ # now create a load library definitions pomset # that will load the two wordcount pomsets defToLoadDef = self.bootstrapLoaderDefinitions()[ID_LOADLIBRARYDEFINITION] defToLoadDefs = DefinitionModule.getNewNestDefinition() notBootstrapLoaderFilter = FilterModule.constructNotFilter() notBootstrapLoaderFilter.addFilter(getBootstrapLoaderPomsetsFilter()) definitions = RelationalModule.Table.reduceRetrieve( self.definitionTable(), notBootstrapLoaderFilter, ['definition'], []) for definitionToLoad in definitions: loadNode = defToLoadDefs.createNode(id=uuid.uuid4()) loadNode.definitionToReference(defToLoadDef) loadNode.isCritical(False) loadNode.name('load %s' % definitionToLoad.url()) loadNode.setParameterBinding('pomset url', definitionToLoad.url()) pass defToLoadDefs.id(ID_BOOTSTRAPLOADER) defToLoadDefs.name('bootstrap pomsets loader') return defToLoadDefs
def createDefinition(self): atomicDefinition = DEFINITION_ECHO compositeDefinition = DefinitionModule.getNewNestDefinition() parameter = ParameterModule.DataParameter( id='item to echo', portDirection=ParameterModule.PORT_DIRECTION_INPUT) compositeDefinition.addParameter(parameter) nodes = [] for index in range(3): node = compositeDefinition.createNode(id='node%s' % index) node.definitionToReference(atomicDefinition) compositeDefinition._connectParameters( compositeDefinition, 'item to echo', node, 'item to echo' ) node.name('echo %s' % index) nodes.append(node) pass for sourceNode, targetNode in zip(nodes[:-1], nodes[1:]): compositeDefinition.connectNodes( sourceNode, 'temporal output', targetNode, 'temporal input' ) pass return compositeDefinition
def createPomsetContainingParameterSweep(): compositeDefinition = DefinitionModule.getNewNestDefinition() # setup the reference definition for parameter sweep mapperNode = compositeDefinition.createNode(id='mapper') mapperNode.definitionToReference(DEFINITION_WORDCOUNT) mapperNode.isParameterSweep('input file', True) mapperNode.isParameterSweep('output file', True) mapperNode.addParameterSweepGroup(['input file', 'output file']) mapperNode.isCritical(True) mapperNode.name('mapper') reducerNode = compositeDefinition.createNode(id='reducer') reducerNode.definitionToReference(DEFINITION_WORDCOUNT_REDUCE) reducerNode.isCritical(True) reducerNode.name('reducer') builder = BuilderModule.Builder() builder.connect( compositeDefinition, mapperNode, 'output file', reducerNode, 'input files') compositeDefinition.name('basic map-reduce') return compositeDefinition
def createDefinition(self): compositeDefinition = DefinitionModule.getNewNestDefinition() # setup the reference definition for parameter sweep mapperNode = compositeDefinition.createNode(id='mapper') mapperNode.definitionToReference(BaseModule.DEFINITION_WORDCOUNT) mapperNode.isParameterSweep('input file', True) mapperNode.isParameterSweep('output file', True) mapperNode.name('mapper') reducerNode = compositeDefinition.createNode(id='reducer') reducerNode.definitionToReference(BaseModule.DEFINITION_WORDCOUNT_REDUCE) reducerNode.name('reducer') inputParameter = \ ParameterModule.DataParameter( id='input file', portDirection=ParameterModule.PORT_DIRECTION_INPUT) inputParameter.setAttribute( ParameterModule.PORT_ATTRIBUTE_PARAMETERSWEEP, True) compositeDefinition.addParameter(inputParameter) compositeDefinition._connectParameters( compositeDefinition, 'input file', mapperNode, 'input file' ) outputParameter = \ ParameterModule.DataParameter( id='output file', portDirection=ParameterModule.PORT_DIRECTION_INPUT) compositeDefinition.addParameter(outputParameter) compositeDefinition._connectParameters( compositeDefinition, 'output file', reducerNode, 'output file' ) blackboardParameter = \ ParameterModule.BlackboardParameter('intermediate file') compositeDefinition.addParameter(blackboardParameter) compositeDefinition._connectParameters( compositeDefinition, 'intermediate file', mapperNode, 'output file' ) edge = compositeDefinition.connectNodes( mapperNode, 'output file', reducerNode, 'input files', ) self.intermediateParameterId = edge.path()[3] return compositeDefinition
def createPomsetContainingLoopDefinition(): compositeDefinition = DefinitionModule.getNewNestDefinition() loopDefinition = DefinitionModule.LoopDefinition() # the loop definition needs to add one (or more nodes) wcNode = loopDefinition.createNode(id='wordcount') wcNode.definitionToReference(DEFINITION_WORDCOUNT) wcNode.name('wordcount') loopNode = compositeDefinition.createNode(id='loop') loopNode.definitionToReference(loopDefinition) return compositeDefinition
def createNewNestPomset(self, name=None): """ """ #TODO: this should construct a command to create the new pomset # and execute the command (or send an event to execute the command) # within the command framework, the command should also # create an event to update the GUI newPomset = DefinitionModule.getNewNestDefinition() if name is None: name = 'pomset %s' % uuid.uuid4().hex[:3] newPomset.name(name) newPomsetContext = ContextModule.wrapPomsetInContext(newPomset) return newPomsetContext
def createPomsetContainingBranchDefinition(): compositeDefinition = DefinitionModule.getNewNestDefinition() branchDefinition = DefinitionModule.BranchDefinition() # the loop definition needs to add one (or more nodes) wcNode = branchDefinition.createNode(id='wordcount') wcNode.definitionToReference(DEFINITION_WORDCOUNT) wcNode.name('wordcount') reduceNode = branchDefinition.createNode(id='reduce') reduceNode.definitionToReference(DEFINITION_WORDCOUNT_REDUCE) reduceNode.name('reduce') branchNode = compositeDefinition.createNode(id='branch') branchNode.definitionToReference(branchDefinition) return compositeDefinition
def createDefinition(self): atomicDefinition = DEFINITION_ECHO compositeDefinition = DefinitionModule.getNewNestDefinition() node = compositeDefinition.createNode(id='node') node.definitionToReference(atomicDefinition) node.name('echo') parameter = ParameterModule.DataParameter( id='item to echo', portDirection=ParameterModule.PORT_DIRECTION_INPUT) parameter.setAttribute(ParameterModule.PORT_ATTRIBUTE_PARAMETERSWEEP, True) compositeDefinition.addParameter(parameter) node.isParameterSweep('item to echo', True) compositeDefinition._connectParameters( compositeDefinition, 'item to echo', node, 'item to echo' ) return compositeDefinition
def testPickleAndLoad(self): """ This verifies a pomset saved out and reloaded still references the same definition - create a pomset that references the two library definitions - save out the pomset - load the pomset again - assert the references are identical, using Python's "is" """ # load the library definitions library = self.initializeLibrary() request = self.automaton.runBootstrapLoader(library) assert not request.exception # create the pomset, add a node # and have that node reference a library definition filter = RelationalModule.ColumnValueFilter( 'definition', FilterModule.IdFilter(TestDefinitionModule.ID_WORDCOUNT)) definitionToReference = library.getDefinition(filter) compositeDefinition = DefinitionModule.getNewNestDefinition() mapperNode = compositeDefinition.createNode(id='mapper') mapperNode.definitionToReference(definitionToReference) # pickle the pomset # unpickle the pomset definition = TestDefinitionModule.pickleAndReloadDefinition( os.sep.join(['', 'tmp', 'foo.pomset']), compositeDefinition ) library.updateWithLibraryDefinitions(definition.definitionToReference()) definitionToReference = library.getDefinition(filter) self.assertTrue(definition.definitionToReference().nodes()[0].definitionToReference() is definitionToReference) return
def createDefinition(self): compositeDefinition = DefinitionModule.getNewNestDefinition() # setup the reference definition for parameter sweep mapperNode = compositeDefinition.createNode(id='mapper') mapperNode.definitionToReference(DEFINITION_WORDCOUNT) mapperNode.isParameterSweep('input file', True) mapperNode.isParameterSweep('output file', True) mapperNode.addParameterSweepGroup(['input file', 'output file']) mapperNode.name('mapper') reducerNode = compositeDefinition.createNode(id='reducer') reducerNode.definitionToReference(DEFINITION_WORDCOUNT_REDUCE) reducerNode.name('reducer') self.builder.exposeNodeParameter( compositeDefinition, 'input file', mapperNode, 'input file', shouldCreate=True) self.builder.exposeNodeParameter( compositeDefinition, 'output file', reducerNode, 'output file', shouldCreate=True) self.builder.exposeNodeParameter( compositeDefinition, 'intermediate file', mapperNode, 'output file', shouldCreate=True) edge = self.builder.connect( compositeDefinition, mapperNode, 'output file', reducerNode, 'input files', ) return compositeDefinition
def createDefinition(self): atomicDefinition = DEFINITION_WORDCOUNT_REDUCE compositeDefinition = DefinitionModule.getNewNestDefinition() node = compositeDefinition.createNode(id='node') node.definitionToReference(atomicDefinition) node.name('reducer') parameterIds = ['input files', 'output file'] for parameterId in parameterIds: parameter = ParameterModule.DataParameter( id=parameterId, portDirection=ParameterModule.PORT_DIRECTION_INPUT) compositeDefinition.addParameter(parameter) compositeDefinition._connectParameters( compositeDefinition, parameterId, node, parameterId ) pass return compositeDefinition
def createDefinition(self): atomicDefinition = DEFINITION_WORDCOUNT compositeDefinition = DefinitionModule.getNewNestDefinition() node = compositeDefinition.createNode(id='node') node.definitionToReference(atomicDefinition) node.name('mapper') # TODO: # the info to expose the parameters on the composite definition # should be retrieved from the definition itself parameterGroup = { 'input file':{ParameterModule.PORT_ATTRIBUTE_PARAMETERSWEEP:True}, 'output file':{ ParameterModule.PORT_ATTRIBUTE_PARAMETERSWEEP:True, ParameterModule.PORT_ATTRIBUTE_ISSIDEEFFECT:True } } for parameterId, attributes in parameterGroup.iteritems(): parameter = ParameterModule.DataParameter( id=parameterId, portDirection=ParameterModule.PORT_DIRECTION_INPUT) for attributeKey, attributeValue in attributes.iteritems(): parameter.setAttribute(attributeKey, attributeValue) pass compositeDefinition.addParameter(parameter) node.isParameterSweep(parameterId, True) compositeDefinition._connectParameters( compositeDefinition, parameterId, node, parameterId ) pass node.addParameterSweepGroup(parameterGroup.keys()) return compositeDefinition
def createNestDefinition(): compositeDefinition = DefinitionModule.getNewNestDefinition() return compositeDefinition