def createDefinition(self):
atomicDefinition = DEFINITION_ECHO
nestDefinition = DefinitionModule.getNewNestDefinition()
parameter = ParameterModule.DataParameter(
id='item to echo',
portDirection=ParameterModule.PORT_DIRECTION_INPUT)
nestDefinition.addParameter(parameter)
node = nestDefinition.createNode(id='node')
node.definitionToReference(atomicDefinition)
node.name('echo')
nestDefinition._connectParameters(
nestDefinition, 'item to echo',
node, 'item to echo'
)
compositeDefinition = DefinitionModule.getNewNestDefinition()
parameter = ParameterModule.DataParameter(
id='item to echo',
portDirection=ParameterModule.PORT_DIRECTION_INPUT)
compositeDefinition.addParameter(parameter)
node = compositeDefinition.createNode(id='nest')
node.definitionToReference(nestDefinition)
compositeDefinition._connectParameters(
compositeDefinition, 'item to echo',
node, 'item to echo'
)
return compositeDefinition
def generateBootstrapLoaderPomset(self):
"""
generates the composite pomset used to load
all the library pomsets
"""
# now create a load library definitions pomset
# that will load the two wordcount pomsets
defToLoadDef = self.bootstrapLoaderDefinitions()[ID_LOADLIBRARYDEFINITION]
defToLoadDefs = DefinitionModule.getNewNestDefinition()
notBootstrapLoaderFilter = FilterModule.constructNotFilter()
notBootstrapLoaderFilter.addFilter(getBootstrapLoaderPomsetsFilter())
definitions = RelationalModule.Table.reduceRetrieve(
self.definitionTable(),
notBootstrapLoaderFilter,
['definition'], [])
for definitionToLoad in definitions:
loadNode = defToLoadDefs.createNode(id=uuid.uuid4())
loadNode.definitionToReference(defToLoadDef)
loadNode.isCritical(False)
loadNode.name('load %s' % definitionToLoad.url())
loadNode.setParameterBinding('pomset url', definitionToLoad.url())
pass
defToLoadDefs.id(ID_BOOTSTRAPLOADER)
defToLoadDefs.name('bootstrap pomsets loader')
return defToLoadDefs
def createDefinition(self):
atomicDefinition = DEFINITION_ECHO
compositeDefinition = DefinitionModule.getNewNestDefinition()
parameter = ParameterModule.DataParameter(
id='item to echo',
portDirection=ParameterModule.PORT_DIRECTION_INPUT)
compositeDefinition.addParameter(parameter)
nodes = []
for index in range(3):
node = compositeDefinition.createNode(id='node%s' % index)
node.definitionToReference(atomicDefinition)
compositeDefinition._connectParameters(
compositeDefinition, 'item to echo',
node, 'item to echo'
)
node.name('echo %s' % index)
nodes.append(node)
pass
for sourceNode, targetNode in zip(nodes[:-1], nodes[1:]):
compositeDefinition.connectNodes(
sourceNode, 'temporal output',
targetNode, 'temporal input'
)
pass
return compositeDefinition
def createPomsetContainingParameterSweep():
compositeDefinition = DefinitionModule.getNewNestDefinition()
# setup the reference definition for parameter sweep
mapperNode = compositeDefinition.createNode(id='mapper')
mapperNode.definitionToReference(DEFINITION_WORDCOUNT)
mapperNode.isParameterSweep('input file', True)
mapperNode.isParameterSweep('output file', True)
mapperNode.addParameterSweepGroup(['input file', 'output file'])
mapperNode.isCritical(True)
mapperNode.name('mapper')
reducerNode = compositeDefinition.createNode(id='reducer')
reducerNode.definitionToReference(DEFINITION_WORDCOUNT_REDUCE)
reducerNode.isCritical(True)
reducerNode.name('reducer')
builder = BuilderModule.Builder()
builder.connect(
compositeDefinition,
mapperNode, 'output file',
reducerNode, 'input files')
compositeDefinition.name('basic map-reduce')
return compositeDefinition
def createDefinition(self):
compositeDefinition = DefinitionModule.getNewNestDefinition()
# setup the reference definition for parameter sweep
mapperNode = compositeDefinition.createNode(id='mapper')
mapperNode.definitionToReference(BaseModule.DEFINITION_WORDCOUNT)
mapperNode.isParameterSweep('input file', True)
mapperNode.isParameterSweep('output file', True)
mapperNode.name('mapper')
reducerNode = compositeDefinition.createNode(id='reducer')
reducerNode.definitionToReference(BaseModule.DEFINITION_WORDCOUNT_REDUCE)
reducerNode.name('reducer')
inputParameter = \
ParameterModule.DataParameter(
id='input file',
portDirection=ParameterModule.PORT_DIRECTION_INPUT)
inputParameter.setAttribute(
ParameterModule.PORT_ATTRIBUTE_PARAMETERSWEEP, True)
compositeDefinition.addParameter(inputParameter)
compositeDefinition._connectParameters(
compositeDefinition, 'input file',
mapperNode, 'input file'
)
outputParameter = \
ParameterModule.DataParameter(
id='output file',
portDirection=ParameterModule.PORT_DIRECTION_INPUT)
compositeDefinition.addParameter(outputParameter)
compositeDefinition._connectParameters(
compositeDefinition, 'output file',
reducerNode, 'output file'
)
blackboardParameter = \
ParameterModule.BlackboardParameter('intermediate file')
compositeDefinition.addParameter(blackboardParameter)
compositeDefinition._connectParameters(
compositeDefinition, 'intermediate file',
mapperNode, 'output file'
)
edge = compositeDefinition.connectNodes(
mapperNode, 'output file',
reducerNode, 'input files',
)
self.intermediateParameterId = edge.path()[3]
return compositeDefinition
def createPomsetContainingLoopDefinition():
compositeDefinition = DefinitionModule.getNewNestDefinition()
loopDefinition = DefinitionModule.LoopDefinition()
# the loop definition needs to add one (or more nodes)
wcNode = loopDefinition.createNode(id='wordcount')
wcNode.definitionToReference(DEFINITION_WORDCOUNT)
wcNode.name('wordcount')
loopNode = compositeDefinition.createNode(id='loop')
loopNode.definitionToReference(loopDefinition)
return compositeDefinition
def createNewNestPomset(self, name=None):
"""
"""
#TODO: this should construct a command to create the new pomset
# and execute the command (or send an event to execute the command)
# within the command framework, the command should also
# create an event to update the GUI
newPomset = DefinitionModule.getNewNestDefinition()
if name is None:
name = 'pomset %s' % uuid.uuid4().hex[:3]
newPomset.name(name)
newPomsetContext = ContextModule.wrapPomsetInContext(newPomset)
return newPomsetContext
def createPomsetContainingBranchDefinition():
compositeDefinition = DefinitionModule.getNewNestDefinition()
branchDefinition = DefinitionModule.BranchDefinition()
# the loop definition needs to add one (or more nodes)
wcNode = branchDefinition.createNode(id='wordcount')
wcNode.definitionToReference(DEFINITION_WORDCOUNT)
wcNode.name('wordcount')
reduceNode = branchDefinition.createNode(id='reduce')
reduceNode.definitionToReference(DEFINITION_WORDCOUNT_REDUCE)
reduceNode.name('reduce')
branchNode = compositeDefinition.createNode(id='branch')
branchNode.definitionToReference(branchDefinition)
return compositeDefinition
def createDefinition(self):
atomicDefinition = DEFINITION_ECHO
compositeDefinition = DefinitionModule.getNewNestDefinition()
node = compositeDefinition.createNode(id='node')
node.definitionToReference(atomicDefinition)
node.name('echo')
parameter = ParameterModule.DataParameter(
id='item to echo',
portDirection=ParameterModule.PORT_DIRECTION_INPUT)
parameter.setAttribute(ParameterModule.PORT_ATTRIBUTE_PARAMETERSWEEP, True)
compositeDefinition.addParameter(parameter)
node.isParameterSweep('item to echo', True)
compositeDefinition._connectParameters(
compositeDefinition, 'item to echo',
node, 'item to echo'
)
return compositeDefinition
def testPickleAndLoad(self):
"""
This verifies a pomset saved out and reloaded
still references the same definition
- create a pomset that references the two library definitions
- save out the pomset
- load the pomset again
- assert the references are identical, using Python's "is"
"""
# load the library definitions
library = self.initializeLibrary()
request = self.automaton.runBootstrapLoader(library)
assert not request.exception
# create the pomset, add a node
# and have that node reference a library definition
filter = RelationalModule.ColumnValueFilter(
'definition',
FilterModule.IdFilter(TestDefinitionModule.ID_WORDCOUNT))
definitionToReference = library.getDefinition(filter)
compositeDefinition = DefinitionModule.getNewNestDefinition()
mapperNode = compositeDefinition.createNode(id='mapper')
mapperNode.definitionToReference(definitionToReference)
# pickle the pomset
# unpickle the pomset
definition = TestDefinitionModule.pickleAndReloadDefinition(
os.sep.join(['', 'tmp', 'foo.pomset']),
compositeDefinition
)
library.updateWithLibraryDefinitions(definition.definitionToReference())
definitionToReference = library.getDefinition(filter)
self.assertTrue(definition.definitionToReference().nodes()[0].definitionToReference() is definitionToReference)
return
def createDefinition(self):
compositeDefinition = DefinitionModule.getNewNestDefinition()
# setup the reference definition for parameter sweep
mapperNode = compositeDefinition.createNode(id='mapper')
mapperNode.definitionToReference(DEFINITION_WORDCOUNT)
mapperNode.isParameterSweep('input file', True)
mapperNode.isParameterSweep('output file', True)
mapperNode.addParameterSweepGroup(['input file', 'output file'])
mapperNode.name('mapper')
reducerNode = compositeDefinition.createNode(id='reducer')
reducerNode.definitionToReference(DEFINITION_WORDCOUNT_REDUCE)
reducerNode.name('reducer')
self.builder.exposeNodeParameter(
compositeDefinition, 'input file',
mapperNode, 'input file',
shouldCreate=True)
self.builder.exposeNodeParameter(
compositeDefinition, 'output file',
reducerNode, 'output file',
shouldCreate=True)
self.builder.exposeNodeParameter(
compositeDefinition, 'intermediate file',
mapperNode, 'output file',
shouldCreate=True)
edge = self.builder.connect(
compositeDefinition,
mapperNode, 'output file',
reducerNode, 'input files',
)
return compositeDefinition
def createDefinition(self):
atomicDefinition = DEFINITION_WORDCOUNT_REDUCE
compositeDefinition = DefinitionModule.getNewNestDefinition()
node = compositeDefinition.createNode(id='node')
node.definitionToReference(atomicDefinition)
node.name('reducer')
parameterIds = ['input files', 'output file']
for parameterId in parameterIds:
parameter = ParameterModule.DataParameter(
id=parameterId,
portDirection=ParameterModule.PORT_DIRECTION_INPUT)
compositeDefinition.addParameter(parameter)
compositeDefinition._connectParameters(
compositeDefinition, parameterId,
node, parameterId
)
pass
return compositeDefinition
def createDefinition(self):
atomicDefinition = DEFINITION_WORDCOUNT
compositeDefinition = DefinitionModule.getNewNestDefinition()
node = compositeDefinition.createNode(id='node')
node.definitionToReference(atomicDefinition)
node.name('mapper')
# TODO:
# the info to expose the parameters on the composite definition
# should be retrieved from the definition itself
parameterGroup = {
'input file':{ParameterModule.PORT_ATTRIBUTE_PARAMETERSWEEP:True},
'output file':{
ParameterModule.PORT_ATTRIBUTE_PARAMETERSWEEP:True,
ParameterModule.PORT_ATTRIBUTE_ISSIDEEFFECT:True
}
}
for parameterId, attributes in parameterGroup.iteritems():
parameter = ParameterModule.DataParameter(
id=parameterId,
portDirection=ParameterModule.PORT_DIRECTION_INPUT)
for attributeKey, attributeValue in attributes.iteritems():
parameter.setAttribute(attributeKey, attributeValue)
pass
compositeDefinition.addParameter(parameter)
node.isParameterSweep(parameterId, True)
compositeDefinition._connectParameters(
compositeDefinition, parameterId,
node, parameterId
)
pass
node.addParameterSweepGroup(parameterGroup.keys())
return compositeDefinition
def createNestDefinition():
compositeDefinition = DefinitionModule.getNewNestDefinition()
return compositeDefinition
