def requires(self):
"""
The list of RunDROPTask that are required by this one.
We use self.__class__ to create the new dependencies so this method
doesn't need to be rewritten by all subclasses
"""
re = []
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Checking requirements for RunDROPTask %s/%s" %(self.data_obj.oid, self.data_obj.uid))
# The requires() method will be called not only when creating the
# initial tree of tasks, but also at runtime. For a given graph in a
# DM that has been connected with to other graph running in a different
# DM, it will mean that at runtime more upstream objects will be found
# for those nodes connected to an external graph. We shouldn't schedule
# those objects though, since they are scheduled by their own DM.
# We simply filter then the upObjs here to return only those that are
# actually an instance of AbstractDROP, thus removing any Pyro
# Proxy instances from the list
upObjs = droputils.getUpstreamObjects(self.data_obj)
upObjs = filter(lambda drop: isinstance(drop, AbstractDROP), upObjs)
for req in upObjs:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Added requirement %s/%s" %(req.oid, req.uid))
re.append(RunDROPTask(req, self.sessionId))
return re
def requires(self):
"""
The list of RunDROPTask that are required by this one.
We use self.__class__ to create the new dependencies so this method
doesn't need to be rewritten by all subclasses
"""
re = []
logger.debug("Checking requirements for RunDROPTask %r", self.data_obj)
# The requires() method will be called not only when creating the
# initial tree of tasks, but also at runtime. For a given graph in a
# DM that has been connected with to other graph running in a different
# DM, it will mean that at runtime more upstream objects will be found
# for those nodes connected to an external graph. We shouldn't schedule
# those objects though, since they are scheduled by their own DM.
# We simply filter then the upObjs here to return only those that are
# actually an instance of AbstractDROP, thus removing any Pyro
# Proxy instances from the list
upObjs = droputils.getUpstreamObjects(self.data_obj)
upObjs = filter(lambda drop: isinstance(drop, AbstractDROP), upObjs)
for req in upObjs:
logger.debug("Added requirement %r", req)
re.append(RunDROPTask(req, self.sessionId))
return re
def assertUpstream(self, node, upstreamNodes):
if not isinstance(upstreamNodes, list):
upstreamNodes = [upstreamNodes]
# Normal check
self.assertSetEqual(set(upstreamNodes), set(droputils.getUpstreamObjects(node)))
# Check the other way too
for upNode in upstreamNodes:
self.assertTrue(node in droputils.getDownstreamObjects(upNode))
def createGraphFromDropSpecList(dropSpecList):
logger.debug("Found %d DROP definitions", len(dropSpecList))
# Step #1: create the actual DROPs
drops = collections.OrderedDict()
logger.info("Creating %d drops", len(dropSpecList))
for n, dropSpec in enumerate(dropSpecList):
check_dropspec(n, dropSpec)
dropType = dropSpec['type']
cf = __CREATION_FUNCTIONS[dropType]
drop = cf(dropSpec)
drops[drop.oid] = drop
# Step #2: establish relationships
logger.info("Establishing relationships between drops")
for dropSpec in dropSpecList:
# 'oid' is mandatory
oid = dropSpec['oid']
drop = drops[oid]
for rel in dropSpec:
# 1-N relationships
if rel in __TOMANY:
link = __TOMANY[rel]
for oid in dropSpec[rel]:
lhDrop = drops[oid]
relFuncName = LINKTYPE_1TON_APPEND_METHOD[link]
try:
relFunc = getattr(drop, relFuncName)
except AttributeError:
logger.error(
'%r cannot be linked to %r due to missing method "%s"',
drop, lhDrop, relFuncName)
raise
relFunc(lhDrop)
# N-1 relationships
elif rel in __TOONE:
link = __TOONE[rel]
lhDrop = drops[dropSpec[rel]]
propName = LINKTYPE_NTO1_PROPERTY[link]
setattr(drop, propName, lhDrop)
# We're done! Return the roots of the graph to the caller
logger.info("Calculating graph roots")
roots = []
for drop in drops.values():
if not droputils.getUpstreamObjects(drop):
roots.append(drop)
logger.info("%d graph roots found, bye-bye!", len(roots))
return roots
def assertDownstream(self, node, downstreamNodes):
if not isinstance(downstreamNodes, list):
downstreamNodes = [downstreamNodes]
# Normal check
self.assertSetEqual(set(downstreamNodes),
set(droputils.getDownstreamObjects(node)))
# Check the other way too
for downNode in downstreamNodes:
self.assertTrue(node in droputils.getUpstreamObjects(downNode))
def createGraphFromDropSpecList(dropSpecList):
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Found %d DROP definitions" % (len(dropSpecList)))
# Step #1: create the actual DROPs
drops = collections.OrderedDict()
for dropSpec in dropSpecList:
# 'type' is mandatory
dropType = dropSpec['type']
cf = __CREATION_FUNCTIONS[dropType]
drop = cf(dropSpec)
drops[drop.oid] = drop
# Step #2: establish relationships
for dropSpec in dropSpecList:
# 'oid' is mandatory
oid = dropSpec['oid']
drop = drops[oid]
# 1-N relationships
for link,rel in __ONE_TO_N_RELS.viewitems():
if rel in dropSpec:
for oid in dropSpec[rel]:
lhDrop = drops[oid]
relFuncName = LINKTYPE_1TON_APPEND_METHOD[link]
try:
relFunc = getattr(drop, relFuncName)
except AttributeError:
logger.error('%r cannot be linked to %r due to missing method "%s"' % (drop, lhDrop, relFuncName))
raise
relFunc(lhDrop)
# N-1 relationships
for link,rel in __N_TO_ONE_RELS.viewitems():
if rel in dropSpec:
lhDrop = drops[dropSpec[rel]]
propName = LINKTYPE_NTO1_PROPERTY[link]
setattr(drop, propName, lhDrop)
# We're done! Return the roots of the graph to the caller
roots = []
for drop in drops.itervalues():
if not droputils.getUpstreamObjects(drop):
roots.append(drop)
return roots
