def addGraphSpec(self, sessionId, graphSpec):
# The first step is to break down the graph into smaller graphs that
# belong to the same host, so we can submit that graph into the individual
# DMs. For this we need to make sure that our graph has a the correct
# attribute set
logger.info('Separating graph')
perPartition = collections.defaultdict(list)
for dropSpec in graphSpec:
if self._partitionAttr not in dropSpec:
msg = "Drop %s doesn't specify a %s attribute" % (
dropSpec['oid'], self._partitionAttr)
raise InvalidGraphException(msg)
partition = dropSpec[self._partitionAttr]
if partition not in self._dmHosts:
msg = "Drop %s's %s %s does not belong to this DM" % (
dropSpec['oid'], self._partitionAttr, partition)
raise InvalidGraphException(msg)
perPartition[partition].append(dropSpec)
# Add the drop specs to our graph
self._graph[uid_for_drop(dropSpec)] = dropSpec
# At each partition the relationships between DROPs should be local at the
# moment of submitting the graph; thus we record the inter-partition
# relationships separately and remove them from the original graph spec
inter_partition_rels = []
for dropSpecs in perPartition.values():
inter_partition_rels += graph_loader.removeUnmetRelationships(
dropSpecs)
sanitize_relations(inter_partition_rels, self._graph)
logger.info('Removed (and sanitized) %d inter-dm relationships',
len(inter_partition_rels))
# Store the inter-partition relationships; later on they have to be
# communicated to the NMs so they can establish them as needed.
drop_rels = collections.defaultdict(
functools.partial(collections.defaultdict, list))
for rel in inter_partition_rels:
rhn = self._graph[rel.rhs]['node']
lhn = self._graph[rel.lhs]['node']
drop_rels[lhn][rhn].append(rel)
drop_rels[rhn][lhn].append(rel)
self._drop_rels[sessionId] = drop_rels
logger.debug("Calculated NM-level drop relationships: %r", drop_rels)
# Create the individual graphs on each DM now that they are correctly
# separated.
logger.info('Adding individual graphSpec of session %s to each DM',
sessionId)
self.replicate(sessionId,
self._addGraphSpec,
"appending graphSpec to individual DMs",
iterable=perPartition.items())
logger.info(
'Successfully added individual graphSpec of session %s to each DM',
sessionId)
def test_removeUnmetRelationships(self):
# Unmet relationsips are
# DROPRel(D, CONSUMER, A)
# DROPRel(D, STREAMING_CONSUMER, C)
# DROPRel(Z, PRODUCER, A)
# DROPRel(X, PRODUCER, A)
graphDesc = [{'oid':'A', 'consumers':['B', 'D'], 'producers':['Z','X']},
{'oid':'B', 'outputs':['C']},
{'oid':'C', 'streamingConsumers':['D']}]
unmetRelationships = graph_loader.removeUnmetRelationships(graphDesc)
self.assertEquals(4, len(unmetRelationships))
self.assertIn(DROPRel('D', DROPLinkType.CONSUMER, 'A'), unmetRelationships)
self.assertIn(DROPRel('D', DROPLinkType.STREAMING_CONSUMER, 'C'), unmetRelationships)
self.assertIn(DROPRel('Z', DROPLinkType.PRODUCER, 'A'), unmetRelationships)
self.assertIn(DROPRel('X', DROPLinkType.PRODUCER, 'A'), unmetRelationships)
# The original dropSpecs have changed as well
a = graphDesc[0]
c = graphDesc[2]
self.assertEquals(1, len(a['consumers']))
self.assertEquals('B', a['consumers'][0])
self.assertFalse('producers' in a)
self.assertFalse('streamingConsumers' in c)
def test_removeUnmetRelationships(self):
# Unmet relationsips are
# DROPRel(D, CONSUMER, A)
# DROPRel(D, STREAMING_CONSUMER, C)
# DROPRel(Z, PRODUCER, A)
# DROPRel(X, PRODUCER, A)
graphDesc = [{
'oid': 'A',
'consumers': ['B', 'D'],
'producers': ['Z', 'X']
}, {
'oid': 'B',
'outputs': ['C']
}, {
'oid': 'C',
'streamingConsumers': ['D']
}]
unmetRelationships = graph_loader.removeUnmetRelationships(graphDesc)
self.assertEqual(4, len(unmetRelationships))
self.assertIn(DROPRel('D', DROPLinkType.CONSUMER, 'A'),
unmetRelationships)
self.assertIn(DROPRel('D', DROPLinkType.STREAMING_CONSUMER, 'C'),
unmetRelationships)
self.assertIn(DROPRel('Z', DROPLinkType.PRODUCER, 'A'),
unmetRelationships)
self.assertIn(DROPRel('X', DROPLinkType.PRODUCER, 'A'),
unmetRelationships)
# The original dropSpecs have changed as well
a = graphDesc[0]
c = graphDesc[2]
self.assertEqual(1, len(a['consumers']))
self.assertEqual('B', a['consumers'][0])
self.assertFalse('producers' in a)
self.assertFalse('streamingConsumers' in c)
def addGraphSpec(self, sessionId, graphSpec):
logger.debug('addGraphSpec - sessionId: {0}'.format(sessionId)) # TODO: KV remove
# The first step is to break down the graph into smaller graphs that
# belong to the same host, so we can submit that graph into the individual
# DMs. For this we need to make sure that our graph has a the correct
# attribute set
perPartition = collections.defaultdict(list)
for dropSpec in graphSpec:
if self._partitionAttr not in dropSpec:
raise Exception("DROP %s doesn't specify a %s attribute" % (dropSpec['oid'], self._partitionAttr))
partition = dropSpec[self._partitionAttr]
if partition not in self._dmHosts:
raise Exception("DROP %s's %s %s does not belong to this DM" % (dropSpec['oid'], self._partitionAttr, partition))
perPartition[partition].append(dropSpec)
# At each partition the relationships between DROPs should be local at the
# moment of submitting the graph; thus we record the inter-DM
# relationships separately and remove them from the original graph spec
logger.debug('addGraphSpec - sessionId: {0}'.format(sessionId)) # TODO: KV remove
interDMRelations = []
for dropSpecs in perPartition.viewvalues():
interDMRelations.extend(graph_loader.removeUnmetRelationships(dropSpecs))
# Create the individual graphs on each DM now that they are correctly
# separated.
if logger.isEnabledFor(logging.INFO):
logger.info('Adding individual graphSpec of session %s to each DM' % (sessionId))
thrExs = {}
self._tp.map(functools.partial(self._addGraphSpec, sessionId, thrExs), [(host, perPartition[host]) for host in self._dmHosts])
if thrExs:
raise Exception("One or more errors occurred while adding the graphSpec to the individual DMs", thrExs)
self._interDMRelations[sessionId].extend(interDMRelations)
