def addGraphSpec(self, sessionId, graphSpec): # The first step is to break down the graph into smaller graphs that # belong to the same host, so we can submit that graph into the individual # DMs. For this we need to make sure that our graph has a the correct # attribute set logger.info('Separating graph') perPartition = collections.defaultdict(list) for dropSpec in graphSpec: if self._partitionAttr not in dropSpec: msg = "Drop %s doesn't specify a %s attribute" % ( dropSpec['oid'], self._partitionAttr) raise InvalidGraphException(msg) partition = dropSpec[self._partitionAttr] if partition not in self._dmHosts: msg = "Drop %s's %s %s does not belong to this DM" % ( dropSpec['oid'], self._partitionAttr, partition) raise InvalidGraphException(msg) perPartition[partition].append(dropSpec) # Add the drop specs to our graph self._graph[uid_for_drop(dropSpec)] = dropSpec # At each partition the relationships between DROPs should be local at the # moment of submitting the graph; thus we record the inter-partition # relationships separately and remove them from the original graph spec inter_partition_rels = [] for dropSpecs in perPartition.values(): inter_partition_rels += graph_loader.removeUnmetRelationships( dropSpecs) sanitize_relations(inter_partition_rels, self._graph) logger.info('Removed (and sanitized) %d inter-dm relationships', len(inter_partition_rels)) # Store the inter-partition relationships; later on they have to be # communicated to the NMs so they can establish them as needed. drop_rels = collections.defaultdict( functools.partial(collections.defaultdict, list)) for rel in inter_partition_rels: rhn = self._graph[rel.rhs]['node'] lhn = self._graph[rel.lhs]['node'] drop_rels[lhn][rhn].append(rel) drop_rels[rhn][lhn].append(rel) self._drop_rels[sessionId] = drop_rels logger.debug("Calculated NM-level drop relationships: %r", drop_rels) # Create the individual graphs on each DM now that they are correctly # separated. logger.info('Adding individual graphSpec of session %s to each DM', sessionId) self.replicate(sessionId, self._addGraphSpec, "appending graphSpec to individual DMs", iterable=perPartition.items()) logger.info( 'Successfully added individual graphSpec of session %s to each DM', sessionId)
def test_removeUnmetRelationships(self): # Unmet relationsips are # DROPRel(D, CONSUMER, A) # DROPRel(D, STREAMING_CONSUMER, C) # DROPRel(Z, PRODUCER, A) # DROPRel(X, PRODUCER, A) graphDesc = [{'oid':'A', 'consumers':['B', 'D'], 'producers':['Z','X']}, {'oid':'B', 'outputs':['C']}, {'oid':'C', 'streamingConsumers':['D']}] unmetRelationships = graph_loader.removeUnmetRelationships(graphDesc) self.assertEquals(4, len(unmetRelationships)) self.assertIn(DROPRel('D', DROPLinkType.CONSUMER, 'A'), unmetRelationships) self.assertIn(DROPRel('D', DROPLinkType.STREAMING_CONSUMER, 'C'), unmetRelationships) self.assertIn(DROPRel('Z', DROPLinkType.PRODUCER, 'A'), unmetRelationships) self.assertIn(DROPRel('X', DROPLinkType.PRODUCER, 'A'), unmetRelationships) # The original dropSpecs have changed as well a = graphDesc[0] c = graphDesc[2] self.assertEquals(1, len(a['consumers'])) self.assertEquals('B', a['consumers'][0]) self.assertFalse('producers' in a) self.assertFalse('streamingConsumers' in c)
def test_removeUnmetRelationships(self): # Unmet relationsips are # DROPRel(D, CONSUMER, A) # DROPRel(D, STREAMING_CONSUMER, C) # DROPRel(Z, PRODUCER, A) # DROPRel(X, PRODUCER, A) graphDesc = [{ 'oid': 'A', 'consumers': ['B', 'D'], 'producers': ['Z', 'X'] }, { 'oid': 'B', 'outputs': ['C'] }, { 'oid': 'C', 'streamingConsumers': ['D'] }] unmetRelationships = graph_loader.removeUnmetRelationships(graphDesc) self.assertEqual(4, len(unmetRelationships)) self.assertIn(DROPRel('D', DROPLinkType.CONSUMER, 'A'), unmetRelationships) self.assertIn(DROPRel('D', DROPLinkType.STREAMING_CONSUMER, 'C'), unmetRelationships) self.assertIn(DROPRel('Z', DROPLinkType.PRODUCER, 'A'), unmetRelationships) self.assertIn(DROPRel('X', DROPLinkType.PRODUCER, 'A'), unmetRelationships) # The original dropSpecs have changed as well a = graphDesc[0] c = graphDesc[2] self.assertEqual(1, len(a['consumers'])) self.assertEqual('B', a['consumers'][0]) self.assertFalse('producers' in a) self.assertFalse('streamingConsumers' in c)
def addGraphSpec(self, sessionId, graphSpec): logger.debug('addGraphSpec - sessionId: {0}'.format(sessionId)) # TODO: KV remove # The first step is to break down the graph into smaller graphs that # belong to the same host, so we can submit that graph into the individual # DMs. For this we need to make sure that our graph has a the correct # attribute set perPartition = collections.defaultdict(list) for dropSpec in graphSpec: if self._partitionAttr not in dropSpec: raise Exception("DROP %s doesn't specify a %s attribute" % (dropSpec['oid'], self._partitionAttr)) partition = dropSpec[self._partitionAttr] if partition not in self._dmHosts: raise Exception("DROP %s's %s %s does not belong to this DM" % (dropSpec['oid'], self._partitionAttr, partition)) perPartition[partition].append(dropSpec) # At each partition the relationships between DROPs should be local at the # moment of submitting the graph; thus we record the inter-DM # relationships separately and remove them from the original graph spec logger.debug('addGraphSpec - sessionId: {0}'.format(sessionId)) # TODO: KV remove interDMRelations = [] for dropSpecs in perPartition.viewvalues(): interDMRelations.extend(graph_loader.removeUnmetRelationships(dropSpecs)) # Create the individual graphs on each DM now that they are correctly # separated. if logger.isEnabledFor(logging.INFO): logger.info('Adding individual graphSpec of session %s to each DM' % (sessionId)) thrExs = {} self._tp.map(functools.partial(self._addGraphSpec, sessionId, thrExs), [(host, perPartition[host]) for host in self._dmHosts]) if thrExs: raise Exception("One or more errors occurred while adding the graphSpec to the individual DMs", thrExs) self._interDMRelations[sessionId].extend(interDMRelations)