def __init__(self, db_connection, build_graphs=True): self.io = StupidIO() self.id_attributes = {'id':(STATIC_ATTRIBUTE, int), 'addr':(STATIC_ATTRIBUTE, str)} self.node_env = {} self.db_connection = db_connection self.build_graphs = build_graphs self.id_only_view_spec = StatementParser.parse(''' CREATE VIEW id_only AS SELECT node.id FROM parent FOREACH BUCKET ; ''')
class HarvesterTest(object): def __init__(self, db_connection, build_graphs=True): self.io = StupidIO() self.id_attributes = {'id':(STATIC_ATTRIBUTE, int), 'addr':(STATIC_ATTRIBUTE, str)} self.node_env = {} self.db_connection = db_connection self.build_graphs = build_graphs self.id_only_view_spec = StatementParser.parse(''' CREATE VIEW id_only AS SELECT node.id FROM parent FOREACH BUCKET ; ''') @staticmethod def build_id_only_node_name(node): return "node[id=%r]" % node.id @staticmethod def _unpack_list(node_tuples): return [ item[1] for item in node_tuples ] def _create_nodes(self, count): node_identifier = ('addr',) connection = self.db_connection attribute_types = self.id_attributes create_node = self._create_node def build_node_filter(l_id): return (lambda node: node.id != l_id) deferred = defer.DeferredList( [ create_node(local_id, connection, attribute_types=attribute_types, node_filter=build_node_filter(local_id), node_identifier_attributes=node_identifier ) for local_id in xrange(count) ] ) deferred.addCallback(self._unpack_list) return deferred def _create_node(self, id, *args, **kwargs): db_name = 'db%010d' % id db = SQLNodeDB(db_name, *args, **kwargs) addr = '127.0.0.1:%05d' % (id%65536) self.node_env[addr] = { 'db':db, 'vr':ViewRegistry(db), 'io':self.io.create_node() } deferred = db.connect() def create_local_node(_): return db.create_local_node(id=id, addr=addr) deferred.addCallback(create_local_node) return deferred def run_circle(self, node_count): deferred = self._create_nodes(node_count) deferred.addCallback(self.run_circle_with_nodes) def run_circle_with_nodes(self, nodes): max_id = len(nodes) - 1 n_count=6 view_def = Template(''' CREATE VIEW circle_neighbours AS SELECT node.id, node.addr RANKED lowest(n_count, min(abs(local.id - node.id), $max_id - abs(local.id - node.id))) FROM db WITH local, n_count ; ''').substitute(max_id=max_id) # FIXME: twisted ... view_spec = StatementParser.parse(view_def) for node in nodes: node_env = self.node_env[node.addr] spec = ViewSpecification(view_spec, new_name='circle_neighbours_%010x' % node.id, new_parents=(node_env['db'].name,)) view = NodeView(spec, node_env['vr'], local=node, n_count=n_count) node_env['spec'] = spec node_env['view'] = view for node in nodes: node_env = self.node_env[node.addr] harvester = BidirectionalGossipHarvester( node_env['io'], node, node_env['db'], node_env['vr'], SQLNodeView, (node_env['spec'], node_env['view']) ) node_env['harvester'] = harvester def is_correct(node): distances = sum( min(abs(nid - view_node.id), max_id - abs(nid - view_node.id)) for view_node in node._view ) return distances <= (n_count ** 2) / 2 # FIXME ... def is_complete(node): return len(self.node_env[node.addr]['view']) < n_count self.run_harvesting(nodes) def run_chord(self, node_count): deferred = self._create_nodes(node_count) deferred.addCallback(self.run_chord_with_nodes) return deferred def run_chord_with_nodes(self, nodes): log_nodes = int(math.log(len(nodes), 2)) max_id = len(nodes) - 1 n_count=6 def chord_dist(n1, n2): dist = n2 - n1 if dist < 0: dist += max_id + 1 return dist sql_chord_dist = """ CREATE FUNCTION %%s(integer, integer) RETURNS integer AS ' SELECT CASE WHEN ($2-$1) < 0 THEN %s + ($2-$1) ELSE ($2-$1) END ' LANGUAGE SQL STRICT; """ % (max_id+1) for node_env in self.node_env.itervalues(): node_db = node_env['db'] node_db.create_function('dist', sql_chord_dist) func_def = {'dist' : chord_dist} view_def = Template(''' CREATE VIEW circle_neighbours AS SELECT node.id, node.addr RANKED highest(i+1, dist(local.id, node.id)) FROM db WITH local HAVING dist(local.id, node.id) in (2^i : 2^(i+1)) FOREACH i IN (0:$log_nodes) ; ''').substitute(log_nodes=log_nodes) view_spec = StatementParser.parse(view_def) deferreds = [] for node in nodes: node_env = self.node_env[node.addr] node_db = node_env['db'] view_reg = node_env['vr'] spec = ViewSpecification(view_spec, func_def, new_name='circle_neighbours_%010x' % node.id, new_parents=(node_db.name,)) node_env['spec'] = spec deferred = node_db.create_view(spec, view_reg) deferreds.append(deferred) def is_correct(node): node_env = self.node_env[node.addr] view = node_env['view'] deferred = view.iterBuckets() nid = node.id def bucket_test(buckets): for (i,), bucket in buckets: if len(bucket) == 0: return False id_range = xrange(2**i, 2**(i+1)) for neighbour in bucket: if chord_dist(nid, neighbour.id) not in id_range: return False return True deferred.addCallback(bucket_test) return deferred def is_complete(node): node_env = self.node_env[node.addr] view = node_env['view'] deferred = view.iterBuckets() def bucket_test(buckets): bucket_count = 0 for (i,), bucket in buckets: bucket_count += 1 if len(bucket) < i: return False return bucket_count > 0 deferred.addCallback(bucket_test) return deferred deferred_list = defer.DeferredList(deferreds) deferred_list.addCallback(self._unpack_list) deferred_list.addCallback(self.build_harvesters, nodes, is_correct, is_complete) return deferred_list def build_harvesters(self, views, nodes, is_correct, is_complete): for view, node in zip(views, nodes): node_env = self.node_env[node.addr] node_db = node_env['db'] view_reg = node_env['vr'] harvester = BidirectionalGossipHarvester( node_env['io'], node, node_db, view_reg, SQLNodeView, (node_env['spec'], view) ) node_env['harvester'] = harvester node_env['view'] = view if self.build_graphs: deferreds = [] for node in nodes: node_env = self.node_env[node.addr] node_db = node_env['db'] spec = ViewSpecification(self.id_only_view_spec, new_name="idov%08x" % node.id, new_parents=(node_env['view'].name,) ) deferred = node_db.create_view(spec, node_env['vr']) def set_id_only_view(view, node_env): node_env['id_only_view'] = view deferred.addCallback(set_id_only_view, node_env) deferreds.append(deferred) deferred_list = defer.DeferredList(deferreds) build_id_only_node_name = self.build_id_only_node_name def build_graph(_): return view_snapshot.ViewCircleSnapshot( [ (build_id_only_node_name(node), (self.node_env[node.addr]['id_only_view'],)) for node in nodes] ) deferred_list.addCallback(build_graph) else: deferred_list = defer.succeed(None) deferred_list.addCallback( self.run_harvesting, nodes, is_correct, is_complete) def rebuild_graph(self, _, graph, node, node_env): return graph.rebuild_edges_from_views( self.build_id_only_node_name(node), (node_env['id_only_view'],) ) def build_image(self, _, graph, image_counter): image = graph.snapshot().resize( (700,700), Image.BICUBIC ) image.save('img/test%010d.gif' % image_counter, 'GIF') def run_harvesting(self, graph, nodes, is_correct, is_complete): self.image_counter = 0 choose1 = random.choice def node_action(node): deferred = is_complete(node) node_env = self.node_env[node.addr] view = node_env['view'] harvester = node_env['harvester'] def find_source(complete): if complete: deferred = view.iterNodes() deferred.addCallback(tuple) else: deferred = defer.succeed(nodes) return deferred deferred.addCallback(find_source) def send_specs(nodes): recipient = choose1(nodes) while recipient is node: recipient = choose1(nodes) recipient_env = self.node_env[recipient.addr] deferred = harvester.send_specs(recipient_env['io']) if self.build_graphs: deferred.addCallback(self.rebuild_graph, graph, node, node_env) deferred.addCallback(self.rebuild_graph, graph, recipient, recipient_env) deferred.addCallback(self.build_image, graph, self.image_counter) self.image_counter += 1 return deferred deferred.addCallback(send_specs) return deferred def run_iteration(_): dlist = defer.DeferredList([ node_action(node) for node in nodes ]) dlist.addCallback(run_iteration) return dlist return run_iteration(None)