def _make_object_rings(self): ringbuilder = builder.RingBuilder(2, 3, 1) devs = [ 'r0z0-127.0.0.1:10000/sda1', 'r0z1-127.0.0.1:10001/sda1', 'r1z0-127.0.0.1:10002/sda1', 'r1z1-127.0.0.1:10003/sda1', ] for raw_dev_str in devs: dev = ring_utils.parse_add_value(raw_dev_str) dev['weight'] = 1.0 ringbuilder.add_dev(dev) ringbuilder.rebalance() ringbuilder.get_ring().save(self.tmpfile_name) ringbuilder = builder.RingBuilder(2, 2, 1) devs = [ 'r0z0-127.0.0.1:10000/sda1', 'r0z1-127.0.0.2:10004/sda1', ] for raw_dev_str in devs: dev = ring_utils.parse_add_value(raw_dev_str) dev['weight'] = 1.0 ringbuilder.add_dev(dev) ringbuilder.rebalance() ringbuilder.get_ring().save(self.tmpfile_name2)
def reconcile(self, devices): def format_device(d): return 'r%(region)sz%(zone)s-%(ip)s:%(port)s/%(device)s' % d changed = False # Add new devices for dev_string in devices: dev = parse_add_value(dev_string) if not len(self._builder.search_devs(dev)): changed = True dev['weight'] = 1 self._builder.add_dev(dev) self._logger.info("adding %s", dev_string) # Remove devices for dev in self._builder.search_devs([]): dev_string = format_device(dev) if dev_string not in devices: changed = True self._builder.remove_dev(dev['id']) self._logger.info("removing " + dev_string) if changed: self._builder.rebalance() else: self._logger.info("no change")
def _parse_add_command(round_index, command_index, command): if len(command) != 3: raise ParseCommandError( 'add command', round_index, command_index, 'expected array of length 3, but got %r' % command) dev_str = command[1] weight_str = command[2] try: dev = parse_add_value(dev_str) except ValueError as err: raise ParseCommandError('device specifier', round_index, command_index, err) dev['weight'] = _parse_weight(round_index, command_index, weight_str) if dev['region'] is None: dev['region'] = 1 default_key_map = { 'replication_ip': 'ip', 'replication_port': 'port', } for empty_key, default_key in default_key_map.items(): if dev[empty_key] is None: dev[empty_key] = dev[default_key] return ['add', dev]
def _parse_add_values(argvish): """ Parse devices to add as specified on the command line. Will exit on error and spew warnings. :returns: array of device dicts """ new_cmd_format, opts, args = validate_args(argvish) # We'll either parse the all-in-one-string format or the # --options format, # but not both. If both are specified, raise an error. parsed_devs = [] if len(args) > 0: if new_cmd_format or len(args) % 2 != 0: print(Commands.add.__doc__.strip()) exit(EXIT_ERROR) devs_and_weights = izip(islice(args, 0, len(args), 2), islice(args, 1, len(args), 2)) for devstr, weightstr in devs_and_weights: dev_dict = parse_add_value(devstr) if dev_dict['region'] is None: stderr.write('WARNING: No region specified for %s. ' 'Defaulting to region 1.\n' % devstr) dev_dict['region'] = 1 if dev_dict['replication_ip'] is None: dev_dict['replication_ip'] = dev_dict['ip'] if dev_dict['replication_port'] is None: dev_dict['replication_port'] = dev_dict['port'] weight = float(weightstr) if weight < 0: raise ValueError('Invalid weight value: %s' % devstr) dev_dict['weight'] = weight parsed_devs.append(dev_dict) else: parsed_devs.append(build_dev_from_opts(opts)) return parsed_devs
def _parse_add_command(round_index, command_index, command): if len(command) != 3: raise ValueError( "Invalid add command (round %d, command %d): expected array of " "length 3, but got %d" % (round_index, command_index, len(command))) dev_str = command[1] weight_str = command[2] try: dev = parse_add_value(dev_str) except ValueError as err: raise ValueError( "Invalid device specifier '%s' in add (round %d, command %d): %s" % (dev_str, round_index, command_index, err)) dev['weight'] = _parse_weight(round_index, command_index, weight_str) if dev['region'] is None: dev['region'] = 1 return ['add', dev]
def test_get_more_nodes_with_zero_weight_region(self): rb = ring.RingBuilder(8, 3, 1) devs = [ ring_utils.parse_add_value(v) for v in [ 'r1z1-127.0.0.1:6200/d1', 'r1z1-127.0.0.1:6201/d2', 'r1z1-127.0.0.1:6202/d3', 'r1z1-127.0.0.1:6203/d4', 'r1z2-127.0.0.2:6200/d1', 'r1z2-127.0.0.2:6201/d2', 'r1z2-127.0.0.2:6202/d3', 'r1z2-127.0.0.2:6203/d4', 'r2z1-127.0.1.1:6200/d1', 'r2z1-127.0.1.1:6201/d2', 'r2z1-127.0.1.1:6202/d3', 'r2z1-127.0.1.1:6203/d4', 'r2z2-127.0.1.2:6200/d1', 'r2z2-127.0.1.2:6201/d2', 'r2z2-127.0.1.2:6202/d3', 'r2z2-127.0.1.2:6203/d4', ] ] for dev in devs: if dev['region'] == 2: dev['weight'] = 0.0 else: dev['weight'] = 1.0 rb.add_dev(dev) rb.rebalance() rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') self.assertEqual(r.version, rb.version) class CountingRingTable(object): def __init__(self, table): self.table = table self.count = 0 def __iter__(self): self._iter = iter(self.table) return self def __next__(self): self.count += 1 return next(self._iter) # complete the api next = __next__ def __getitem__(self, key): return self.table[key] histogram = collections.defaultdict(int) for part in range(r.partition_count): counting_table = CountingRingTable(r._replica2part2dev_id) with mock.patch.object(r, '_replica2part2dev_id', counting_table): node_iter = r.get_more_nodes(part) next(node_iter) histogram[counting_table.count] += 1 # Don't let our summing muddy our histogram histogram = dict(histogram) # sanity self.assertEqual(1, r._num_regions) self.assertEqual(2, r._num_zones) self.assertEqual(256, r.partition_count) # We always do one loop (including the StopIteration) while getting # primaries, so every part should hit next() at least 5 times self.assertEqual(sum(histogram.get(x, 0) for x in range(5)), 0, histogram) # Most of the parts should find a handoff device in the next partition, # but because some of the primary devices may *also* be used for that # partition, that means 5, 6, or 7 calls to next(). self.assertGreater(sum(histogram.get(x, 0) for x in range(8)), 160, histogram) # Want 90% confidence that it'll happen within two partitions self.assertGreater(sum(histogram.get(x, 0) for x in range(12)), 230, histogram) # Tail should fall off fairly quickly self.assertLess(sum(histogram.get(x, 0) for x in range(20, 100)), 5, histogram) # Hard limit at 50 (we've seen as bad as 41, 45) self.assertEqual(sum(histogram.get(x, 0) for x in range(50, 100)), 0, histogram)
def test_get_more_nodes_with_zero_weight_region(self): rb = ring.RingBuilder(8, 3, 1) devs = [ ring_utils.parse_add_value(v) for v in [ 'r1z1-127.0.0.1:6200/d1', 'r1z1-127.0.0.1:6201/d2', 'r1z1-127.0.0.1:6202/d3', 'r1z1-127.0.0.1:6203/d4', 'r1z2-127.0.0.2:6200/d1', 'r1z2-127.0.0.2:6201/d2', 'r1z2-127.0.0.2:6202/d3', 'r1z2-127.0.0.2:6203/d4', 'r2z1-127.0.1.1:6200/d1', 'r2z1-127.0.1.1:6201/d2', 'r2z1-127.0.1.1:6202/d3', 'r2z1-127.0.1.1:6203/d4', 'r2z2-127.0.1.2:6200/d1', 'r2z2-127.0.1.2:6201/d2', 'r2z2-127.0.1.2:6202/d3', 'r2z2-127.0.1.2:6203/d4', ] ] for dev in devs: if dev['region'] == 2: dev['weight'] = 0.0 else: dev['weight'] = 1.0 rb.add_dev(dev) rb.rebalance(seed=1) rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') class CountingRingTable(object): def __init__(self, table): self.table = table self.count = 0 def __iter__(self): self._iter = iter(self.table) return self def __next__(self): self.count += 1 return next(self._iter) # complete the api next = __next__ def __getitem__(self, key): return self.table[key] counting_table = CountingRingTable(r._replica2part2dev_id) r._replica2part2dev_id = counting_table part = random.randint(0, r.partition_count) node_iter = r.get_more_nodes(part) next(node_iter) self.assertEqual(5, counting_table.count) # sanity self.assertEqual(1, r._num_regions) self.assertEqual(2, r._num_zones)
def test_get_more_nodes_with_zero_weight_region(self): rb = ring.RingBuilder(8, 3, 1) devs = [ ring_utils.parse_add_value(v) for v in [ 'r1z1-127.0.0.1:6200/d1', 'r1z1-127.0.0.1:6201/d2', 'r1z1-127.0.0.1:6202/d3', 'r1z1-127.0.0.1:6203/d4', 'r1z2-127.0.0.2:6200/d1', 'r1z2-127.0.0.2:6201/d2', 'r1z2-127.0.0.2:6202/d3', 'r1z2-127.0.0.2:6203/d4', 'r2z1-127.0.1.1:6200/d1', 'r2z1-127.0.1.1:6201/d2', 'r2z1-127.0.1.1:6202/d3', 'r2z1-127.0.1.1:6203/d4', 'r2z2-127.0.1.2:6200/d1', 'r2z2-127.0.1.2:6201/d2', 'r2z2-127.0.1.2:6202/d3', 'r2z2-127.0.1.2:6203/d4', ] ] for dev in devs: if dev['region'] == 2: dev['weight'] = 0.0 else: dev['weight'] = 1.0 rb.add_dev(dev) rb.rebalance() rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') class CountingRingTable(object): def __init__(self, table): self.table = table self.count = 0 def __iter__(self): self._iter = iter(self.table) return self def __next__(self): self.count += 1 return next(self._iter) # complete the api next = __next__ def __getitem__(self, key): return self.table[key] histogram = collections.defaultdict(int) for part in range(r.partition_count): counting_table = CountingRingTable(r._replica2part2dev_id) with mock.patch.object(r, '_replica2part2dev_id', counting_table): node_iter = r.get_more_nodes(part) next(node_iter) histogram[counting_table.count] += 1 # Don't let our summing muddy our histogram histogram = dict(histogram) # sanity self.assertEqual(1, r._num_regions) self.assertEqual(2, r._num_zones) self.assertEqual(256, r.partition_count) # We always do one loop (including the StopIteration) while getting # primaries, so every part should hit next() at least 5 times self.assertEqual(sum(histogram.get(x, 0) for x in range(5)), 0, histogram) # Most of the parts should find a handoff device in the next partition, # but because some of the primary devices may *also* be used for that # partition, that means 5, 6, or 7 calls to next(). self.assertGreater(sum(histogram.get(x, 0) for x in range(8)), 160, histogram) # Want 90% confidence that it'll happen within two partitions self.assertGreater(sum(histogram.get(x, 0) for x in range(12)), 230, histogram) # Tail should fall off fairly quickly self.assertLess(sum(histogram.get(x, 0) for x in range(20, 100)), 5, histogram) # Hard limit at 50 (we've seen as bad as 41, 45) self.assertEqual(sum(histogram.get(x, 0) for x in range(50, 100)), 0, histogram)
def test_get_more_nodes_with_zero_weight_region(self): rb = ring.RingBuilder(8, 3, 1) devs = [ ring_utils.parse_add_value(v) for v in [ 'r1z1-127.0.0.1:6000/d1', 'r1z1-127.0.0.1:6001/d2', 'r1z1-127.0.0.1:6002/d3', 'r1z1-127.0.0.1:6003/d4', 'r1z2-127.0.0.2:6000/d1', 'r1z2-127.0.0.2:6001/d2', 'r1z2-127.0.0.2:6002/d3', 'r1z2-127.0.0.2:6003/d4', 'r2z1-127.0.1.1:6000/d1', 'r2z1-127.0.1.1:6001/d2', 'r2z1-127.0.1.1:6002/d3', 'r2z1-127.0.1.1:6003/d4', 'r2z2-127.0.1.2:6000/d1', 'r2z2-127.0.1.2:6001/d2', 'r2z2-127.0.1.2:6002/d3', 'r2z2-127.0.1.2:6003/d4', ] ] for dev in devs: if dev['region'] == 2: dev['weight'] = 0.0 else: dev['weight'] = 1.0 rb.add_dev(dev) rb.rebalance(seed=1) rb.get_ring().save(self.testgz) r = ring.Ring(self.testdir, ring_name='whatever') class CountingRingTable(object): def __init__(self, table): self.table = table self.count = 0 def __iter__(self): self._iter = iter(self.table) return self def __next__(self): self.count += 1 return next(self._iter) # complete the api next = __next__ def __getitem__(self, key): return self.table[key] counting_table = CountingRingTable(r._replica2part2dev_id) r._replica2part2dev_id = counting_table part = random.randint(0, r.partition_count) node_iter = r.get_more_nodes(part) next(node_iter) self.assertEqual(5, counting_table.count) # sanity self.assertEqual(1, r._num_regions) self.assertEqual(2, r._num_zones)