def test_reconcile_remove_device(self): # given a ring builder with two devices b = RingBuilder(10, 1, 1) r = RingController(b, "object") ds = ["r1z1-192.168.0.2:6000/d3", "r1z2-192.168.2.2:5000/d1"] r.reconcile(ds) # when device is removed ds = ["r1z2-192.168.2.2:5000/d1"] r.reconcile(ds) # then only one device is present devs = b.search_devs([]) self.assertEqual(1, len(devs)) self.assertEqual(devs, [{ 'device': 'd1', 'id': 1, 'ip': '192.168.2.2', 'meta': '', 'parts': 1024, 'parts_wanted': 0, 'port': 5000, 'region': 1, 'replication_ip': None, 'replication_port': None, 'weight': 1.0, 'zone': 2 }])
def create(): """ swift-ring-builder <builder_file> create <part_power> <replicas> <min_part_hours> Creates <builder_file> with 2^<part_power> partitions and <replicas>. <min_part_hours> is number of hours to restrict moving a partition more than once. """ if len(argv) < 6: print(Commands.create.__doc__.strip()) exit(EXIT_ERROR) # 1、生成RingBuilder对象实例 builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5])) # 2、创建备份目录 backup_dir = pathjoin(dirname(builder_file), 'backups') try: mkdir(backup_dir) except OSError as err: if err.errno != EEXIST: raise # 3、保存原始数据到备份目录,以及/etc/swift目录中 builder.save(pathjoin(backup_dir, '%d.' % time() + basename(builder_file))) builder.save(builder_file) exit(EXIT_SUCCESS)
def test_composite_same_device_in_the_different_rings_error(self): builders = self.create_sample_ringbuilders(2) same_device = copy.deepcopy(builders[0].devs[0]) # create one more ring which duplicates a device in the first ring builder = RingBuilder(6, 3, 1) _, fname = tempfile.mkstemp(dir=self.tmpdir) # add info to feed to add_dev same_device.update({'region': 2, 'weight': 100}) builder.add_dev(same_device) # add rest of the devices, which are unique for _ in range(3): dev = self.pop_region_device(2) builder.add_dev(dev) builder.rebalance() builder.save(fname) # sanity self.assertTrue(os.path.exists(fname)) builders.append(builder) with self.assertRaises(ValueError) as cm: compose_rings(builders) self.assertIn( 'Duplicate ip/port/device combination %(ip)s/%(port)s/%(device)s ' 'found in builders at indexes 0 and 2' % same_device, cm.exception.message)
def storage_policies(request): """ Creates a storage policy to swift with an specific ring. Allows create replication storage policies and erasure code storage policies """ if request.method == "GET": try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) keys = r.keys("storage-policy:*") storage_policy_list = [] for key in keys: storage_policy = r.hgetall(key) to_json_bools(storage_policy, 'deprecated', 'default', 'deployed') storage_policy['id'] = str(key).split(':')[-1] storage_policy['devices'] = json.loads(storage_policy['devices']) storage_policy_list.append(storage_policy) return JSONResponse(storage_policy_list, status=status.HTTP_200_OK) if request.method == "POST": try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) data = JSONParser().parse(request) if data['policy_type'] == 'EC': data['replicas'] = int(data['ec_num_data_fragments']) + int( data['ec_num_parity_fragments']) try: sp_id = str(r.incr('storage-policies:id')) key = 'storage-policy:' + sp_id ring = RingBuilder(int(data['partition_power']), int(data['replicas']), int(data['time'])) ring.save(get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, sp_id)) r.hmset(key, data) except: return JSONResponse('Error creating the Storage Policy', status=status.HTTP_500_INTERNAL_SERVER_ERROR) return JSONResponse('Account created successfully', status=status.HTTP_201_CREATED) return JSONResponse('Only HTTP POST requests allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def test_load(self): rb = ring.RingBuilder(8, 3, 1) devs = [{'id': 0, 'region': 0, 'zone': 0, 'weight': 1, 'ip': '127.0.0.0', 'port': 10000, 'device': 'sda1', 'meta': 'meta0'}, {'id': 1, 'region': 0, 'zone': 1, 'weight': 1, 'ip': '127.0.0.1', 'port': 10001, 'device': 'sdb1', 'meta': 'meta1'}, {'id': 2, 'region': 0, 'zone': 2, 'weight': 2, 'ip': '127.0.0.2', 'port': 10002, 'device': 'sdc1', 'meta': 'meta2'}, {'id': 3, 'region': 0, 'zone': 3, 'weight': 2, 'ip': '127.0.0.3', 'port': 10003, 'device': 'sdd1'}] for d in devs: rb.add_dev(d) rb.rebalance() real_pickle = pickle.load try: #test a legit builder fake_pickle = Mock(return_value=rb) fake_open = Mock(return_value=None) pickle.load = fake_pickle builder = RingBuilder.load('fake.builder', open=fake_open) self.assertEquals(fake_pickle.call_count, 1) fake_open.assert_has_calls([mock_call('fake.builder', 'rb')]) self.assertEquals(builder, rb) fake_pickle.reset_mock() fake_open.reset_mock() #test old style builder fake_pickle.return_value = rb.to_dict() pickle.load = fake_pickle builder = RingBuilder.load('fake.builder', open=fake_open) fake_open.assert_has_calls([mock_call('fake.builder', 'rb')]) self.assertEquals(builder.devs, rb.devs) fake_pickle.reset_mock() fake_open.reset_mock() #test old devs but no meta no_meta_builder = rb for dev in no_meta_builder.devs: del(dev['meta']) fake_pickle.return_value = no_meta_builder pickle.load = fake_pickle builder = RingBuilder.load('fake.builder', open=fake_open) fake_open.assert_has_calls([mock_call('fake.builder', 'rb')]) self.assertEquals(builder.devs, rb.devs) fake_pickle.reset_mock() finally: pickle.load = real_pickle
def write_stub_builder(tmpdir, region=1, name=''): """ Pretty much just a three node, three replica, 8 part power builder... :param tmpdir: a place to write the builder, be sure to clean it up! :param region: an integer, fills in region and ip :param name: the name of the builder (i.e. <name>.builder) """ name = name or str(region) replicas = 3 builder = RingBuilder(8, replicas, 1) for i in range(replicas): dev = { 'weight': 100, 'region': '%d' % region, 'zone': '1', 'ip': '10.0.0.%d' % region, 'port': '3600', 'device': 'sdb%d' % i } builder.add_dev(dev) builder.rebalance() builder_file = os.path.join(tmpdir, '%s.builder' % name) builder.save(builder_file) return builder, builder_file
def test_different_part_power_error(self): # create a ring builder # (default, part power is 6 with create_sample_ringbuilders) builders = self.create_sample_ringbuilders(1) # prepare another ring which has different part power incorrect_builder = RingBuilder(4, 3, 1) _, fname = tempfile.mkstemp(dir=self.tmpdir) for _ in range(4): dev = self.pop_region_device(1) incorrect_builder.add_dev(dev) incorrect_builder.rebalance() incorrect_builder.save(fname) # sanity self.assertTrue(os.path.exists(fname)) # sanity correct_builder = builders[0] self.assertNotEqual(correct_builder.part_shift, incorrect_builder.part_shift) self.assertNotEqual(correct_builder.part_power, incorrect_builder.part_power) builders.append(incorrect_builder) with self.assertRaises(ValueError) as cm: compose_rings(builders) self.assertIn("All builders must have same value for 'part_power'", cm.exception.message)
def test_compose_rings_float_replica_count_builder_error(self): builders = self.create_sample_ringbuilders(1) # prepare another ring which has float replica count incorrect_builder = RingBuilder(6, 1.5, 1) _, fname = tempfile.mkstemp(dir=self.tmpdir) for _ in range(4): dev = self.pop_region_device(1) incorrect_builder.add_dev(dev) incorrect_builder.rebalance() incorrect_builder.save(fname) # sanity self.assertTrue(os.path.exists(fname)) self.assertEqual(1.5, incorrect_builder.replicas) # the first replica has 2 ** 6 partitions self.assertEqual( 2 ** 6, len(incorrect_builder._replica2part2dev[0])) # but the second replica has the half of the first partitions self.assertEqual( 2 ** 5, len(incorrect_builder._replica2part2dev[1])) builders.append(incorrect_builder) with self.assertRaises(ValueError) as cm: compose_rings(builders) self.assertIn("Problem with builders", cm.exception.message) self.assertIn("Non integer replica count", cm.exception.message)
def create(): """ swift-ring-builder <builder_file> create <part_power> <replicas> <min_part_hours> Creates <builder_file> with 2^<part_power> partitions and <replicas>. <min_part_hours> is number of hours to restrict moving a partition more than once. OR swift-ring-builder <builder_file> create <part_power> <replicas> <min_part_hours> <policy_info_file> policy_info_file contains the information about the policy """ if len(argv) < 6: print Commands.create.__doc__.strip() exit(EXIT_ERROR) if len(argv) > 6: builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]), argv[6]) else: builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5])) backup_dir = pathjoin(dirname(argv[1]), 'backups') try: mkdir(backup_dir) except OSError as err: if err.errno != EEXIST: raise builder.save(pathjoin(backup_dir, '%d.' % time() + basename(argv[1]))) builder.save(argv[1]) exit(EXIT_SUCCESS)
def create_sample_ring(self): """ Create a sample ring with two devices At least two devices are needed to test removing a device, since removing the last device of a ring is not allowed """ # Ensure there is no existing test builder file because # create_sample_ring() might be used more than once in a single test try: os.remove(self.tmpfile) except OSError: pass ring = RingBuilder(6, 3, 1) ring.add_dev({'weight': 100.0, 'region': 0, 'zone': 0, 'ip': '127.0.0.1', 'port': 6000, 'device': 'sda1', 'meta': 'some meta data', }) ring.add_dev({'weight': 100.0, 'region': 1, 'zone': 1, 'ip': '127.0.0.2', 'port': 6001, 'device': 'sda2' }) ring.save(self.tmpfile)
def change_meta(self, builder_type, dev_meta, lasthash, start_response, env): """ Change meta info for devices :param builder_type: the builder_type to use when loading the builder :param dev_meta: a dict of device id and meta info :param lasthash: the hash to use when verifying state """ with lock_file(self.bf_path[builder_type], timeout=1, unlink=False): self.verify_current_hash(self.bf_path[builder_type], lasthash) builder = RingBuilder.load(self.bf_path[builder_type]) try: modified = False for dev_id in dev_meta: sleep() # so we don't starve/block for device in builder.devs: if not device: continue if device['id'] == int(dev_id): modified = True device['meta'] = '%s' % dev_meta[dev_id] if modified: newmd5 = self.write_builder(builder, self.bf_path[builder_type]) return self.return_response(True, newmd5, None, start_response, env) else: return self.return_response(False, lasthash, 'Invalid dev id %s.' % dev_id, start_response, env) except ValueError as err: return self.return_response(False, lasthash, str(err), start_response, env)
def __init__(self, builder_file, swiftdir='/etc/swift', backup_dir_name='backups', verbose=False): self.verbose = verbose self.builder_file = builder_file self.swiftdir = swiftdir self.backup_dir = pathjoin(self.swiftdir, backup_dir_name) self.builder = RingBuilder.load(builder_file)
def rebalance(self, builder_type, lasthash, start_response, env): """ rebalance a ring note: rebalance doesn't yield. """ with lock_file(self.bf_path[builder_type], timeout=1, unlink=False): self.verify_current_hash(self.bf_path[builder_type], lasthash) builder = RingBuilder.load(self.bf_path[builder_type]) devs_changed = builder.devs_changed try: last_balance = builder.get_balance() parts, balance = builder.rebalance() except RingBuilderError, err: self.logger.exception(_("Error during ring validation.")) return self.return_response(False, None, err.message, start_response, env) if not parts: msg = 'Either none need to be assigned or none can be due ' \ 'to min_part_hours [%s].' % builder.min_part_hours self.logger.error(_(msg)) return self.return_response(False, None, msg, start_response, env) if not devs_changed and abs(last_balance - balance) < 1: msg = 'Refusing to save rebalance. Did not change at least 1%.' self.logger.error(_(msg)) return self.return_response(False, None, msg, start_response, env) try: builder.validate() except RingValidationError, err: self.logger.exception(_("Error during ring validation.")) return self.return_response(False, None, err.message, start_response, env)
def remove_devs(self, builder_type, devices, lasthash, start_response, env): """ remove devices from the builder :params builder_type: the builder_type to use when loading the builder :params devices: list of device ids to be removed. :params lasthash: the hash to use when verifying state """ with lock_file(self.bf_path[builder_type], timeout=1, unlink=False): self.verify_current_hash(self.bf_path[builder_type], lasthash) builder = RingBuilder.load(self.bf_path[builder_type]) if not isinstance(devices, list): return self.return_response(False, lasthash, 'Malformed request.', start_response, env) for dev_id in devices: sleep() # so we don't starve/block try: builder.remove_dev(int(dev_id)) except (IndexError, TypeError): return self.return_response(False, lasthash, 'Invalid dev id %s.' % dev_id, start_response, env) except RingBuilderError as err: return self.return_response(False, lasthash, 'Error removing %s - %s.' % (dev_id, err), start_response, env) except ValueError as err: return self.return_response(False, lasthash, str(err), start_response, env) newmd5 = self.write_builder(builder, self.bf_path[builder_type]) return self.return_response(True, newmd5, None, start_response, env)
def change_weight(self, builder_type, dev_weights, lasthash, start_response, env): """ Change weight of devices :param builder_type: the builder_type to use when loading the builder :param dev_weights: a dict of device id and weight :param lasthash: the hash to use when verifying state """ with lock_file(self.bf_path[builder_type], timeout=1, unlink=False): self.verify_current_hash(self.bf_path[builder_type], lasthash) builder = RingBuilder.load(self.bf_path[builder_type]) for dev_id in dev_weights: sleep() # so we don't starve/block try: builder.set_dev_weight(int(dev_id), float(dev_weights[dev_id])) except (IndexError, TypeError): return self.return_response(False, lasthash, 'Invalid dev id %s.' % dev_id, start_response, env) except ValueError as err: return self.return_response(False, lasthash, str(err), start_response, env) newmd5 = self.write_builder(builder, self.bf_path[builder_type]) return self.return_response(True, newmd5, None, start_response, env)
def add_to_ring(self, builder_type, body, lasthash, start_response, env): """ Handle a add device post """ with lock_file(self.bf_path[builder_type], timeout=1, unlink=False): self.verify_current_hash(self.bf_path[builder_type], lasthash) builder = RingBuilder.load(self.bf_path[builder_type]) ring_modified = False try: for device in body['devices']: sleep() # so we don't starve/block if not self._is_existing_dev(builder, device['ip'], int(device['port']), device['device']): self._add_device(builder, int(device['zone']), device['ip'], int(device['port']), device['device'], float(device['weight']), device['meta']) ring_modified = True except (AttributeError, KeyError, ValueError, TypeError) as err: return self.return_response(False, lasthash, "Malformed request.", start_response, env) if ring_modified: newmd5 = self.write_builder(builder, self.bf_path[builder_type]) return self.return_response(True, newmd5, None, start_response, env) else: return self.return_response(False, lasthash, 'Ring remains unchanged.', start_response, env)
def test_validate(self): self.create_sample_ring() ring = RingBuilder.load(self.tmpfile) ring.rebalance() ring.save(self.tmpfile) argv = ["", self.tmpfile, "validate"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv)
def test_create_ring(self): argv = ["", self.tmpfile, "create", "6", "3.14159265359", "1"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.part_power, 6) self.assertEqual(ring.replicas, 3.14159265359) self.assertEqual(ring.min_part_hours, 1)
def test_set_info(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "set_info", search_value, "127.0.1.1:8000/sda1_other meta data"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) # Check that device was created with given data ring = RingBuilder.load(self.tmpfile) dev = [d for d in ring.devs if d['id'] == 0][0] self.assertEqual(dev['ip'], '127.0.1.1') self.assertEqual(dev['port'], 8000) self.assertEqual(dev['device'], 'sda1') self.assertEqual(dev['meta'], 'other meta data') # Check that second device in ring is not affected dev = [d for d in ring.devs if d['id'] == 1][0] self.assertEqual(dev['ip'], '127.0.0.2') self.assertEqual(dev['port'], 6001) self.assertEqual(dev['device'], 'sda2') self.assertEqual(dev['meta'], '') # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate())
def test_remove_device(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "remove", search_value] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) # Check that weight was set to 0 dev = [d for d in ring.devs if d['id'] == 0][0] self.assertEqual(dev['weight'], 0) # Check that device is in list of devices to be removed dev = [d for d in ring._remove_devs if d['id'] == 0][0] self.assertEqual(dev['region'], 0) self.assertEqual(dev['zone'], 0) self.assertEqual(dev['ip'], '127.0.0.1') self.assertEqual(dev['port'], 6000) self.assertEqual(dev['device'], 'sda1') self.assertEqual(dev['weight'], 0) self.assertEqual(dev['replication_ip'], '127.0.0.1') self.assertEqual(dev['replication_port'], 6000) self.assertEqual(dev['meta'], 'some meta data') # Check that second device in ring is not affected dev = [d for d in ring.devs if d['id'] == 1][0] self.assertEqual(dev['weight'], 100) self.assertFalse([d for d in ring._remove_devs if d['id'] == 1]) # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate())
def write_ring(args, devices, builderfile): # Make an educated guess about the used port. These are the defaults for # TripleO-based deployments in Mitaka builder_fname = os.path.basename(builderfile) if 'account' in builder_fname: port = 6002 elif 'container' in builder_fname: port = 6001 elif 'object' in builder_fname: port = 6000 else: port = 6000 logging.debug('Set port for new devices to %d' % port) if not os.path.isfile(builderfile): logging.info( '%s not found, creating new builder file', builderfile) rb = RingBuilder(args.part_power, args.replicas, args.min_part_hours) else: logging.info('Using existing builder file %s', builderfile) rb = RingBuilder.load(builderfile) # Add all missing devices for dev in devices: _dev = rb.search_devs({'meta': dev['meta']}) if not _dev: dev['weight'] = float(dev.get('size')) / 10**9 dev['region'] = 1 dev['zone'] = 1 dev['port'] = port dev['replication_ip'] = dev['ip'] dev['replication_port'] = dev['port'] rb.add_dev(dev) logging.info('Added device %s / %s', dev['ip'], dev['device']) else: logging.info( 'Ignoring existing device %s / %s', dev['ip'], dev['device']) try: rb.rebalance() except RingValidationError as exc: logging.error(exc) rb.save(builderfile) ring_file = os.path.splitext(builderfile)[0] + '.ring.gz' ring_data = rb.get_ring() ring_data.save(ring_file) return [builderfile, ring_file]
def storage_policies(request): """ Creates a storage policy to swift with an specific ring. Allows create replication storage policies and erasure code storage policies """ if request.method == "GET": try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) keys = r.keys("storage-policy:*") storage_policy_list = [] for key in keys: storage_policy = r.hgetall(key) to_json_bools(storage_policy, 'deprecated', 'default', 'deployed') storage_policy['id'] = str(key).split(':')[-1] storage_policy['devices'] = json.loads(storage_policy['devices']) storage_policy_list.append(storage_policy) return JSONResponse(storage_policy_list, status=status.HTTP_200_OK) if request.method == "POST": try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) data = JSONParser().parse(request) if data['policy_type'] == 'EC': data['replicas'] = int(data['ec_num_data_fragments']) + int(data['ec_num_parity_fragments']) try: sp_id = str(r.incr('storage-policies:id')) key = 'storage-policy:' + sp_id ring = RingBuilder(int(data['partition_power']), int(data['replicas']), int(data['time'])) ring.save(get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, sp_id)) r.hmset(key, data) except: return JSONResponse('Error creating the Storage Policy', status=status.HTTP_500_INTERNAL_SERVER_ERROR) return JSONResponse('Account created successfully', status=status.HTTP_201_CREATED) return JSONResponse('Only HTTP POST requests allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def test_server_overload_crazy_high(self): self.create_sample_ring() argv = "set_overload 10".split() out, err = self.run_srb(*argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.overload, 10.0) self.assertTrue('Warning overload is greater than 100%' in out) self.assertTrue('1000.00%' in out) self.assertTrue('10.000000' in out) # but it's cool if you do it on purpose argv[-1] = '1000%' out, err = self.run_srb(*argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.overload, 10.0) self.assertTrue('Warning overload is greater than 100%' not in out) self.assertTrue('1000.00%' in out) self.assertTrue('10.000000' in out)
def test_set_overload_percent(self): self.create_sample_ring() argv = "set_overload 10%".split() out, err = self.run_srb(*argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.overload, 0.1) self.assertTrue('10.00%' in out) self.assertTrue('0.100000' in out)
def test_load(self): rb = ring.RingBuilder(8, 3, 1) devs = [ {"id": 0, "zone": 0, "weight": 1, "ip": "127.0.0.0", "port": 10000, "device": "sda1", "meta": "meta0"}, {"id": 1, "zone": 1, "weight": 1, "ip": "127.0.0.1", "port": 10001, "device": "sdb1", "meta": "meta1"}, {"id": 2, "zone": 2, "weight": 2, "ip": "127.0.0.2", "port": 10002, "device": "sdc1", "meta": "meta2"}, {"id": 3, "zone": 3, "weight": 2, "ip": "127.0.0.3", "port": 10003, "device": "sdd1"}, ] for d in devs: rb.add_dev(d) rb.rebalance() real_pickle = pickle.load try: # test a legit builder fake_pickle = Mock(return_value=rb) fake_open = Mock(return_value=None) pickle.load = fake_pickle builder = RingBuilder.load("fake.builder", open=fake_open) self.assertEquals(fake_pickle.call_count, 1) fake_open.assert_has_calls([mock_call("fake.builder", "rb")]) self.assertEquals(builder, rb) fake_pickle.reset_mock() fake_open.reset_mock() # test old style builder fake_pickle.return_value = rb.to_dict() pickle.load = fake_pickle builder = RingBuilder.load("fake.builder", open=fake_open) fake_open.assert_has_calls([mock_call("fake.builder", "rb")]) self.assertEquals(builder.devs, rb.devs) fake_pickle.reset_mock() fake_open.reset_mock() # test old devs but no meta no_meta_builder = rb for dev in no_meta_builder.devs: del (dev["meta"]) fake_pickle.return_value = no_meta_builder pickle.load = fake_pickle builder = RingBuilder.load("fake.builder", open=fake_open) fake_open.assert_has_calls([mock_call("fake.builder", "rb")]) self.assertEquals(builder.devs, rb.devs) fake_pickle.reset_mock() finally: pickle.load = real_pickle
def test_set_overload_percent_strange_input(self): self.create_sample_ring() argv = "set_overload 26%%%%".split() out, err = self.run_srb(*argv) ring = RingBuilder.load(self.tmpfile) self.assertEqual(ring.overload, 0.26) self.assertTrue('26.00%' in out) self.assertTrue('0.260000' in out)
def create_sample_ring(self): """ Create a sample ring with two devices At least two devices are needed to test removing a device, since removing the last device of a ring is not allowed """ # Ensure there is no existing test builder file because # create_sample_ring() might be used more than once in a single test try: os.remove(self.tmpfile) except OSError: pass ring = RingBuilder(6, 3, 1) ring.add_dev( { "weight": 100.0, "region": 0, "zone": 0, "ip": "127.0.0.1", "port": 6000, "device": "sda1", "meta": "some meta data", } ) ring.add_dev({"weight": 100.0, "region": 1, "zone": 1, "ip": "127.0.0.2", "port": 6001, "device": "sda2"}) ring.save(self.tmpfile)
def gen_builder(self, balanced=False): builder = RingBuilder(18, 3, 1) for i in xrange(self.device_count): zone = i ipaddr = "1.1.1.1" port = 6010 device_name = "sd%s" % i weight = 100.0 meta = "meta for %s" % i next_dev_id = 0 if builder.devs: next_dev_id = max(d['id'] for d in builder.devs if d) + 1 builder.add_dev({'id': next_dev_id, 'zone': zone, 'ip': ipaddr, 'port': int(port), 'device': device_name, 'weight': weight, 'meta': meta}) if balanced: builder.rebalance() return builder
def check_builder(builder_file, save_builder=False, fix_replicas=False, set_overload=None, set_min_part_hours=None, **kwargs): """ Create a builder from builder_file and rebalance, return the stats. :param builder_file: path to builder on disk :param save_builder: bool, if true save builder after rebalance :param fix_replicas: bool, if true reduce replica count on ring to max(count_of_devices_with_weight, current_replica_count) :param set_overload: float or None, if float set_overload on the builder before rebalance :param set_min_part_hours: int or None, if int set_min_part_hours on the builder before rebalance :returns: stats, a dict, information about the check """ builder = RingBuilder.load(builder_file) count_of_devices_with_weight = len([d for d in builder._iter_devs() if d['weight'] > 0]) stats = { 'builder_file': builder_file, 'parts': builder.parts, 'replicas': builder.replicas, 'num_devs': count_of_devices_with_weight, 'overload': builder.overload, 'min_part_hours': builder.min_part_hours, } if count_of_devices_with_weight < 1: return stats builder._build_dispersion_graph() stats.update({ 'initial_balance': builder.get_balance(), 'initial_dispersion': builder.dispersion, }) if fix_replicas: if builder.replicas > count_of_devices_with_weight: builder.set_replicas(float(count_of_devices_with_weight)) if set_overload is not None: builder.set_overload(set_overload) if set_min_part_hours is not None: builder.change_min_part_hours(set_min_part_hours) start = time.time() parts_moved, final_balance = builder.rebalance()[:2] builder.validate() if save_builder: builder.save(builder_file) stats.update({ 'final_replicas': builder.replicas, 'final_overload': builder.overload, 'final_min_part_hours': builder.min_part_hours, 'parts_moved': parts_moved, 'final_balance': final_balance, 'final_dispersion': builder.dispersion, 'rebalance_time': time.time() - start, }) return stats
def _get_devs_from_builder(self, builder_filename): ring_name = builder_filename[0:builder_filename.find('.builder')] try: builder = SwiftRingBuilder.load(builder_filename) except Exception as e: raise IOError('ERROR: swift-ring-builder Problem occurred while ' 'reading builder file: %s. %s' % (builder_filename, e)) self.partitions = builder.parts self.replica_count = builder.replicas balance = 0 if builder.devs: balance = builder.get_balance() self.balance = balance self.dispersion = 0.00 if builder.dispersion: self.dispersion = float(builder.dispersion) self.min_part_hours = builder.min_part_hours self.remaining = str(timedelta(seconds=builder.min_part_seconds_left)) self.overload = builder.overload if builder.devs: balance_per_dev = builder._build_balance_per_dev() for dev in builder._iter_devs(): if dev in builder._remove_devs: # This device was added and then later removed without # doing a rebalance in the meantime. We can ignore # since it's marked for deletion. continue device_info = DeviceInfo({ 'ring_name': ring_name, 'zone_id': dev['zone'], 'region_id': dev['region'], 'server_ip': dev['ip'], 'server_bind_port': dev['port'], 'replication_ip': dev['replication_ip'], 'replication_bind_port': dev['replication_port'], 'swift_drive_name': dev['device'], 'current_weight': dev['weight'], 'balance': balance_per_dev[dev['id']], 'meta': dev['meta'], 'presence': 'present' }) yield device_info
def main(arguments=None): global argv, backup_dir, builder, builder_file, ring_file if arguments: argv = arguments else: argv = sys_argv if len(argv) < 2: print "swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % \ globals() print Commands.default.__doc__.strip() print cmds = [c for c, f in Commands.__dict__.iteritems() if f.__doc__ and c[0] != '_' and c != 'default'] cmds.sort() for cmd in cmds: print Commands.__dict__[cmd].__doc__.strip() print print parse_search_value.__doc__.strip() print for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ', subsequent_indent=' '): print line print('Exit codes: 0 = operation successful\n' ' 1 = operation completed with warnings\n' ' 2 = error') exit(EXIT_SUCCESS) builder_file, ring_file = parse_builder_ring_filename_args(argv) if exists(builder_file): builder = RingBuilder.load(builder_file) elif len(argv) < 3 or argv[2] not in('create', 'write_builder'): print 'Ring Builder file does not exist: %s' % argv[1] exit(EXIT_ERROR) backup_dir = pathjoin(dirname(argv[1]), 'backups') try: mkdir(backup_dir) except OSError as err: if err.errno != EEXIST: raise if len(argv) == 2: command = "default" else: command = argv[2] if argv[0].endswith('-safe'): try: with lock_parent_directory(abspath(argv[1]), 15): Commands.__dict__.get(command, Commands.unknown.im_func)() except exceptions.LockTimeout: print "Ring/builder dir currently locked." exit(2) else: Commands.__dict__.get(command, Commands.unknown.im_func)()
def main(arguments=None): global argv, backup_dir, builder, builder_file, ring_file if arguments: argv = arguments else: argv = sys_argv if len(argv) < 2: print "swift-ring-builder %(MAJOR_VERSION)s.%(MINOR_VERSION)s\n" % \ globals() print Commands.default.__doc__.strip() print cmds = [c for c, f in Commands.__dict__.iteritems() if f.__doc__ and c[0] != '_' and c != 'default'] cmds.sort() for cmd in cmds: print Commands.__dict__[cmd].__doc__.strip() print print parse_search_value.__doc__.strip() print for line in wrap(' '.join(cmds), 79, initial_indent='Quick list: ', subsequent_indent=' '): print line print ('Exit codes: 0 = operation successful\n' ' 1 = operation completed with warnings\n' ' 2 = error') exit(EXIT_SUCCESS) builder_file, ring_file = parse_builder_ring_filename_args(argv) if exists(builder_file): builder = RingBuilder.load(builder_file) elif len(argv) < 3 or argv[2] not in('create', 'write_builder'): print 'Ring Builder file does not exist: %s' % argv[1] exit(EXIT_ERROR) backup_dir = pathjoin(dirname(argv[1]), 'backups') try: mkdir(backup_dir) except OSError as err: if err.errno != EEXIST: raise if len(argv) == 2: command = "default" else: command = argv[2] if argv[0].endswith('-safe'): try: with lock_parent_directory(abspath(argv[1]), 15): Commands.__dict__.get(command, Commands.unknown.im_func)() except exceptions.LockTimeout: print "Ring/builder dir currently locked." exit(2) else: Commands.__dict__.get(command, Commands.unknown.im_func)()
def test_warn_at_risk(self): self.create_sample_ring() ring = RingBuilder.load(self.tmpfile) ring.devs[0]['weight'] = 10 ring.save(self.tmpfile) argv = ["", self.tmpfile, "rebalance"] try: swift.cli.ringbuilder.main(argv) except SystemExit as e: self.assertEquals(e.code, 1)
def deploy_storage_policy(request, storage_policy_id): try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) key = "storage-policy:" + storage_policy_id if request.method == "POST": if r.exists(key): try: tmp_policy_file = get_policy_file_path( settings.SWIFT_CFG_TMP_DIR, storage_policy_id) deploy_policy_file = get_policy_file_path( settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id) deploy_gzip_filename = deploy_policy_file.replace( 'builder', 'ring.gz') ring = RingBuilder.load(tmp_policy_file) ring.rebalance() ring.save(tmp_policy_file) ringdata = ring.get_ring() ringdata.save(deploy_gzip_filename) data = r.hgetall(key) update_sp_files( settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id, { 'name': data['name'], 'deprecated': data['deprecated'], 'default': data['default'] }) copyfile(tmp_policy_file, deploy_policy_file) rsync_dir_with_nodes(settings.SWIFT_CFG_DEPLOY_DIR, '/etc/swift') r.hset(key, 'deployed', 'True') return JSONResponse('Storage policy deployed correctly', status=status.HTTP_200_OK) except RedisError: return JSONResponse('Storage policy could not be deployed', status=status.HTTP_400_BAD_REQUEST) except exceptions.RingBuilderError, e: return JSONResponse( 'Storage policy could not be deployed. Error message: %s' % e.message, status=status.HTTP_400_BAD_REQUEST) else: return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND)
def list_devices(self, builder_type, start_response, env): """ list ALL devices in the ring :params builder_type: the builder_type to use when loading the builder :returns: list of boolean status, md5sum of the current ring, and all builder.devs """ with lock_file(self.bf_path[builder_type], timeout=1, unlink=False): builder = RingBuilder.load(self.bf_path[builder_type]) current_md5sum = self._get_md5sum(self.bf_path[builder_type]) return self.return_response(True, current_md5sum, builder.devs, start_response, env)
def deploy_storage_policy(request, storage_policy_id): try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) key = "storage-policy:" + storage_policy_id if request.method == "POST": if r.exists(key): try: tmp_policy_file = get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, storage_policy_id) deploy_policy_file = get_policy_file_path(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id) deploy_gzip_filename = deploy_policy_file.replace('builder', 'ring.gz') ring = RingBuilder.load(tmp_policy_file) ring.rebalance() ring.save(tmp_policy_file) ringdata = ring.get_ring() ringdata.save(deploy_gzip_filename) data = r.hgetall(key) policy = {'name': data['name'], 'deprecated': data['deprecated'], 'default': data['default']} if data['policy_type'] == 'EC': policy.update({'policy_type': 'erasure_coding', 'ec_type': data['ec_type'], 'ec_num_data_fragments': data['ec_num_data_fragments'], 'ec_num_parity_fragments': data['ec_num_parity_fragments'], 'ec_object_segment_size': data['ec_object_segment_size'], 'ec_duplication_factor': data['ec_duplication_factor']}) update_sp_files(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id, policy) else: update_sp_files(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id, policy) copyfile(tmp_policy_file, deploy_policy_file) rsync_dir_with_nodes(settings.SWIFT_CFG_DEPLOY_DIR, '/etc/swift') r.hset(key, 'deployed', 'True') return JSONResponse('Storage policy deployed correctly', status=status.HTTP_200_OK) except RedisError: return JSONResponse('Storage policy could not be deployed', status=status.HTTP_400_BAD_REQUEST) except exceptions.RingBuilderError as e: return JSONResponse('Storage policy could not be deployed. Error message: %s' % e.message, status=status.HTTP_400_BAD_REQUEST) else: return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND) return JSONResponse('Only HTTP POST requests allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def create(): """ swift-ring-builder <builder_file> create <part_power> <replicas> <min_part_hours> Creates <builder_file> with 2^<part_power> partitions and <replicas>. <min_part_hours> is number of hours to restrict moving a partition more than once. OR swift-ring-builder <builder_file> create <part_power> <replicas> <min_part_hours> <policy_info_file> policy_info_file contains the information about the policy """ if len(argv) < 6: print Commands.create.__doc__.strip() exit(EXIT_ERROR) if len (argv) > 6: builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5]), argv[6]) else: builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5])) backup_dir = pathjoin(dirname(argv[1]), 'backups') try: mkdir(backup_dir) except OSError as err: if err.errno != EEXIST: raise builder.save(pathjoin(backup_dir, '%d.' % time() + basename(argv[1]))) builder.save(argv[1]) exit(EXIT_SUCCESS)
def gen_builder(self, balanced=False): builder = RingBuilder(18, 3, 1) for i in xrange(self.device_count): zone = i ipaddr = "1.1.1.1" port = 6010 device_name = "sd%s" % i weight = 100.0 meta = "meta for %s" % i next_dev_id = 0 if builder.devs: next_dev_id = max(d['id'] for d in builder.devs if d) + 1 builder.add_dev({ 'id': next_dev_id, 'zone': zone, 'ip': ipaddr, 'port': int(port), 'device': device_name, 'weight': weight, 'meta': meta }) # add an empty dev builder.devs.append(None) if balanced: builder.rebalance() return builder
def create(): """ swift-ring-builder <builder_file> create <part_power> <replicas> <min_part_hours> Creates <builder_file> with 2^<part_power> partitions and <replicas>. <min_part_hours> is number of hours to restrict moving a partition more than once. """ if len(argv) < 6: print(Commands.create.__doc__.strip()) exit(EXIT_ERROR) # 1、生成RingBuilder对象实例 builder = RingBuilder(int(argv[3]), float(argv[4]), int(argv[5])) # 2、创建备份目录 backup_dir = pathjoin(dirname(builder_file), 'backups') try: mkdir(backup_dir) except OSError as err: if err.errno != EEXIST: raise # 3、保存原始数据到备份目录,以及/etc/swift目录中 builder.save( pathjoin(backup_dir, '%d.' % time() + basename(builder_file))) builder.save(builder_file) exit(EXIT_SUCCESS)
def test_reconcile_add_devices(self): # given a empty ring builder b = RingBuilder(10, 1, 1) r = RingController(b, "object") # when new devices are added ds = ["r1z1-192.168.0.2:6000/d3", "r1z1-192.168.2.2:5000/d1"] r.reconcile(ds) # then new devices are present devs = b.search_devs([]) self.assertEqual(2, len(devs)) self.assertEqual(devs, [{ 'device': 'd3', 'id': 0, 'ip': '192.168.0.2', 'meta': '', 'parts': 512, 'parts_wanted': 0, 'port': 6000, 'region': 1, 'replication_ip': None, 'replication_port': None, 'weight': 1.0, 'zone': 1 }, { 'device': 'd1', 'id': 1, 'ip': '192.168.2.2', 'meta': '', 'parts': 512, 'parts_wanted': 0, 'port': 5000, 'region': 1, 'replication_ip': None, 'replication_port': None, 'weight': 1.0, 'zone': 1 }])
def delete_storage_policy_disks(request, storage_policy_id, disk_id): try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) key = "storage-policy:" + storage_policy_id if request.method == 'DELETE': if r.exists(key): try: tmp_policy_file = get_policy_file_path( settings.SWIFT_CFG_TMP_DIR, storage_policy_id) found = False storage_policy = r.hgetall(key) storage_policy['devices'] = json.loads( storage_policy['devices']) for i, disk in enumerate(storage_policy['devices']): if disk_id == disk[0]: found = True ring = RingBuilder.load(tmp_policy_file) ring.remove_dev(disk[1]) ring.save(tmp_policy_file) del storage_policy['devices'][i] storage_policy['devices'] = json.dumps( storage_policy['devices']) r.hset(key, 'devices', storage_policy['devices']) r.hset(key, 'deployed', False) return JSONResponse("Disk removed", status=status.HTTP_204_NO_CONTENT) if not found: return JSONResponse('Disk not found', status=status.HTTP_404_NOT_FOUND) except RedisError: return JSONResponse("Error updating storage policy", status=status.HTTP_400_BAD_REQUEST) else: return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND) return JSONResponse('Method not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def main(argv): setup_logging() args = read_args() logger.debug(args) load_config() v1 = client.CoreV1Api() config_map = load_config_map(v1, args.config_map_name) if config_map.binary_data != None and args.ring_type in config_map.binary_data: builder_ = base64.b64decode(config_map.binary_data[args.ring_type]) logger.info("loading existing ring builder") builder = RingBuilder.load("", open=lambda a, b: io.BytesIO(builder_)) else: logger.info("creating a new ring builder") builder = RingBuilder(10, 1, 1) r = RingController(builder, args.ring_type, logger=logger) logger.info("reconciling ring") r.reconcile(args.devices) ring_data = r.get_ring_data() patch_config_map(v1, config_map.metadata, ring_data)
def _write_stub_builder(self, region): replicas = 3 builder = RingBuilder(8, replicas, 1) for i in range(replicas): dev = { 'weight': 100, 'region': '%d' % region, 'zone': '1', 'ip': '10.0.0.%d' % region, 'port': '3600', 'device': 'sdb%d' % i } builder.add_dev(dev) builder.rebalance() builder_file = os.path.join(self.tmpdir, '%d.builder' % region) builder.save(builder_file) return builder, builder_file
def test_validate_corrupted_file(self): self.create_sample_ring() ring = RingBuilder.load(self.tmpfile) ring.rebalance() self.assertTrue(ring.validate()) # ring is valid until now ring.save(self.tmpfile) argv = ["", self.tmpfile, "validate"] # corrupt the file with open(self.tmpfile, 'wb') as f: f.write(os.urandom(1024)) try: swift.cli.ringbuilder.main(argv) except SystemExit as e: self.assertEquals(e.code, 2)
def test_get_ring_data(self): # given a ring builder with two devices b = RingBuilder(10, 1, 1) r = RingController(b, "object") ds = ["r1z1-192.168.0.2:6000/d3", "r1z2-192.168.2.2:5000/d1"] r.reconcile(ds) # when ring is serialized ring_data = r.get_ring_data() # then ring and builder is returned self.assertIn("object.ring.gz", ring_data) self.assertIn("object", ring_data) # then correct compressed ring is returned object_ring = decompress(base64.b64decode(ring_data["object.ring.gz"])) expected_ring = b'{"byteorder": "little", "devs": [{"device": "d3", "id": 0, "ip": "192.168.0.2", "meta": "", "port": 6000, "region": 1, "replication_ip": null, "replication_port": null, "weight": 1.0, "zone": 1}, {"device": "d1", "id": 1, "ip": "192.168.2.2", "meta": "", "port": 5000, "region": 1, "replication_ip": null, "replication_port": null, "weight": 1.0, "zone": 2}], "part_shift": 22, "replica_count": 1, "version": 3}' self.assertIn(expected_ring, object_ring) # then correct serialized builder is returned builder_ = base64.b64decode(ring_data["object"]) loadedBuilder = RingBuilder.load( "", open=lambda a, b: io.BytesIO(builder_)) self.assertEqual(b.devs, loadedBuilder.devs)
def write_builder(): """ swift-ring-builder <ring_file> write_builder [min_part_hours] Recreate a builder from a ring file (lossy) if you lost your builder backups. (Protip: don't lose your builder backups). [min_part_hours] is one of those numbers lost to the builder, you can change it with set_min_part_hours. """ if exists(builder_file): print('Cowardly refusing to overwrite existing ' 'Ring Builder file: %s' % builder_file) exit(EXIT_ERROR) if len(argv) > 3: min_part_hours = int(argv[3]) else: stderr.write("WARNING: default min_part_hours may not match " "the value in the lost builder.\n") min_part_hours = 24 ring = Ring(ring_file) for dev in ring.devs: if dev is None: continue dev.update({ 'parts': 0, 'parts_wanted': 0, }) builder_dict = { 'part_power': 32 - ring._part_shift, 'replicas': float(ring.replica_count), 'min_part_hours': min_part_hours, 'parts': ring.partition_count, 'devs': ring.devs, 'devs_changed': False, 'version': 0, '_replica2part2dev': ring._replica2part2dev_id, '_last_part_moves_epoch': None, '_last_part_moves': None, '_last_part_gather_start': 0, '_remove_devs': [], } builder = RingBuilder.from_dict(builder_dict) for parts in builder._replica2part2dev: for dev_id in parts: builder.devs[dev_id]['parts'] += 1 builder._set_parts_wanted() builder.save(builder_file)
def check_builder(builder_file, save_builder=False, fix_replicas=False, **kwargs): """ Create a builder from builder_file and rebalance, return the stats. :param builder_file: path to builder on disk :param save_builder: bool, if true save builder after rebalance :param fix_replicas: bool, if true reduce replica count on ring to max(count_of_devices_with_weight, current_replica_count) :returns: stats, a dict, information about the check """ builder = RingBuilder.load(builder_file) count_of_devices_with_weight = len([d for d in builder._iter_devs() if d['weight'] > 0]) stats = { 'builder_file': builder_file, 'parts': builder.parts, 'replicas': builder.replicas, 'num_devs': count_of_devices_with_weight, } if count_of_devices_with_weight < 1: return stats builder._build_dispersion_graph() stats.update({ 'initial_balance': builder.get_balance(), 'initial_dispersion': builder.dispersion, }) if fix_replicas: if builder.replicas > count_of_devices_with_weight: builder.set_replicas(float(count_of_devices_with_weight)) start = time.time() parts_moved, final_balance = builder.rebalance()[:2] builder.validate() if save_builder: builder.save(builder_file) stats.update({ 'final_replicas': builder.replicas, 'parts_moved': parts_moved, 'final_balance': final_balance, 'final_dispersion': builder.dispersion, 'rebalance_time': time.time() - start, }) return stats
def test_set_weight(self): for search_value in self.search_values: self.create_sample_ring() argv = ["", self.tmpfile, "set_weight", search_value, "3.14159265359"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) ring = RingBuilder.load(self.tmpfile) # Check that weight was changed dev = [d for d in ring.devs if d["id"] == 0][0] self.assertEqual(dev["weight"], 3.14159265359) # Check that second device in ring is not affected dev = [d for d in ring.devs if d["id"] == 1][0] self.assertEqual(dev["weight"], 100) # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate())
def delete_storage_policy_disks(request, storage_policy_id, disk_id): try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) key = "storage-policy:" + storage_policy_id if request.method == 'DELETE': if r.exists(key): try: tmp_policy_file = get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, storage_policy_id) found = False storage_policy = r.hgetall(key) storage_policy['devices'] = json.loads(storage_policy['devices']) for i, disk in enumerate(storage_policy['devices']): if disk_id == disk[0]: found = True ring = RingBuilder.load(tmp_policy_file) ring.remove_dev(disk[1]) ring.save(tmp_policy_file) del storage_policy['devices'][i] storage_policy['devices'] = json.dumps(storage_policy['devices']) r.hset(key, 'devices', storage_policy['devices']) r.hset(key, 'deployed', False) return JSONResponse("Disk removed", status=status.HTTP_204_NO_CONTENT) if not found: return JSONResponse('Disk not found', status=status.HTTP_404_NOT_FOUND) except RedisError: return JSONResponse("Error updating storage policy", status=status.HTTP_400_BAD_REQUEST) else: return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND) return JSONResponse('Method not allowed.', status=status.HTTP_405_METHOD_NOT_ALLOWED)
def search(self, builder_type, search_pattern, start_response, env): """ search the builder for devices matching search pattern :params builder_type: the builder_type to use when loading the builder :params search_values: the value to search for :returns: list of boolean status, md5sum of current builder file on disk, and error message or dict of matched devices. """ with lock_file(self.bf_path[builder_type], timeout=1, unlink=False): builder = RingBuilder.load(self.bf_path[builder_type]) try: search_result = builder.search_devs(str(search_pattern)) return self.return_response(True, self._get_md5sum( self.bf_path[builder_type]), search_result, start_response, env) except ValueError: return self.return_response(False, self._get_md5sum( self.bf_path[builder_type]), 'Invalid search term', start_response, env)
def test_add_device(self): self.create_sample_ring() argv = ["", self.tmpfile, "add", "r2z3-127.0.0.1:6000/sda3_some meta data", "3.14159265359"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) # Check that device was created with given data ring = RingBuilder.load(self.tmpfile) dev = [d for d in ring.devs if d["id"] == 2][0] self.assertEqual(dev["region"], 2) self.assertEqual(dev["zone"], 3) self.assertEqual(dev["ip"], "127.0.0.1") self.assertEqual(dev["port"], 6000) self.assertEqual(dev["device"], "sda3") self.assertEqual(dev["weight"], 3.14159265359) self.assertEqual(dev["replication_ip"], "127.0.0.1") self.assertEqual(dev["replication_port"], 6000) self.assertEqual(dev["meta"], "some meta data") # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate())
def deploy_storage_policy(request, storage_policy_id): try: r = get_redis_connection() except RedisError: return JSONResponse('Error connecting with DB', status=status.HTTP_500_INTERNAL_SERVER_ERROR) key = "storage-policy:" + storage_policy_id if request.method == "POST": if r.exists(key): try: tmp_policy_file = get_policy_file_path(settings.SWIFT_CFG_TMP_DIR, storage_policy_id) deploy_policy_file = get_policy_file_path(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id) deploy_gzip_filename = deploy_policy_file.replace('builder', 'ring.gz') ring = RingBuilder.load(tmp_policy_file) ring.rebalance() ring.save(tmp_policy_file) ringdata = ring.get_ring() ringdata.save(deploy_gzip_filename) data = r.hgetall(key) update_sp_files(settings.SWIFT_CFG_DEPLOY_DIR, storage_policy_id, {'name': data['name'], 'deprecated': data['deprecated'], 'default': data['default']}) copyfile(tmp_policy_file, deploy_policy_file) rsync_dir_with_nodes(settings.SWIFT_CFG_DEPLOY_DIR, '/etc/swift') r.hset(key, 'deployed', 'True') return JSONResponse('Storage policy deployed correctly', status=status.HTTP_200_OK) except RedisError: return JSONResponse('Storage policy could not be deployed', status=status.HTTP_400_BAD_REQUEST) except exceptions.RingBuilderError, e: return JSONResponse('Storage policy could not be deployed. Error message: %s' % e.message, status=status.HTTP_400_BAD_REQUEST) else: return JSONResponse('Storage policy not found.', status=status.HTTP_404_NOT_FOUND)
def test_add_device(self): self.create_sample_ring() argv = ["", self.tmpfile, "add", "r2z3-127.0.0.1:6000/sda3_some meta data", "3.14159265359"] self.assertRaises(SystemExit, swift.cli.ringbuilder.main, argv) # Check that device was created with given data ring = RingBuilder.load(self.tmpfile) dev = [d for d in ring.devs if d['id'] == 2][0] self.assertEqual(dev['region'], 2) self.assertEqual(dev['zone'], 3) self.assertEqual(dev['ip'], '127.0.0.1') self.assertEqual(dev['port'], 6000) self.assertEqual(dev['device'], 'sda3') self.assertEqual(dev['weight'], 3.14159265359) self.assertEqual(dev['replication_ip'], '127.0.0.1') self.assertEqual(dev['replication_port'], 6000) self.assertEqual(dev['meta'], 'some meta data') # Final check, rebalance and check ring is ok ring.rebalance() self.assertTrue(ring.validate())