def test_missing_ring_specification(self): options = DummyInputOptions() config_paths = CloudMultiSite(options) input_model = ServersModel('standard', 'ccp', config=safe_load(standard_input_model), consumes_model=standard_swf_rng_consumes) # Change the disk model so a bad ring name is referenced for server in input_model.servers: disk_model = server.get('disk_model') for device_group in disk_model.get('device_groups', []): consumer = device_group.get('consumer') if consumer and consumer.get('name', 'other') == 'swift': attrs = consumer.get('attrs') attrs.get('rings').append({'name': 'object-99'}) ring_model = RingSpecifications('standard', 'ccp', model=None) ring_model.load_configuration('standard', 'ccp', safe_load(standard_configuration_data)) rings = RingBuilder(self.builder_dir, False) drive_configurations = dummy_osconfig_load( standard_drive_configurations) delta = RingDelta() try: generate_delta(config_paths, input_model, ring_model, rings, drive_configurations, options, delta) self.assertTrue(False, msg='should not get here') except SwiftModelException as err: self.assertTrue('Cannot proceed' in str(err))
def test_build_rings(self): options = DummyInputOptions() config_paths = CloudMultiSite(options) input_model = ServersModel('standard', 'ccp', config=safe_load(standard_input_model), consumes_model=standard_swf_rng_consumes) # Use rings from configuration-data object ring_model = RingSpecifications('standard', 'ccp', model=None) ring_model.load_configuration('standard', 'ccp', safe_load(standard_configuration_data)) rings = RingBuilder(self.builder_dir, False) drive_configurations = dummy_osconfig_load( standard_drive_configurations) delta = RingDelta() generate_delta(config_paths, input_model, ring_model, rings, drive_configurations, options, delta) options.dry_run = True # Validate pretend has no effect since all rings are new options.pretend_min_part_hours_passed = True cmds = rebalance(delta, rings, options) assert_cmds_are_same(self, expected_cmds, cmds)
def main(): parser = OptionParser(usage=usage) parser.add_option('--etc', dest='etc', default=DEFAULT_ETC, help='Overrides /etc/swiftlm (for testing)') parser.add_option('--cloud', dest='cloud', default=None, help='The name of the cloud') parser.add_option('--control-plane', dest='control_plane', default=None, help='The name of the control plane') parser.add_option('--ring-delta', dest='ring_delta', default=None, help='Name of ring-delta file (as output or input' ' A value of "-" (on output means to write' ' to stdout') parser.add_option('--format', dest='fmt', default='yaml', help='One of yaml or json.' ' When used with --ring-delta, specifies the' ' format of the file.') parser.add_option('--detail', dest='detail', default='summary', help='Level of detail to use with --report.' ' Use summary or full') parser.add_option('--report', dest='report', default=False, action="store_true", help='Explain what the ring delta represents.' ' Optionally use --detail.') parser.add_option('--dry-run', dest='dry_run', default=False, action="store_true", help='Show the proposed swift-ring-builder commands') parser.add_option('--pretend-min-part-hours-passed', dest='pretend_min_part_hours_passed', default=False, action="store_true", help='Executes the pretend_min_part_hours_passed command' ' on each ring before running rebalance.' ' Use with caution.') parser.add_option('--make-delta', dest='make_delta', default=False, action="store_true", help='Make a ring delta file') parser.add_option('--rebalance', dest='rebalance', default=False, action="store_true", help='Build (or rebalance) rings') parser.add_option('--limit-ring', dest='limit_ring', default=None, help='Limits actions to given ring') parser.add_option('--size-to-weight', dest='size_to_weight', default=float(1024 * 1024 * 1024), help='Conversion factor for size to weight. Default is' ' 1GB is weight of 1 (a 4Tb drive would be assigned' ' a weight of 4096') parser.add_option('--weight-step', dest='weight_step', default=None, help='When set, weights are changed by at most this' ' value. Overrides value in ring specification.') parser.add_option('--allow-partitions', dest='allow_partitions', default=False, action='store_true', help='Allow devices to be assigned to partitions.' ' Default is to use a full disk drive.') parser.add_option('--stop-on-warnings', dest='stop_on_warnings', default=False, action='store_true', help='Used with --make-delta. Exit with error if there' ' are model missmatch warnings.' ' Default is to only exit with error for errors.') parser.add_option('--unittest', dest='unittest', default=False, action='store_true', help='Set by unittests. Never set on command line.') (options, args) = parser.parse_args() if not (options.cloud and options.control_plane): sys.exit('Must specify both --cloud and --control_plane') sites = CloudMultiSite(options) my_cloud = sites.my_cloud my_control_plane = sites.my_control_plane my_config = sites.path(my_cloud, my_control_plane) # # Work out what we need to do. Validate arguments needed by an action # are present. # actions = [] if options.make_delta: actions.append('init-delta') actions.append('input-from-model') actions.append('read-builder-dir') actions.append('open-osconfig-dir') actions.append('make-delta') actions.append('write-to-delta') if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if options.report: actions.append('init-delta') actions.append('read-from-delta') actions.append('report') if options.detail not in ['summary', 'full']: sys.exit('Invalid value for --detail') if options.rebalance: actions.append('init-delta') actions.append('open-builder-dir') actions.append('read-from-delta') actions.append('rebalance') if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if len(actions) == 0: sys.exit('Missing an option to perform some action') if options.report and (options.make_delta or options.rebalance): sys.exit('Do not mix --report with other actions') # # Perform actions # if 'init-delta' in actions: delta = RingDelta() if 'input-from-model' in actions: servers_model = ServersModel('unused', 'unused') consumes = Consumes() ring_model = RingSpecifications(my_cloud, my_control_plane) for cloud, control_plane in sites.control_planes(): config = sites.path(cloud, control_plane) input_model_fd = None try: input_model_fd = open(config.get('input-model'), 'r') except IOError as err: pass # File may not exist since its a legacy item try: cp_server_fd = None cp_server_fd = open(config.get('control_plane_servers'), 'r') except IOError as err: sys.exit('Error on control_plane_server.yml: %s' % err) try: control_plane_servers = None if cp_server_fd: control_plane_servers = safe_load(cp_server_fd) except scanner.ScannerError as err: sys.exit('ERROR reading/parsing: %s' % err) try: consumes_fd = open(config.get('swift_ring_builder_consumes'), 'r') except IOError as err: sys.exit('ERROR: %s' % err) try: input_vars = {'global': {}} if input_model_fd: input_vars = safe_load(input_model_fd) consumes_model = safe_load(consumes_fd) except scanner.ScannerError as err: sys.exit('ERROR reading/parsing: %s' % err) try: if control_plane_servers: servers = control_plane_servers.get( 'control_plane_servers') elif input_vars.get('global').get('all_servers'): servers = input_vars.get('global').get('all_servers') else: sys.exit('No servers found in control plane') servers_model.add_servers(cloud, control_plane, servers) consumes.load_model(consumes_model) except SwiftModelException as err: sys.exit(err) servers_model.register_consumes(consumes) try: config_data_fd = open(my_config.get('configuration_data'), 'r') config_data = safe_load(config_data_fd) except (IOError, scanner.ScannerError) as err: sys.exit('Rings should be in configuration-data.' ' Using old configuration processor?') try: rings_loaded = False if input_vars.get('global').get('all_ring_specifications'): # Model contains Ardana old-style rings ring_model = RingSpecifications(my_cloud, my_control_plane, model=input_vars) rings_loaded = True if config_data and config_data.get( 'control_plane_rings', config_data.get('control-plane-rings')): # Model contains new-style rings --- use instead ring_model.load_configuration(my_cloud, my_control_plane, config_data) rings_loaded = True if not rings_loaded: sys.exit('No ring specifications in input model') except SwiftModelException as err: sys.exit(err) if 'open-builder-dir' or 'read-builder-dir' in actions: try: read_rings = False if 'read-builder-dir' in actions: read_rings = True rings = RingBuilder(my_config.get('builder_dir'), read_rings=read_rings) except IOError as err: sys.exit('ERROR: %s' % err) if 'open-osconfig-dir' in actions: drive_configurations = osconfig_load(sites) if 'make-delta' in actions: try: generate_delta(sites, servers_model, ring_model, rings, drive_configurations, options, delta) except SwiftModelException as err: sys.exit('ERROR: %s' % err) if 'write-to-delta' in actions: if my_config.get('ring-delta') == '-': write_to_file_fd = sys.stdout else: write_to_file_fd = open(my_config.get('ring-delta'), 'w') delta.write_to_file(write_to_file_fd, options.fmt) if 'read-from-delta' in actions: if my_config.get('ring-delta') == '-': sys.exit('--ring-delta- is invalid (read from stdin' 'not supported)') try: delta = RingDelta() read_from_delta_fd = open(my_config.get('ring-delta'), 'r') delta.read_from_file(read_from_delta_fd, options.fmt) except IOError as err: sys.exit('ERROR: %s' % err) if 'report' in actions: print(delta.get_report(options)) if 'rebalance' in actions: rebalance(delta, rings, options)