def main(): parser = OptionParser(usage=usage) parser.add_option('--etc', dest='etc', default=DEFAULT_ETC, help='Overrides /etc/swiftlm (for testing)') parser.add_option('--input-vars', dest='input_vars', default=None, help='The name of yaml containing vars') parser.add_option('--builder-dir', dest='builder_dir', default=None, help='Builder file directory') parser.add_option('--osconfig', dest='osconfig', default=None, help='Directory where drive size data files exist.' ' If omitted, the correct weights cannot' ' be assigned to drives.') parser.add_option('--hosts', dest='hosts', default=None, help='Hosts file (/etc/hosts or ./net/hosts.hf)') parser.add_option('--swift-ring-builder-consumes', dest='swift_ring_builder_consumes', default=None, help='File containing the SWF_RNG variable') parser.add_option('--ring-delta', dest='ring_delta', default=None, help='Name of ring-delta file (as output or input' ' A value of "-" (on output means to write' ' to stdout') parser.add_option('--format', dest='fmt', default='yaml', help='One of yaml or json.' ' When used with --ring-delta, specifies the' ' format of the file.') parser.add_option('--detail', dest='detail', default='summary', help='Level of detail to use with --report.' ' Use summary or full') parser.add_option('--report', dest='report', default=False, action="store_true", help='Explain what the ring delta represents.' ' Optionally use --detail.') parser.add_option('--dry-run', dest='dry_run', default=False, action="store_true", help='Show the proposed swift-ring-builder commands') parser.add_option('--make-delta', dest='make_delta', default=False, action="store_true", help='Make a ring delta file') parser.add_option('--rebalance', dest='rebalance', default=False, action="store_true", help='Build (or rebalance) rings') parser.add_option('--storage-policies', dest='storage_policies', default=False, action="store_true", help='Extract storage policy data.' ' Use this to register the storage_policies' ' variable.') parser.add_option('--region-name', dest='region_name', default=None, help='Region name for use with --storage-policies') parser.add_option('--size-to-weight', dest='size_to_weight', default=float(1024 * 1024 * 1024), help='Conversion factor for size to weight. Default is' ' 1GB is weight of 1 (a 4Tb drive would be assigned' ' a weight of 4096') parser.add_option('--weight-step', dest='weight_step', default=None, help='When set, weights are changed by at most this' ' value. Overrides value in ring specification.') parser.add_option('--allow-partitions', dest='allow_partitions', default=False, action='store_true', help='Allow devices to be assigned to partitions.' ' Default is to use a full disk drive.') parser.add_option('--stop-on-warnings', dest='stop_on_warnings', default=False, action='store_true', help='Used with --make-delta. Exit with error if there' ' are model missmatch warnings.' ' Default is to only exit with error for errors.') (options, args) = parser.parse_args() if not options.input_vars: options.input_vars = os.path.join(options.etc, DEFAULT_INPUT_VARS) if not options.builder_dir: options.builder_dir = os.path.join(options.etc, DEFAULT_BUILDER_DIR) if not options.hosts: options.hosts = os.path.join(options.etc, DEFAULT_HOSTS) if not options.swift_ring_builder_consumes: options.swift_ring_builder_consumes = os.path.join( options.etc, DEFAULT_SWIFT_RING_BUILDER_CONSUMES) if not options.ring_delta: options.ring_delta = os.path.join(options.etc, DEFAULT_RING_DELTA) if not options.osconfig: options.osconfig = os.path.join(options.etc, DEFAULT_OSCONFIG) # # Work out what we need to do. Validate arguments needed by an action # are present. # actions = [] if options.storage_policies: actions.append('input-from-model') actions.append('emit-storage-policies') if not options.region_name: print('Need --region-name') sys.exit(1) if options.make_delta: actions.append('init-delta') actions.append('input-from-model') actions.append('read-builder-dir') actions.append('open-osconfig-dir') actions.append('make-delta') actions.append('write-to-delta') if not options.ring_delta: print('Need --ring-delta file to write to') sys.exit(1) if not (options.input_vars and options.hosts and options.osconfig and options.swift_ring_builder_consumes): print('Need --input-vars, --hosts and --osconfig and' ' --swift-ring-builder-consumes inputs') sys.exit(1) if not options.builder_dir: print('Need --builder-dir option') sys.exit(1) if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if options.report: actions.append('init-delta') actions.append('read-from-delta') actions.append('report') if not options.ring_delta: print('Need --ring-delta file as input') sys.exit(1) if options.detail not in ['summary', 'full']: print('Invalid value for --detail') sys.exit(1) if options.rebalance: actions.append('init-delta') actions.append('open-builder-dir') actions.append('read-from-delta') actions.append('rebalance') if not options.ring_delta: print('Need --ring-delta file as input') sys.exit(1) if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if len(actions) == 0: print('Missing an option to perform some action') sys.exit(1) if options.storage_policies and (options.make_delta or options.rebalance or options.report): print('Do not mix --storage-policies with other actions') sys.exit(1) if options.report and (options.make_delta or options.rebalance or options.storage_policies): print('Do not mix --report with other actions') sys.exit(1) # # Perform actions # if 'init-delta' in actions: delta = RingDelta() if 'input-from-model' in actions: try: input_model_fd = open(options.input_vars, 'r') hosts_fd = open(options.hosts, 'r') consumes_fd = open(options.swift_ring_builder_consumes, 'r') except IOError as err: print('ERROR: %s' % err) sys.exit(1) try: input_vars = safe_load(input_model_fd) consumes_model = safe_load(consumes_fd) except scanner.ScannerError as err: print('ERROR in %s: %s' % (options.input_vars, err)) sys.exit(err) try: input_model = InputModel(config=input_vars, hosts_fd=hosts_fd, consumes=consumes_model) ring_model = RingSpecifications(model=input_vars) except SwiftModelException as err: sys.exit(err) if 'emit-storage-policies' in actions: obj_ring_policies = ring_model.get_storage_policies( options.region_name) print('%s' % safe_dump(obj_ring_policies, default_flow_style=False)) # Unlike the following actions, we exit after this action sys.exit(0) if 'open-builder-dir' or 'read-builder-dir' in actions: try: read_write = False if 'read-builder-dir' in actions: read_write = True rings = RingBuilder(options.builder_dir, True) except IOError as err: print('ERROR: %s' % err) sys.exit(1) if 'open-osconfig-dir' in actions: drive_configurations = osconfig_load(options.osconfig) if 'make-delta' in actions: try: generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) except SwiftModelException as err: print('ERROR: %s' % err) sys.exit(1) if 'write-to-delta' in actions: if options.ring_delta == '-': write_to_file_fd = sys.stdout else: write_to_file_fd = open(options.ring_delta, 'w') delta.write_to_file(write_to_file_fd, options.fmt) if 'read-from-delta' in actions: if options.ring_delta == '-': print('--ring-delta=- is invalid (read from stdin not supported)') sys.exit(1) try: delta = RingDelta() read_from_delta_fd = open(options.ring_delta, 'r') delta.read_from_file(read_from_delta_fd, options.fmt) except IOError as err: print('ERROR: %s' % err) sys.exit(1) if 'report' in actions: print(delta.get_report(options.detail)) if 'rebalance' in actions: rebalance(delta, rings, options.dry_run)
def main(): parser = OptionParser(usage=usage) parser.add_option('--etc', dest='etc', default=DEFAULT_ETC, help='Overrides /etc/swiftlm (for testing)') parser.add_option('--input-vars', dest='input_vars', default=None, help='The name of yaml containing vars') parser.add_option('--builder-dir', dest='builder_dir', default=None, help='Builder file directory') parser.add_option('--osconfig', dest='osconfig', default=None, help='Directory where drive size data files exist.' ' If omitted, the correct weights cannot' ' be assigned to drives.') parser.add_option('--hosts', dest='hosts', default=None, help='Hosts file (/etc/hosts or ./net/hosts.hf)') parser.add_option('--swift-ring-builder-consumes', dest='swift_ring_builder_consumes', default=None, help='File containing the SWF_RNG variable') parser.add_option('--ring-delta', dest='ring_delta', default=None, help='Name of ring-delta file (as output or input' ' A value of "-" (on output means to write' ' to stdout') parser.add_option('--format', dest='fmt', default='yaml', help='One of yaml or json.' ' When used with --ring-delta, specifies the' ' format of the file.') parser.add_option('--detail', dest='detail', default='summary', help='Level of detail to use with --report.' ' Use summary or full') parser.add_option('--report', dest='report', default=False, action="store_true", help='Explain what the ring delta represents.' ' Optionally use --detail.') parser.add_option('--dry-run', dest='dry_run', default=False, action="store_true", help='Show the proposed swift-ring-builder commands') parser.add_option('--make-delta', dest='make_delta', default=False, action="store_true", help='Make a ring delta file') parser.add_option('--rebalance', dest='rebalance', default=False, action="store_true", help='Build (or rebalance) rings') parser.add_option('--storage-policies', dest='storage_policies', default=False, action="store_true", help='Extract storage policy data.' ' Use this to register the storage_policies' ' variable.') parser.add_option('--region-name', dest='region_name', default=None, help='Region name for use with --storage-policies') parser.add_option('--size-to-weight', dest='size_to_weight', default=float(1024 * 1024 * 1024), help='Conversion factor for size to weight. Default is' ' 1GB is weight of 1 (a 4Tb drive would be assigned' ' a weight of 4096') parser.add_option('--weight-step', dest='weight_step', default=None, help='When set, weights are changed by at most this' ' value. Overrides value in ring specification.') parser.add_option('--allow-partitions', dest='allow_partitions', default=False, action='store_true', help='Allow devices to be assigned to partitions.' ' Default is to use a full disk drive.') parser.add_option('--stop-on-warnings', dest='stop_on_warnings', default=False, action='store_true', help='Used with --make-delta. Exit with error if there' ' are model missmatch warnings.' ' Default is to only exit with error for errors.') (options, args) = parser.parse_args() if not options.input_vars: options.input_vars = os.path.join(options.etc, DEFAULT_INPUT_VARS) if not options.builder_dir: options.builder_dir = os.path.join(options.etc, DEFAULT_BUILDER_DIR) if not options.hosts: options.hosts = os.path.join(options.etc, DEFAULT_HOSTS) if not options.swift_ring_builder_consumes: options.swift_ring_builder_consumes = os.path.join( options.etc, DEFAULT_SWIFT_RING_BUILDER_CONSUMES) if not options.ring_delta: options.ring_delta = os.path.join(options.etc, DEFAULT_RING_DELTA) if not options.osconfig: options.osconfig = os.path.join(options.etc, DEFAULT_OSCONFIG) # # Work out what we need to do. Validate arguments needed by an action # are present. # actions = [] if options.storage_policies: actions.append('input-from-model') actions.append('emit-storage-policies') if not options.region_name: print('Need --region-name') sys.exit(1) if options.make_delta: actions.append('init-delta') actions.append('input-from-model') actions.append('read-builder-dir') actions.append('open-osconfig-dir') actions.append('make-delta') actions.append('write-to-delta') if not options.ring_delta: print('Need --ring-delta file to write to') sys.exit(1) if not (options.input_vars and options.hosts and options.osconfig and options.swift_ring_builder_consumes): print('Need --input-vars, --hosts and --osconfig and' ' --swift-ring-builder-consumes inputs') sys.exit(1) if not options.builder_dir: print('Need --builder-dir option') sys.exit(1) if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if options.report: actions.append('init-delta') actions.append('read-from-delta') actions.append('report') if not options.ring_delta: print('Need --ring-delta file as input') sys.exit(1) if options.detail not in ['summary', 'full']: print('Invalid value for --detail') sys.exit(1) if options.rebalance: actions.append('init-delta') actions.append('open-builder-dir') actions.append('read-from-delta') actions.append('rebalance') if not options.ring_delta: print('Need --ring-delta file as input') sys.exit(1) if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if len(actions) == 0: print('Missing an option to perform some action') sys.exit(1) if options.storage_policies and (options.make_delta or options.rebalance or options.report): print('Do not mix --storage-policies with other actions') sys.exit(1) if options.report and (options.make_delta or options.rebalance or options.storage_policies): print('Do not mix --report with other actions') sys.exit(1) # # Perform actions # if 'init-delta' in actions: delta = RingDelta() if 'input-from-model' in actions: try: input_model_fd = open(options.input_vars, 'r') hosts_fd = open(options.hosts, 'r') consumes_fd = open(options.swift_ring_builder_consumes, 'r') except IOError as err: print('ERROR: %s' % err) sys.exit(1) try: input_vars = safe_load(input_model_fd) consumes_model = safe_load(consumes_fd) except scanner.ScannerError as err: print('ERROR in %s: %s' % (options.input_vars, err)) sys.exit(err) try: input_model = InputModel(config=input_vars, hosts_fd=hosts_fd, consumes=consumes_model) ring_model = RingSpecifications(model=input_vars) except SwiftModelException as err: sys.exit(err) if 'emit-storage-policies' in actions: obj_ring_policies = ring_model.get_storage_policies( options.region_name) print('%s' % safe_dump(obj_ring_policies, default_flow_style=False)) # Unlike the following actions, we exit after this action sys.exit(0) if 'open-builder-dir' or 'read-builder-dir' in actions: try: read_write = False if 'read-builder-dir' in actions: read_write = True rings = RingBuilder(options.builder_dir, True) except IOError as err: print('ERROR: %s' % err) sys.exit(1) if 'open-osconfig-dir' in actions: drive_configurations = osconfig_load(options.osconfig) if 'make-delta' in actions: try: generate_delta(input_model, ring_model, rings, drive_configurations, options, delta) except SwiftModelException as err: print('ERROR: %s' % err) sys.exit(1) if 'write-to-delta' in actions: if options.ring_delta == '-': write_to_file_fd = sys.stdout else: write_to_file_fd = open(options.ring_delta, 'w') delta.write_to_file(write_to_file_fd, options.fmt) if 'read-from-delta' in actions: if options.ring_delta == '-': print('--ring-delta=- is invalid (read from stdin not supported)') sys.exit(1) try: delta = RingDelta() read_from_delta_fd = open(options.ring_delta, 'r') delta.read_from_file(read_from_delta_fd, options.fmt) except IOError as err: print('ERROR: %s' % err) sys.exit(1) if 'report' in actions: print(delta.get_report(options.detail)) if 'rebalance' in actions: rebalance(delta, rings, options.dry_run)
def main(): parser = OptionParser(usage=usage) parser.add_option('--etc', dest='etc', default=DEFAULT_ETC, help='Overrides /etc/swiftlm (for testing)') parser.add_option('--cloud', dest='cloud', default=None, help='The name of the cloud') parser.add_option('--control-plane', dest='control_plane', default=None, help='The name of the control plane') parser.add_option('--ring-delta', dest='ring_delta', default=None, help='Name of ring-delta file (as output or input' ' A value of "-" (on output means to write' ' to stdout') parser.add_option('--format', dest='fmt', default='yaml', help='One of yaml or json.' ' When used with --ring-delta, specifies the' ' format of the file.') parser.add_option('--detail', dest='detail', default='summary', help='Level of detail to use with --report.' ' Use summary or full') parser.add_option('--report', dest='report', default=False, action="store_true", help='Explain what the ring delta represents.' ' Optionally use --detail.') parser.add_option('--dry-run', dest='dry_run', default=False, action="store_true", help='Show the proposed swift-ring-builder commands') parser.add_option('--pretend-min-part-hours-passed', dest='pretend_min_part_hours_passed', default=False, action="store_true", help='Executes the pretend_min_part_hours_passed command' ' on each ring before running rebalance.' ' Use with caution.') parser.add_option('--make-delta', dest='make_delta', default=False, action="store_true", help='Make a ring delta file') parser.add_option('--rebalance', dest='rebalance', default=False, action="store_true", help='Build (or rebalance) rings') parser.add_option('--limit-ring', dest='limit_ring', default=None, help='Limits actions to given ring') parser.add_option('--size-to-weight', dest='size_to_weight', default=float(1024 * 1024 * 1024), help='Conversion factor for size to weight. Default is' ' 1GB is weight of 1 (a 4Tb drive would be assigned' ' a weight of 4096') parser.add_option('--weight-step', dest='weight_step', default=None, help='When set, weights are changed by at most this' ' value. Overrides value in ring specification.') parser.add_option('--allow-partitions', dest='allow_partitions', default=False, action='store_true', help='Allow devices to be assigned to partitions.' ' Default is to use a full disk drive.') parser.add_option('--stop-on-warnings', dest='stop_on_warnings', default=False, action='store_true', help='Used with --make-delta. Exit with error if there' ' are model missmatch warnings.' ' Default is to only exit with error for errors.') parser.add_option('--unittest', dest='unittest', default=False, action='store_true', help='Set by unittests. Never set on command line.') (options, args) = parser.parse_args() if not (options.cloud and options.control_plane): sys.exit('Must specify both --cloud and --control_plane') sites = CloudMultiSite(options) my_cloud = sites.my_cloud my_control_plane = sites.my_control_plane my_config = sites.path(my_cloud, my_control_plane) # # Work out what we need to do. Validate arguments needed by an action # are present. # actions = [] if options.make_delta: actions.append('init-delta') actions.append('input-from-model') actions.append('read-builder-dir') actions.append('open-osconfig-dir') actions.append('make-delta') actions.append('write-to-delta') if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if options.report: actions.append('init-delta') actions.append('read-from-delta') actions.append('report') if options.detail not in ['summary', 'full']: sys.exit('Invalid value for --detail') if options.rebalance: actions.append('init-delta') actions.append('open-builder-dir') actions.append('read-from-delta') actions.append('rebalance') if options.fmt not in ['yaml', 'json']: print('Invalid value for --format') if len(actions) == 0: sys.exit('Missing an option to perform some action') if options.report and (options.make_delta or options.rebalance): sys.exit('Do not mix --report with other actions') # # Perform actions # if 'init-delta' in actions: delta = RingDelta() if 'input-from-model' in actions: servers_model = ServersModel('unused', 'unused') consumes = Consumes() ring_model = RingSpecifications(my_cloud, my_control_plane) for cloud, control_plane in sites.control_planes(): config = sites.path(cloud, control_plane) input_model_fd = None try: input_model_fd = open(config.get('input-model'), 'r') except IOError as err: pass # File may not exist since its a legacy item try: cp_server_fd = None cp_server_fd = open(config.get('control_plane_servers'), 'r') except IOError as err: sys.exit('Error on control_plane_server.yml: %s' % err) try: control_plane_servers = None if cp_server_fd: control_plane_servers = safe_load(cp_server_fd) except scanner.ScannerError as err: sys.exit('ERROR reading/parsing: %s' % err) try: consumes_fd = open(config.get('swift_ring_builder_consumes'), 'r') except IOError as err: sys.exit('ERROR: %s' % err) try: input_vars = {'global': {}} if input_model_fd: input_vars = safe_load(input_model_fd) consumes_model = safe_load(consumes_fd) except scanner.ScannerError as err: sys.exit('ERROR reading/parsing: %s' % err) try: if control_plane_servers: servers = control_plane_servers.get( 'control_plane_servers') elif input_vars.get('global').get('all_servers'): servers = input_vars.get('global').get('all_servers') else: sys.exit('No servers found in control plane') servers_model.add_servers(cloud, control_plane, servers) consumes.load_model(consumes_model) except SwiftModelException as err: sys.exit(err) servers_model.register_consumes(consumes) try: config_data_fd = open(my_config.get('configuration_data'), 'r') config_data = safe_load(config_data_fd) except (IOError, scanner.ScannerError) as err: sys.exit('Rings should be in configuration-data.' ' Using old configuration processor?') try: rings_loaded = False if input_vars.get('global').get('all_ring_specifications'): # Model contains Ardana old-style rings ring_model = RingSpecifications(my_cloud, my_control_plane, model=input_vars) rings_loaded = True if config_data and config_data.get( 'control_plane_rings', config_data.get('control-plane-rings')): # Model contains new-style rings --- use instead ring_model.load_configuration(my_cloud, my_control_plane, config_data) rings_loaded = True if not rings_loaded: sys.exit('No ring specifications in input model') except SwiftModelException as err: sys.exit(err) if 'open-builder-dir' or 'read-builder-dir' in actions: try: read_rings = False if 'read-builder-dir' in actions: read_rings = True rings = RingBuilder(my_config.get('builder_dir'), read_rings=read_rings) except IOError as err: sys.exit('ERROR: %s' % err) if 'open-osconfig-dir' in actions: drive_configurations = osconfig_load(sites) if 'make-delta' in actions: try: generate_delta(sites, servers_model, ring_model, rings, drive_configurations, options, delta) except SwiftModelException as err: sys.exit('ERROR: %s' % err) if 'write-to-delta' in actions: if my_config.get('ring-delta') == '-': write_to_file_fd = sys.stdout else: write_to_file_fd = open(my_config.get('ring-delta'), 'w') delta.write_to_file(write_to_file_fd, options.fmt) if 'read-from-delta' in actions: if my_config.get('ring-delta') == '-': sys.exit('--ring-delta- is invalid (read from stdin' 'not supported)') try: delta = RingDelta() read_from_delta_fd = open(my_config.get('ring-delta'), 'r') delta.read_from_file(read_from_delta_fd, options.fmt) except IOError as err: sys.exit('ERROR: %s' % err) if 'report' in actions: print(delta.get_report(options)) if 'rebalance' in actions: rebalance(delta, rings, options)