def setup(self): super(RGWMultisiteTests, self).setup() overrides = self.ctx.config.get('overrides', {}) misc.deep_merge(self.config, overrides.get('rgw-multisite-tests', {})) if not self.ctx.rgw_multisite: raise ConfigError( 'rgw-multisite-tests must run after the rgw-multisite task') realm = self.ctx.rgw_multisite.realm master_zone = realm.meta_master_zone() branch = self.config.get('branch') if not branch: # run from suite_path suite_path = self.ctx.config.get('suite_path') self.module_path = suite_path + '/../src/test/rgw/rgw_multi' else: # clone the qa branch repo = self.config.get('repo', teuth_config.get_ceph_qa_suite_git_url()) log.info("cloning suite branch %s from %s...", branch, repo) clonedir = fetch_repo(repo, branch) # import its version of rgw_multi self.module_path = clonedir + '/src/test/rgw/rgw_multi' log.info("importing tests from %s", self.module_path) spec = importlib.util.spec_from_file_location( 'rgw_multi', self.module_path + '/__init__.py') module = importlib.util.module_from_spec(spec) sys.modules[spec.name] = module spec.loader.exec_module(module) from rgw_multi import multisite, tests # create the test user log.info('creating test user..') user = multisite.User('rgw-multisite-test-user') user.create(master_zone, [ '--display-name', 'Multisite Test User', '--gen-access-key', '--gen-secret', '--caps', 'roles=*' ]) config = self.config.get('config', {}) tests.init_multi(realm, user, tests.Config(**config)) tests.realm_meta_checkpoint(realm)
def setup(self): super(RGWMultisiteTests, self).setup() overrides = self.ctx.config.get('overrides', {}) misc.deep_merge(self.config, overrides.get('rgw-multisite-tests', {})) if not self.ctx.rgw_multisite: raise ConfigError('rgw-multisite-tests must run after the rgw-multisite task') realm = self.ctx.rgw_multisite.realm master_zone = realm.meta_master_zone() # create the test user log.info('creating test user..') user = multisite.User('rgw-multisite-test-user') user.create(master_zone, ['--display-name', 'Multisite Test User', '--gen-access-key', '--gen-secret']) config = self.config.get('config', {}) tests.init_multi(realm, user, tests.Config(**config)) tests.realm_meta_checkpoint(realm)
def init(parse_args): cfg = configparser.RawConfigParser({ 'num_zonegroups': 1, 'num_zones': 3, 'num_ps_zones': 0, 'num_az_zones': 0, 'gateways_per_zone': 2, 'no_bootstrap': 'false', 'log_level': 20, 'log_file': None, 'file_log_level': 20, 'tenant': None, 'checkpoint_retries': 60, 'checkpoint_delay': 5, 'reconfigure_delay': 5, 'use_ssl': 'false', }) try: path = os.environ['RGW_MULTI_TEST_CONF'] except KeyError: path = test_path + 'test_multi.conf' try: with open(path) as f: cfg.readfp(f) except: print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable') pass parser = argparse.ArgumentParser( description='Run rgw multi-site tests', usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]') section = 'DEFAULT' parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups')) parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones')) parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone')) parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap')) parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level')) parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file')) parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level')) parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant')) parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries')) parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay')) parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay')) parser.add_argument('--num-ps-zones', type=int, default=cfg.getint(section, 'num_ps_zones')) parser.add_argument('--use-ssl', type=bool, default=cfg.getboolean(section, 'use_ssl')) es_cfg = [] cloud_cfg = [] ps_cfg = [] az_cfg = [] for s in cfg.sections(): if s.startswith('elasticsearch'): es_cfg.append(ESZoneConfig(cfg, s)) elif s.startswith('cloud'): cloud_cfg.append(CloudZoneConfig(cfg, s)) elif s.startswith('pubsub'): ps_cfg.append(PSZoneConfig(cfg, s)) elif s.startswith('archive'): az_cfg.append(AZoneConfig(cfg, s)) argv = [] if parse_args: argv = sys.argv[1:] args = parser.parse_args(argv) bootstrap = not args.no_bootstrap setup_logging(args.log_level, args.log_file, args.file_log_level) # start first cluster c1 = Cluster(cluster_name(1)) if bootstrap: c1.start() clusters = [] clusters.append(c1) admin_creds = gen_credentials() admin_user = multisite.User('zone.user') user_creds = gen_credentials() user = multisite.User('tester', tenant=args.tenant) realm = multisite.Realm('r') if bootstrap: # create the realm on c1 realm.create(c1) else: realm.get(c1) period = multisite.Period(realm=realm) realm.current_period = period num_es_zones = len(es_cfg) num_cloud_zones = len(cloud_cfg) num_ps_zones_from_conf = len(ps_cfg) num_az_zones = cfg.getint(section, 'num_az_zones') num_ps_zones = args.num_ps_zones if num_ps_zones_from_conf == 0 else num_ps_zones_from_conf num_zones = args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones + num_az_zones use_ssl = cfg.getboolean(section, 'use_ssl') if use_ssl and bootstrap: cmd = ['openssl', 'req', '-x509', '-newkey', 'rsa:4096', '-sha256', '-nodes', '-keyout', 'key.pem', '-out', 'cert.pem', '-subj', '/CN=localhost', '-days', '3650'] bash(cmd) # append key to cert fkey = open('./key.pem', 'r') if fkey.mode == 'r': fcert = open('./cert.pem', 'a') fcert.write(fkey.read()) fcert.close() fkey.close() for zg in range(0, args.num_zonegroups): zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period) period.zonegroups.append(zonegroup) is_master_zg = zg == 0 if is_master_zg: period.master_zonegroup = zonegroup for z in range(0, num_zones): is_master = z == 0 # start a cluster, or use c1 for first zone cluster = None if is_master_zg and is_master: cluster = c1 else: cluster = Cluster(cluster_name(len(clusters) + 1)) clusters.append(cluster) if bootstrap: cluster.start() # pull realm configuration from the master's gateway gateway = realm.meta_master_zone().gateways[0] realm.pull(cluster, gateway, admin_creds) endpoints = zone_endpoints(zg, z, args.gateways_per_zone) if is_master: if bootstrap: # create the zonegroup on its first zone's cluster arg = [] if is_master_zg: arg += ['--master'] if len(endpoints): # use master zone's endpoints arg += ['--endpoints', ','.join(endpoints)] zonegroup.create(cluster, arg) else: zonegroup.get(cluster) es_zone = (z >= args.num_zones and z < args.num_zones + num_es_zones) cloud_zone = (z >= args.num_zones + num_es_zones and z < args.num_zones + num_es_zones + num_cloud_zones) ps_zone = (z >= args.num_zones + num_es_zones + num_cloud_zones and z < args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones) az_zone = (z >= args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones) # create the zone in its zonegroup zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster) if es_zone: zone_index = z - args.num_zones zone = ESZone(zone_name(zg, z), es_cfg[zone_index].endpoint, zonegroup, cluster) elif cloud_zone: zone_index = z - args.num_zones - num_es_zones ccfg = cloud_cfg[zone_index] zone = CloudZone(zone_name(zg, z), ccfg.endpoint, ccfg.credentials, ccfg.source_bucket, ccfg.target_path, zonegroup, cluster) elif ps_zone: zone_index = z - args.num_zones - num_es_zones - num_cloud_zones if num_ps_zones_from_conf == 0: zone = PSZone(zone_name(zg, z), zonegroup, cluster) else: pscfg = ps_cfg[zone_index] zone = PSZone(zone_name(zg, z), zonegroup, cluster, full_sync=pscfg.full_sync, retention_days=pscfg.retention_days) elif az_zone: zone_index = z - args.num_zones - num_es_zones - num_cloud_zones - num_ps_zones zone = AZone(zone_name(zg, z), zonegroup, cluster) else: zone = RadosZone(zone_name(zg, z), zonegroup, cluster) if bootstrap: arg = admin_creds.credential_args() if is_master: arg += ['--master'] if len(endpoints): arg += ['--endpoints', ','.join(endpoints)] zone.create(cluster, arg) else: zone.get(cluster) zonegroup.zones.append(zone) if is_master: zonegroup.master_zone = zone zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone) if zone.is_read_only(): zonegroup.ro_zones.append(zone) else: zonegroup.rw_zones.append(zone) # update/commit the period if bootstrap: period.update(zone, commit=True) ssl_port_offset = 1000 # start the gateways for g in range(0, args.gateways_per_zone): port = gateway_port(zg, g + z * args.gateways_per_zone) client_id = gateway_name(zg, z, g) gateway = Gateway(client_id, 'localhost', port, cluster, zone, ssl_port = port+ssl_port_offset if use_ssl else 0) if bootstrap: gateway.start() zone.gateways.append(gateway) if is_master_zg and is_master: if bootstrap: # create admin user arg = ['--display-name', '"Zone User"', '--system'] arg += admin_creds.credential_args() admin_user.create(zone, arg) # create test user arg = ['--display-name', '"Test User"'] arg += user_creds.credential_args() user.create(zone, arg) else: # read users and update keys admin_user.info(zone) admin_creds = admin_user.credentials[0] arg = [] user.info(zone, arg) user_creds = user.credentials[0] if not bootstrap: period.get(c1) config = Config(checkpoint_retries=args.checkpoint_retries, checkpoint_delay=args.checkpoint_delay, reconfigure_delay=args.reconfigure_delay, tenant=args.tenant) init_multi(realm, user, config)
def init(parse_args): cfg = configparser.RawConfigParser({ 'num_zonegroups': 1, 'num_zones': 3, 'num_es_zones': 0, 'gateways_per_zone': 2, 'no_bootstrap': 'false', 'log_level': 20, 'log_file': None, 'file_log_level': 20, 'tenant': None, 'checkpoint_retries': 60, 'checkpoint_delay': 5, 'reconfigure_delay': 5, 'es_endpoint': None, }) try: path = os.environ['RGW_MULTI_TEST_CONF'] except KeyError: path = test_path + 'test_multi.conf' try: with open(path) as f: cfg.readfp(f) except: print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable') pass parser = argparse.ArgumentParser( description='Run rgw multi-site tests', usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]') section = 'DEFAULT' parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups')) parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones')) parser.add_argument('--num-es-zones', type=int, default=cfg.getint(section, 'num_es_zones')) parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone')) parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap')) parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level')) parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file')) parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level')) parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant')) parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries')) parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay')) parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay')) parser.add_argument('--es-endpoint', type=str, default=cfg.get(section, 'es_endpoint')) argv = [] if parse_args: argv = sys.argv[1:] args = parser.parse_args(argv) bootstrap = not args.no_bootstrap # if num_es_zones is defined, need to have es_endpoint defined too assert(args.num_es_zones == 0 or args.es_endpoint) setup_logging(args.log_level, args.log_file, args.file_log_level) # start first cluster c1 = Cluster(cluster_name(1)) if bootstrap: c1.start() clusters = [] clusters.append(c1) admin_creds = gen_credentials() admin_user = multisite.User('zone.user') user_creds = gen_credentials() user = multisite.User('tester') realm = multisite.Realm('r') if bootstrap: # create the realm on c1 realm.create(c1) else: realm.get(c1) period = multisite.Period(realm=realm) realm.current_period = period num_zones = args.num_zones + args.num_es_zones for zg in range(0, args.num_zonegroups): zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period) period.zonegroups.append(zonegroup) is_master_zg = zg == 0 if is_master_zg: period.master_zonegroup = zonegroup for z in range(0, num_zones): is_master = z == 0 # start a cluster, or use c1 for first zone cluster = None if is_master_zg and is_master: cluster = c1 else: cluster = Cluster(cluster_name(len(clusters) + 1)) clusters.append(cluster) if bootstrap: cluster.start() # pull realm configuration from the master's gateway gateway = realm.meta_master_zone().gateways[0] realm.pull(cluster, gateway, admin_creds) endpoints = zone_endpoints(zg, z, args.gateways_per_zone) if is_master: if bootstrap: # create the zonegroup on its first zone's cluster arg = [] if is_master_zg: arg += ['--master'] if len(endpoints): # use master zone's endpoints arg += ['--endpoints', ','.join(endpoints)] zonegroup.create(cluster, arg) else: zonegroup.get(cluster) es_zone = (z >= args.num_zones) # create the zone in its zonegroup zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster) if es_zone: zone = ESZone(zone_name(zg, z), args.es_endpoint, zonegroup, cluster) else: zone = RadosZone(zone_name(zg, z), zonegroup, cluster) if bootstrap: arg = admin_creds.credential_args() if is_master: arg += ['--master'] if len(endpoints): arg += ['--endpoints', ','.join(endpoints)] zone.create(cluster, arg) else: zone.get(cluster) zonegroup.zones.append(zone) if is_master: zonegroup.master_zone = zone zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone) if zone.is_read_only(): zonegroup.ro_zones.append(zone) else: zonegroup.rw_zones.append(zone) # update/commit the period if bootstrap: period.update(zone, commit=True) # start the gateways for g in range(0, args.gateways_per_zone): port = gateway_port(zg, g + z * args.gateways_per_zone) client_id = gateway_name(zg, z, g) gateway = Gateway(client_id, 'localhost', port, cluster, zone) if bootstrap: gateway.start() zone.gateways.append(gateway) if is_master_zg and is_master: if bootstrap: # create admin user arg = ['--display-name', '"Zone User"', '--system'] arg += admin_creds.credential_args() admin_user.create(zone, arg) # create test user arg = ['--display-name', '"Test User"'] arg += user_creds.credential_args() if args.tenant: cmd += ['--tenant', args.tenant] user.create(zone, arg) else: # read users and update keys admin_user.info(zone) admin_creds = admin_user.credentials[0] user.info(zone) user_creds = user.credentials[0] if not bootstrap: period.get(c1) config = Config(checkpoint_retries=args.checkpoint_retries, checkpoint_delay=args.checkpoint_delay, reconfigure_delay=args.reconfigure_delay) init_multi(realm, user, config)
def setup(self): super(RGWMultisite, self).setup() overrides = self.ctx.config.get('overrides', {}) misc.deep_merge(self.config, overrides.get('rgw-multisite', {})) if not self.ctx.rgw: raise ConfigError('rgw-multisite must run after the rgw task') role_endpoints = self.ctx.rgw.role_endpoints # construct Clusters and Gateways for each client in the rgw task clusters, gateways = extract_clusters_and_gateways( self.ctx, role_endpoints) # get the master zone and zonegroup configuration mz, mzg = extract_master_zone_zonegroup(self.config['zonegroups']) cluster1 = cluster_for_zone(clusters, mz) # create the realm and period on the master zone's cluster log.info('creating realm..') realm = create_realm(cluster1, self.config['realm']) period = realm.current_period creds = gen_credentials() # create the master zonegroup and its master zone log.info('creating master zonegroup..') master_zonegroup = create_zonegroup(cluster1, gateways, period, deepcopy(mzg)) period.master_zonegroup = master_zonegroup log.info('creating master zone..') master_zone = create_zone(self.ctx, cluster1, gateways, creds, master_zonegroup, deepcopy(mz)) master_zonegroup.master_zone = master_zone period.update(master_zone, commit=True) restart_zone_gateways(master_zone) # restart with --rgw-zone # create the admin user on the master zone log.info('creating admin user..') user_args = ['--display-name', 'Realm Admin', '--system'] user_args += creds.credential_args() admin_user = multisite.User('realm-admin') admin_user.create(master_zone, user_args) # process 'zonegroups' for zg_config in self.config['zonegroups']: zones_config = zg_config.pop('zones') zonegroup = None for zone_config in zones_config: # get the cluster for this zone cluster = cluster_for_zone(clusters, zone_config) if cluster != cluster1: # already created on master cluster log.info('pulling realm configuration to %s', cluster.name) realm.pull(cluster, master_zone.gateways[0], creds) # use the first zone's cluster to create the zonegroup if not zonegroup: if zg_config['name'] == master_zonegroup.name: zonegroup = master_zonegroup else: log.info('creating zonegroup..') zonegroup = create_zonegroup(cluster, gateways, period, zg_config) if zone_config['name'] == master_zone.name: # master zone was already created zone = master_zone else: # create the zone and commit the period log.info('creating zone..') zone = create_zone(self.ctx, cluster, gateways, creds, zonegroup, zone_config) period.update(zone, commit=True) restart_zone_gateways(zone) # restart with --rgw-zone # attach configuration to the ctx for other tasks self.ctx.rgw_multisite = argparse.Namespace() self.ctx.rgw_multisite.clusters = clusters self.ctx.rgw_multisite.gateways = gateways self.ctx.rgw_multisite.realm = realm self.ctx.rgw_multisite.admin_user = admin_user log.info('rgw multisite configuration completed')