def setup(self):
super(RGWMultisiteTests, self).setup()
overrides = self.ctx.config.get('overrides', {})
misc.deep_merge(self.config, overrides.get('rgw-multisite-tests', {}))
if not self.ctx.rgw_multisite:
raise ConfigError(
'rgw-multisite-tests must run after the rgw-multisite task')
realm = self.ctx.rgw_multisite.realm
master_zone = realm.meta_master_zone()
branch = self.config.get('branch')
if not branch:
# run from suite_path
suite_path = self.ctx.config.get('suite_path')
self.module_path = suite_path + '/../src/test/rgw/rgw_multi'
else:
# clone the qa branch
repo = self.config.get('repo',
teuth_config.get_ceph_qa_suite_git_url())
log.info("cloning suite branch %s from %s...", branch, repo)
clonedir = fetch_repo(repo, branch)
# import its version of rgw_multi
self.module_path = clonedir + '/src/test/rgw/rgw_multi'
log.info("importing tests from %s", self.module_path)
spec = importlib.util.spec_from_file_location(
'rgw_multi', self.module_path + '/__init__.py')
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
from rgw_multi import multisite, tests
# create the test user
log.info('creating test user..')
user = multisite.User('rgw-multisite-test-user')
user.create(master_zone, [
'--display-name', 'Multisite Test User', '--gen-access-key',
'--gen-secret', '--caps', 'roles=*'
])
config = self.config.get('config', {})
tests.init_multi(realm, user, tests.Config(**config))
tests.realm_meta_checkpoint(realm)
def setup(self):
super(RGWMultisiteTests, self).setup()
overrides = self.ctx.config.get('overrides', {})
misc.deep_merge(self.config, overrides.get('rgw-multisite-tests', {}))
if not self.ctx.rgw_multisite:
raise ConfigError('rgw-multisite-tests must run after the rgw-multisite task')
realm = self.ctx.rgw_multisite.realm
master_zone = realm.meta_master_zone()
# create the test user
log.info('creating test user..')
user = multisite.User('rgw-multisite-test-user')
user.create(master_zone, ['--display-name', 'Multisite Test User',
'--gen-access-key', '--gen-secret'])
config = self.config.get('config', {})
tests.init_multi(realm, user, tests.Config(**config))
tests.realm_meta_checkpoint(realm)
def init(parse_args):
cfg = configparser.RawConfigParser({
'num_zonegroups': 1,
'num_zones': 3,
'num_ps_zones': 0,
'num_az_zones': 0,
'gateways_per_zone': 2,
'no_bootstrap': 'false',
'log_level': 20,
'log_file': None,
'file_log_level': 20,
'tenant': None,
'checkpoint_retries': 60,
'checkpoint_delay': 5,
'reconfigure_delay': 5,
'use_ssl': 'false',
})
try:
path = os.environ['RGW_MULTI_TEST_CONF']
except KeyError:
path = test_path + 'test_multi.conf'
try:
with open(path) as f:
cfg.readfp(f)
except:
print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
pass
parser = argparse.ArgumentParser(
description='Run rgw multi-site tests',
usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')
section = 'DEFAULT'
parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries'))
parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay'))
parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay'))
parser.add_argument('--num-ps-zones', type=int, default=cfg.getint(section, 'num_ps_zones'))
parser.add_argument('--use-ssl', type=bool, default=cfg.getboolean(section, 'use_ssl'))
es_cfg = []
cloud_cfg = []
ps_cfg = []
az_cfg = []
for s in cfg.sections():
if s.startswith('elasticsearch'):
es_cfg.append(ESZoneConfig(cfg, s))
elif s.startswith('cloud'):
cloud_cfg.append(CloudZoneConfig(cfg, s))
elif s.startswith('pubsub'):
ps_cfg.append(PSZoneConfig(cfg, s))
elif s.startswith('archive'):
az_cfg.append(AZoneConfig(cfg, s))
argv = []
if parse_args:
argv = sys.argv[1:]
args = parser.parse_args(argv)
bootstrap = not args.no_bootstrap
setup_logging(args.log_level, args.log_file, args.file_log_level)
# start first cluster
c1 = Cluster(cluster_name(1))
if bootstrap:
c1.start()
clusters = []
clusters.append(c1)
admin_creds = gen_credentials()
admin_user = multisite.User('zone.user')
user_creds = gen_credentials()
user = multisite.User('tester', tenant=args.tenant)
realm = multisite.Realm('r')
if bootstrap:
# create the realm on c1
realm.create(c1)
else:
realm.get(c1)
period = multisite.Period(realm=realm)
realm.current_period = period
num_es_zones = len(es_cfg)
num_cloud_zones = len(cloud_cfg)
num_ps_zones_from_conf = len(ps_cfg)
num_az_zones = cfg.getint(section, 'num_az_zones')
num_ps_zones = args.num_ps_zones if num_ps_zones_from_conf == 0 else num_ps_zones_from_conf
num_zones = args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones + num_az_zones
use_ssl = cfg.getboolean(section, 'use_ssl')
if use_ssl and bootstrap:
cmd = ['openssl', 'req',
'-x509',
'-newkey', 'rsa:4096',
'-sha256',
'-nodes',
'-keyout', 'key.pem',
'-out', 'cert.pem',
'-subj', '/CN=localhost',
'-days', '3650']
bash(cmd)
# append key to cert
fkey = open('./key.pem', 'r')
if fkey.mode == 'r':
fcert = open('./cert.pem', 'a')
fcert.write(fkey.read())
fcert.close()
fkey.close()
for zg in range(0, args.num_zonegroups):
zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
period.zonegroups.append(zonegroup)
is_master_zg = zg == 0
if is_master_zg:
period.master_zonegroup = zonegroup
for z in range(0, num_zones):
is_master = z == 0
# start a cluster, or use c1 for first zone
cluster = None
if is_master_zg and is_master:
cluster = c1
else:
cluster = Cluster(cluster_name(len(clusters) + 1))
clusters.append(cluster)
if bootstrap:
cluster.start()
# pull realm configuration from the master's gateway
gateway = realm.meta_master_zone().gateways[0]
realm.pull(cluster, gateway, admin_creds)
endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
if is_master:
if bootstrap:
# create the zonegroup on its first zone's cluster
arg = []
if is_master_zg:
arg += ['--master']
if len(endpoints): # use master zone's endpoints
arg += ['--endpoints', ','.join(endpoints)]
zonegroup.create(cluster, arg)
else:
zonegroup.get(cluster)
es_zone = (z >= args.num_zones and z < args.num_zones + num_es_zones)
cloud_zone = (z >= args.num_zones + num_es_zones and z < args.num_zones + num_es_zones + num_cloud_zones)
ps_zone = (z >= args.num_zones + num_es_zones + num_cloud_zones and z < args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones)
az_zone = (z >= args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones)
# create the zone in its zonegroup
zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
if es_zone:
zone_index = z - args.num_zones
zone = ESZone(zone_name(zg, z), es_cfg[zone_index].endpoint, zonegroup, cluster)
elif cloud_zone:
zone_index = z - args.num_zones - num_es_zones
ccfg = cloud_cfg[zone_index]
zone = CloudZone(zone_name(zg, z), ccfg.endpoint, ccfg.credentials, ccfg.source_bucket,
ccfg.target_path, zonegroup, cluster)
elif ps_zone:
zone_index = z - args.num_zones - num_es_zones - num_cloud_zones
if num_ps_zones_from_conf == 0:
zone = PSZone(zone_name(zg, z), zonegroup, cluster)
else:
pscfg = ps_cfg[zone_index]
zone = PSZone(zone_name(zg, z), zonegroup, cluster,
full_sync=pscfg.full_sync, retention_days=pscfg.retention_days)
elif az_zone:
zone_index = z - args.num_zones - num_es_zones - num_cloud_zones - num_ps_zones
zone = AZone(zone_name(zg, z), zonegroup, cluster)
else:
zone = RadosZone(zone_name(zg, z), zonegroup, cluster)
if bootstrap:
arg = admin_creds.credential_args()
if is_master:
arg += ['--master']
if len(endpoints):
arg += ['--endpoints', ','.join(endpoints)]
zone.create(cluster, arg)
else:
zone.get(cluster)
zonegroup.zones.append(zone)
if is_master:
zonegroup.master_zone = zone
zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
if zone.is_read_only():
zonegroup.ro_zones.append(zone)
else:
zonegroup.rw_zones.append(zone)
# update/commit the period
if bootstrap:
period.update(zone, commit=True)
ssl_port_offset = 1000
# start the gateways
for g in range(0, args.gateways_per_zone):
port = gateway_port(zg, g + z * args.gateways_per_zone)
client_id = gateway_name(zg, z, g)
gateway = Gateway(client_id, 'localhost', port, cluster, zone,
ssl_port = port+ssl_port_offset if use_ssl else 0)
if bootstrap:
gateway.start()
zone.gateways.append(gateway)
if is_master_zg and is_master:
if bootstrap:
# create admin user
arg = ['--display-name', '"Zone User"', '--system']
arg += admin_creds.credential_args()
admin_user.create(zone, arg)
# create test user
arg = ['--display-name', '"Test User"']
arg += user_creds.credential_args()
user.create(zone, arg)
else:
# read users and update keys
admin_user.info(zone)
admin_creds = admin_user.credentials[0]
arg = []
user.info(zone, arg)
user_creds = user.credentials[0]
if not bootstrap:
period.get(c1)
config = Config(checkpoint_retries=args.checkpoint_retries,
checkpoint_delay=args.checkpoint_delay,
reconfigure_delay=args.reconfigure_delay,
tenant=args.tenant)
init_multi(realm, user, config)
def init(parse_args):
cfg = configparser.RawConfigParser({
'num_zonegroups': 1,
'num_zones': 3,
'num_es_zones': 0,
'gateways_per_zone': 2,
'no_bootstrap': 'false',
'log_level': 20,
'log_file': None,
'file_log_level': 20,
'tenant': None,
'checkpoint_retries': 60,
'checkpoint_delay': 5,
'reconfigure_delay': 5,
'es_endpoint': None,
})
try:
path = os.environ['RGW_MULTI_TEST_CONF']
except KeyError:
path = test_path + 'test_multi.conf'
try:
with open(path) as f:
cfg.readfp(f)
except:
print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
pass
parser = argparse.ArgumentParser(
description='Run rgw multi-site tests',
usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')
section = 'DEFAULT'
parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
parser.add_argument('--num-es-zones', type=int, default=cfg.getint(section, 'num_es_zones'))
parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries'))
parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay'))
parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay'))
parser.add_argument('--es-endpoint', type=str, default=cfg.get(section, 'es_endpoint'))
argv = []
if parse_args:
argv = sys.argv[1:]
args = parser.parse_args(argv)
bootstrap = not args.no_bootstrap
# if num_es_zones is defined, need to have es_endpoint defined too
assert(args.num_es_zones == 0 or args.es_endpoint)
setup_logging(args.log_level, args.log_file, args.file_log_level)
# start first cluster
c1 = Cluster(cluster_name(1))
if bootstrap:
c1.start()
clusters = []
clusters.append(c1)
admin_creds = gen_credentials()
admin_user = multisite.User('zone.user')
user_creds = gen_credentials()
user = multisite.User('tester')
realm = multisite.Realm('r')
if bootstrap:
# create the realm on c1
realm.create(c1)
else:
realm.get(c1)
period = multisite.Period(realm=realm)
realm.current_period = period
num_zones = args.num_zones + args.num_es_zones
for zg in range(0, args.num_zonegroups):
zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
period.zonegroups.append(zonegroup)
is_master_zg = zg == 0
if is_master_zg:
period.master_zonegroup = zonegroup
for z in range(0, num_zones):
is_master = z == 0
# start a cluster, or use c1 for first zone
cluster = None
if is_master_zg and is_master:
cluster = c1
else:
cluster = Cluster(cluster_name(len(clusters) + 1))
clusters.append(cluster)
if bootstrap:
cluster.start()
# pull realm configuration from the master's gateway
gateway = realm.meta_master_zone().gateways[0]
realm.pull(cluster, gateway, admin_creds)
endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
if is_master:
if bootstrap:
# create the zonegroup on its first zone's cluster
arg = []
if is_master_zg:
arg += ['--master']
if len(endpoints): # use master zone's endpoints
arg += ['--endpoints', ','.join(endpoints)]
zonegroup.create(cluster, arg)
else:
zonegroup.get(cluster)
es_zone = (z >= args.num_zones)
# create the zone in its zonegroup
zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
if es_zone:
zone = ESZone(zone_name(zg, z), args.es_endpoint, zonegroup, cluster)
else:
zone = RadosZone(zone_name(zg, z), zonegroup, cluster)
if bootstrap:
arg = admin_creds.credential_args()
if is_master:
arg += ['--master']
if len(endpoints):
arg += ['--endpoints', ','.join(endpoints)]
zone.create(cluster, arg)
else:
zone.get(cluster)
zonegroup.zones.append(zone)
if is_master:
zonegroup.master_zone = zone
zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
if zone.is_read_only():
zonegroup.ro_zones.append(zone)
else:
zonegroup.rw_zones.append(zone)
# update/commit the period
if bootstrap:
period.update(zone, commit=True)
# start the gateways
for g in range(0, args.gateways_per_zone):
port = gateway_port(zg, g + z * args.gateways_per_zone)
client_id = gateway_name(zg, z, g)
gateway = Gateway(client_id, 'localhost', port, cluster, zone)
if bootstrap:
gateway.start()
zone.gateways.append(gateway)
if is_master_zg and is_master:
if bootstrap:
# create admin user
arg = ['--display-name', '"Zone User"', '--system']
arg += admin_creds.credential_args()
admin_user.create(zone, arg)
# create test user
arg = ['--display-name', '"Test User"']
arg += user_creds.credential_args()
if args.tenant:
cmd += ['--tenant', args.tenant]
user.create(zone, arg)
else:
# read users and update keys
admin_user.info(zone)
admin_creds = admin_user.credentials[0]
user.info(zone)
user_creds = user.credentials[0]
if not bootstrap:
period.get(c1)
config = Config(checkpoint_retries=args.checkpoint_retries,
checkpoint_delay=args.checkpoint_delay,
reconfigure_delay=args.reconfigure_delay)
init_multi(realm, user, config)
def setup(self):
super(RGWMultisite, self).setup()
overrides = self.ctx.config.get('overrides', {})
misc.deep_merge(self.config, overrides.get('rgw-multisite', {}))
if not self.ctx.rgw:
raise ConfigError('rgw-multisite must run after the rgw task')
role_endpoints = self.ctx.rgw.role_endpoints
# construct Clusters and Gateways for each client in the rgw task
clusters, gateways = extract_clusters_and_gateways(
self.ctx, role_endpoints)
# get the master zone and zonegroup configuration
mz, mzg = extract_master_zone_zonegroup(self.config['zonegroups'])
cluster1 = cluster_for_zone(clusters, mz)
# create the realm and period on the master zone's cluster
log.info('creating realm..')
realm = create_realm(cluster1, self.config['realm'])
period = realm.current_period
creds = gen_credentials()
# create the master zonegroup and its master zone
log.info('creating master zonegroup..')
master_zonegroup = create_zonegroup(cluster1, gateways, period,
deepcopy(mzg))
period.master_zonegroup = master_zonegroup
log.info('creating master zone..')
master_zone = create_zone(self.ctx, cluster1, gateways, creds,
master_zonegroup, deepcopy(mz))
master_zonegroup.master_zone = master_zone
period.update(master_zone, commit=True)
restart_zone_gateways(master_zone) # restart with --rgw-zone
# create the admin user on the master zone
log.info('creating admin user..')
user_args = ['--display-name', 'Realm Admin', '--system']
user_args += creds.credential_args()
admin_user = multisite.User('realm-admin')
admin_user.create(master_zone, user_args)
# process 'zonegroups'
for zg_config in self.config['zonegroups']:
zones_config = zg_config.pop('zones')
zonegroup = None
for zone_config in zones_config:
# get the cluster for this zone
cluster = cluster_for_zone(clusters, zone_config)
if cluster != cluster1: # already created on master cluster
log.info('pulling realm configuration to %s', cluster.name)
realm.pull(cluster, master_zone.gateways[0], creds)
# use the first zone's cluster to create the zonegroup
if not zonegroup:
if zg_config['name'] == master_zonegroup.name:
zonegroup = master_zonegroup
else:
log.info('creating zonegroup..')
zonegroup = create_zonegroup(cluster, gateways, period,
zg_config)
if zone_config['name'] == master_zone.name:
# master zone was already created
zone = master_zone
else:
# create the zone and commit the period
log.info('creating zone..')
zone = create_zone(self.ctx, cluster, gateways, creds,
zonegroup, zone_config)
period.update(zone, commit=True)
restart_zone_gateways(zone) # restart with --rgw-zone
# attach configuration to the ctx for other tasks
self.ctx.rgw_multisite = argparse.Namespace()
self.ctx.rgw_multisite.clusters = clusters
self.ctx.rgw_multisite.gateways = gateways
self.ctx.rgw_multisite.realm = realm
self.ctx.rgw_multisite.admin_user = admin_user
log.info('rgw multisite configuration completed')
