def create_zone(ctx, cluster, gateways, creds, zonegroup, config):
""" create a zone with the given configuration """
zone = multisite.Zone(config['name'], zonegroup, cluster)
if config.pop('is_pubsub', False):
zone = PSZone(config['name'], zonegroup, cluster)
else:
zone = RadosZone(config['name'], zonegroup, cluster)
# collect Gateways for the zone's endpoints
endpoints = config.get('endpoints')
if not endpoints:
raise ConfigError('no \'endpoints\' for zone %s' % config['name'])
zone.gateways = [gateways[role] for role in endpoints]
for gateway in zone.gateways:
gateway.set_zone(zone)
# format the gateway endpoints
endpoints = [g.endpoint() for g in zone.gateways]
args = is_default_arg(config)
args += is_master_arg(config)
args += creds.credential_args()
if len(endpoints):
args += ['--endpoints', ','.join(endpoints)]
zone.create(cluster, args)
zonegroup.zones.append(zone)
create_zone_pools(ctx, zone)
if ctx.rgw.compression_type:
configure_zone_compression(zone, ctx.rgw.compression_type)
zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
if zone.is_read_only():
zonegroup.ro_zones.append(zone)
else:
zonegroup.rw_zones.append(zone)
return zone
def create_zone(ctx, cluster, gateways, creds, zonegroup, config):
""" create a zone with the given configuration """
zone = multisite.Zone(config['name'], zonegroup, cluster)
zone = RadosZone(config['name'], zonegroup, cluster)
# collect Gateways for the zone's endpoints
endpoints = config.get('endpoints')
if not endpoints:
raise ConfigError('no \'endpoints\' for zone %s' % config['name'])
zone.gateways = [gateways[role] for role in endpoints]
for gateway in zone.gateways:
gateway.set_zone(zone)
# format the gateway endpoints
endpoints = [g.endpoint() for g in zone.gateways]
args = is_default_arg(config)
args += is_master_arg(config)
args += creds.credential_args()
if len(endpoints):
args += ['--endpoints', ','.join(endpoints)]
zone.create(cluster, args)
zonegroup.zones.append(zone)
create_zone_pools(ctx, zone)
if ctx.rgw.compression_type:
configure_zone_compression(zone, ctx.rgw.compression_type)
zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
if zone.is_read_only():
zonegroup.ro_zones.append(zone)
else:
zonegroup.rw_zones.append(zone)
return zone
def init(parse_args):
cfg = configparser.RawConfigParser({
'num_zonegroups': 1,
'num_zones': 3,
'num_ps_zones': 0,
'num_az_zones': 0,
'gateways_per_zone': 2,
'no_bootstrap': 'false',
'log_level': 20,
'log_file': None,
'file_log_level': 20,
'tenant': None,
'checkpoint_retries': 60,
'checkpoint_delay': 5,
'reconfigure_delay': 5,
'use_ssl': 'false',
})
try:
path = os.environ['RGW_MULTI_TEST_CONF']
except KeyError:
path = test_path + 'test_multi.conf'
try:
with open(path) as f:
cfg.readfp(f)
except:
print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
pass
parser = argparse.ArgumentParser(
description='Run rgw multi-site tests',
usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')
section = 'DEFAULT'
parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries'))
parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay'))
parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay'))
parser.add_argument('--num-ps-zones', type=int, default=cfg.getint(section, 'num_ps_zones'))
parser.add_argument('--use-ssl', type=bool, default=cfg.getboolean(section, 'use_ssl'))
es_cfg = []
cloud_cfg = []
ps_cfg = []
az_cfg = []
for s in cfg.sections():
if s.startswith('elasticsearch'):
es_cfg.append(ESZoneConfig(cfg, s))
elif s.startswith('cloud'):
cloud_cfg.append(CloudZoneConfig(cfg, s))
elif s.startswith('pubsub'):
ps_cfg.append(PSZoneConfig(cfg, s))
elif s.startswith('archive'):
az_cfg.append(AZoneConfig(cfg, s))
argv = []
if parse_args:
argv = sys.argv[1:]
args = parser.parse_args(argv)
bootstrap = not args.no_bootstrap
setup_logging(args.log_level, args.log_file, args.file_log_level)
# start first cluster
c1 = Cluster(cluster_name(1))
if bootstrap:
c1.start()
clusters = []
clusters.append(c1)
admin_creds = gen_credentials()
admin_user = multisite.User('zone.user')
user_creds = gen_credentials()
user = multisite.User('tester', tenant=args.tenant)
realm = multisite.Realm('r')
if bootstrap:
# create the realm on c1
realm.create(c1)
else:
realm.get(c1)
period = multisite.Period(realm=realm)
realm.current_period = period
num_es_zones = len(es_cfg)
num_cloud_zones = len(cloud_cfg)
num_ps_zones_from_conf = len(ps_cfg)
num_az_zones = cfg.getint(section, 'num_az_zones')
num_ps_zones = args.num_ps_zones if num_ps_zones_from_conf == 0 else num_ps_zones_from_conf
num_zones = args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones + num_az_zones
use_ssl = cfg.getboolean(section, 'use_ssl')
if use_ssl and bootstrap:
cmd = ['openssl', 'req',
'-x509',
'-newkey', 'rsa:4096',
'-sha256',
'-nodes',
'-keyout', 'key.pem',
'-out', 'cert.pem',
'-subj', '/CN=localhost',
'-days', '3650']
bash(cmd)
# append key to cert
fkey = open('./key.pem', 'r')
if fkey.mode == 'r':
fcert = open('./cert.pem', 'a')
fcert.write(fkey.read())
fcert.close()
fkey.close()
for zg in range(0, args.num_zonegroups):
zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
period.zonegroups.append(zonegroup)
is_master_zg = zg == 0
if is_master_zg:
period.master_zonegroup = zonegroup
for z in range(0, num_zones):
is_master = z == 0
# start a cluster, or use c1 for first zone
cluster = None
if is_master_zg and is_master:
cluster = c1
else:
cluster = Cluster(cluster_name(len(clusters) + 1))
clusters.append(cluster)
if bootstrap:
cluster.start()
# pull realm configuration from the master's gateway
gateway = realm.meta_master_zone().gateways[0]
realm.pull(cluster, gateway, admin_creds)
endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
if is_master:
if bootstrap:
# create the zonegroup on its first zone's cluster
arg = []
if is_master_zg:
arg += ['--master']
if len(endpoints): # use master zone's endpoints
arg += ['--endpoints', ','.join(endpoints)]
zonegroup.create(cluster, arg)
else:
zonegroup.get(cluster)
es_zone = (z >= args.num_zones and z < args.num_zones + num_es_zones)
cloud_zone = (z >= args.num_zones + num_es_zones and z < args.num_zones + num_es_zones + num_cloud_zones)
ps_zone = (z >= args.num_zones + num_es_zones + num_cloud_zones and z < args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones)
az_zone = (z >= args.num_zones + num_es_zones + num_cloud_zones + num_ps_zones)
# create the zone in its zonegroup
zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
if es_zone:
zone_index = z - args.num_zones
zone = ESZone(zone_name(zg, z), es_cfg[zone_index].endpoint, zonegroup, cluster)
elif cloud_zone:
zone_index = z - args.num_zones - num_es_zones
ccfg = cloud_cfg[zone_index]
zone = CloudZone(zone_name(zg, z), ccfg.endpoint, ccfg.credentials, ccfg.source_bucket,
ccfg.target_path, zonegroup, cluster)
elif ps_zone:
zone_index = z - args.num_zones - num_es_zones - num_cloud_zones
if num_ps_zones_from_conf == 0:
zone = PSZone(zone_name(zg, z), zonegroup, cluster)
else:
pscfg = ps_cfg[zone_index]
zone = PSZone(zone_name(zg, z), zonegroup, cluster,
full_sync=pscfg.full_sync, retention_days=pscfg.retention_days)
elif az_zone:
zone_index = z - args.num_zones - num_es_zones - num_cloud_zones - num_ps_zones
zone = AZone(zone_name(zg, z), zonegroup, cluster)
else:
zone = RadosZone(zone_name(zg, z), zonegroup, cluster)
if bootstrap:
arg = admin_creds.credential_args()
if is_master:
arg += ['--master']
if len(endpoints):
arg += ['--endpoints', ','.join(endpoints)]
zone.create(cluster, arg)
else:
zone.get(cluster)
zonegroup.zones.append(zone)
if is_master:
zonegroup.master_zone = zone
zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
if zone.is_read_only():
zonegroup.ro_zones.append(zone)
else:
zonegroup.rw_zones.append(zone)
# update/commit the period
if bootstrap:
period.update(zone, commit=True)
ssl_port_offset = 1000
# start the gateways
for g in range(0, args.gateways_per_zone):
port = gateway_port(zg, g + z * args.gateways_per_zone)
client_id = gateway_name(zg, z, g)
gateway = Gateway(client_id, 'localhost', port, cluster, zone,
ssl_port = port+ssl_port_offset if use_ssl else 0)
if bootstrap:
gateway.start()
zone.gateways.append(gateway)
if is_master_zg and is_master:
if bootstrap:
# create admin user
arg = ['--display-name', '"Zone User"', '--system']
arg += admin_creds.credential_args()
admin_user.create(zone, arg)
# create test user
arg = ['--display-name', '"Test User"']
arg += user_creds.credential_args()
user.create(zone, arg)
else:
# read users and update keys
admin_user.info(zone)
admin_creds = admin_user.credentials[0]
arg = []
user.info(zone, arg)
user_creds = user.credentials[0]
if not bootstrap:
period.get(c1)
config = Config(checkpoint_retries=args.checkpoint_retries,
checkpoint_delay=args.checkpoint_delay,
reconfigure_delay=args.reconfigure_delay,
tenant=args.tenant)
init_multi(realm, user, config)
def init(parse_args):
cfg = configparser.RawConfigParser({
'num_zonegroups': 1,
'num_zones': 3,
'num_es_zones': 0,
'gateways_per_zone': 2,
'no_bootstrap': 'false',
'log_level': 20,
'log_file': None,
'file_log_level': 20,
'tenant': None,
'checkpoint_retries': 60,
'checkpoint_delay': 5,
'reconfigure_delay': 5,
'es_endpoint': None,
})
try:
path = os.environ['RGW_MULTI_TEST_CONF']
except KeyError:
path = test_path + 'test_multi.conf'
try:
with open(path) as f:
cfg.readfp(f)
except:
print('WARNING: error reading test config. Path can be set through the RGW_MULTI_TEST_CONF env variable')
pass
parser = argparse.ArgumentParser(
description='Run rgw multi-site tests',
usage='test_multi [--num-zonegroups <num>] [--num-zones <num>] [--no-bootstrap]')
section = 'DEFAULT'
parser.add_argument('--num-zonegroups', type=int, default=cfg.getint(section, 'num_zonegroups'))
parser.add_argument('--num-zones', type=int, default=cfg.getint(section, 'num_zones'))
parser.add_argument('--num-es-zones', type=int, default=cfg.getint(section, 'num_es_zones'))
parser.add_argument('--gateways-per-zone', type=int, default=cfg.getint(section, 'gateways_per_zone'))
parser.add_argument('--no-bootstrap', action='store_true', default=cfg.getboolean(section, 'no_bootstrap'))
parser.add_argument('--log-level', type=int, default=cfg.getint(section, 'log_level'))
parser.add_argument('--log-file', type=str, default=cfg.get(section, 'log_file'))
parser.add_argument('--file-log-level', type=int, default=cfg.getint(section, 'file_log_level'))
parser.add_argument('--tenant', type=str, default=cfg.get(section, 'tenant'))
parser.add_argument('--checkpoint-retries', type=int, default=cfg.getint(section, 'checkpoint_retries'))
parser.add_argument('--checkpoint-delay', type=int, default=cfg.getint(section, 'checkpoint_delay'))
parser.add_argument('--reconfigure-delay', type=int, default=cfg.getint(section, 'reconfigure_delay'))
parser.add_argument('--es-endpoint', type=str, default=cfg.get(section, 'es_endpoint'))
argv = []
if parse_args:
argv = sys.argv[1:]
args = parser.parse_args(argv)
bootstrap = not args.no_bootstrap
# if num_es_zones is defined, need to have es_endpoint defined too
assert(args.num_es_zones == 0 or args.es_endpoint)
setup_logging(args.log_level, args.log_file, args.file_log_level)
# start first cluster
c1 = Cluster(cluster_name(1))
if bootstrap:
c1.start()
clusters = []
clusters.append(c1)
admin_creds = gen_credentials()
admin_user = multisite.User('zone.user')
user_creds = gen_credentials()
user = multisite.User('tester')
realm = multisite.Realm('r')
if bootstrap:
# create the realm on c1
realm.create(c1)
else:
realm.get(c1)
period = multisite.Period(realm=realm)
realm.current_period = period
num_zones = args.num_zones + args.num_es_zones
for zg in range(0, args.num_zonegroups):
zonegroup = multisite.ZoneGroup(zonegroup_name(zg), period)
period.zonegroups.append(zonegroup)
is_master_zg = zg == 0
if is_master_zg:
period.master_zonegroup = zonegroup
for z in range(0, num_zones):
is_master = z == 0
# start a cluster, or use c1 for first zone
cluster = None
if is_master_zg and is_master:
cluster = c1
else:
cluster = Cluster(cluster_name(len(clusters) + 1))
clusters.append(cluster)
if bootstrap:
cluster.start()
# pull realm configuration from the master's gateway
gateway = realm.meta_master_zone().gateways[0]
realm.pull(cluster, gateway, admin_creds)
endpoints = zone_endpoints(zg, z, args.gateways_per_zone)
if is_master:
if bootstrap:
# create the zonegroup on its first zone's cluster
arg = []
if is_master_zg:
arg += ['--master']
if len(endpoints): # use master zone's endpoints
arg += ['--endpoints', ','.join(endpoints)]
zonegroup.create(cluster, arg)
else:
zonegroup.get(cluster)
es_zone = (z >= args.num_zones)
# create the zone in its zonegroup
zone = multisite.Zone(zone_name(zg, z), zonegroup, cluster)
if es_zone:
zone = ESZone(zone_name(zg, z), args.es_endpoint, zonegroup, cluster)
else:
zone = RadosZone(zone_name(zg, z), zonegroup, cluster)
if bootstrap:
arg = admin_creds.credential_args()
if is_master:
arg += ['--master']
if len(endpoints):
arg += ['--endpoints', ','.join(endpoints)]
zone.create(cluster, arg)
else:
zone.get(cluster)
zonegroup.zones.append(zone)
if is_master:
zonegroup.master_zone = zone
zonegroup.zones_by_type.setdefault(zone.tier_type(), []).append(zone)
if zone.is_read_only():
zonegroup.ro_zones.append(zone)
else:
zonegroup.rw_zones.append(zone)
# update/commit the period
if bootstrap:
period.update(zone, commit=True)
# start the gateways
for g in range(0, args.gateways_per_zone):
port = gateway_port(zg, g + z * args.gateways_per_zone)
client_id = gateway_name(zg, z, g)
gateway = Gateway(client_id, 'localhost', port, cluster, zone)
if bootstrap:
gateway.start()
zone.gateways.append(gateway)
if is_master_zg and is_master:
if bootstrap:
# create admin user
arg = ['--display-name', '"Zone User"', '--system']
arg += admin_creds.credential_args()
admin_user.create(zone, arg)
# create test user
arg = ['--display-name', '"Test User"']
arg += user_creds.credential_args()
if args.tenant:
cmd += ['--tenant', args.tenant]
user.create(zone, arg)
else:
# read users and update keys
admin_user.info(zone)
admin_creds = admin_user.credentials[0]
user.info(zone)
user_creds = user.credentials[0]
if not bootstrap:
period.get(c1)
config = Config(checkpoint_retries=args.checkpoint_retries,
checkpoint_delay=args.checkpoint_delay,
reconfigure_delay=args.reconfigure_delay)
init_multi(realm, user, config)