def run_radosgw_agent(ctx, config):
"""
Run a single radosgw-agent. See task() for config format.
"""
return_list = list()
for (client, cconf) in config.items():
# don't process entries that are not clients
if not client.startswith('client.'):
log.debug('key {data} does not start with \'client.\', moving on'.format(
data=client))
continue
src_client = cconf['src']
dest_client = cconf['dest']
src_zone = rgw_utils.zone_for_client(ctx, src_client)
dest_zone = rgw_utils.zone_for_client(ctx, dest_client)
log.info("source is %s", src_zone)
log.info("dest is %s", dest_zone)
testdir = teuthology.get_testdir(ctx)
(remote,) = ctx.cluster.only(client).remotes.keys()
# figure out which branch to pull from
branch = cconf.get('force-branch', None)
if not branch:
branch = cconf.get('branch', 'master')
sha1 = cconf.get('sha1')
remote.run(
args=[
'cd', testdir, run.Raw('&&'),
'git', 'clone',
'-b', branch,
# 'https://github.com/ceph/radosgw-agent.git',
'git://git.ceph.com/radosgw-agent.git',
'radosgw-agent.{client}'.format(client=client),
]
)
if sha1 is not None:
remote.run(
args=[
'cd', testdir, run.Raw('&&'),
run.Raw('&&'),
'git', 'reset', '--hard', sha1,
]
)
remote.run(
args=[
'cd', testdir, run.Raw('&&'),
'cd', 'radosgw-agent.{client}'.format(client=client),
run.Raw('&&'),
'./bootstrap',
]
)
src_host, src_port = rgw_utils.get_zone_host_and_port(ctx, src_client,
src_zone)
dest_host, dest_port = rgw_utils.get_zone_host_and_port(ctx, dest_client,
dest_zone)
src_access, src_secret = rgw_utils.get_zone_system_keys(ctx, src_client,
src_zone)
dest_access, dest_secret = rgw_utils.get_zone_system_keys(ctx, dest_client,
dest_zone)
sync_scope = cconf.get('sync-scope', None)
port = cconf.get('port', 8000)
daemon_name = '{host}.{port}.syncdaemon'.format(host=remote.name, port=port)
in_args=[
'daemon-helper',
'kill',
'{tdir}/radosgw-agent.{client}/radosgw-agent'.format(tdir=testdir,
client=client),
'-v',
'--src-access-key', src_access,
'--src-secret-key', src_secret,
'--source', "http://{addr}:{port}".format(addr=src_host, port=src_port),
'--dest-access-key', dest_access,
'--dest-secret-key', dest_secret,
'--max-entries', str(cconf.get('max-entries', 1000)),
'--log-file', '{tdir}/archive/rgw_sync_agent.{client}.log'.format(
tdir=testdir,
client=client),
'--object-sync-timeout', '30',
]
if cconf.get('metadata-only', False):
in_args.append('--metadata-only')
# the test server and full/incremental flags are mutually exclusive
if sync_scope is None:
in_args.append('--test-server-host')
in_args.append('0.0.0.0')
in_args.append('--test-server-port')
in_args.append(str(port))
log.debug('Starting a sync test server on {client}'.format(client=client))
# Stash the radosgw-agent server / port # for use by subsequent tasks
ctx.radosgw_agent.endpoint = (client, str(port))
else:
in_args.append('--sync-scope')
in_args.append(sync_scope)
log.debug('Starting a {scope} sync on {client}'.format(scope=sync_scope,client=client))
# positional arg for destination must come last
in_args.append("http://{addr}:{port}".format(addr=dest_host,
port=dest_port))
return_list.append((client, remote.run(
args=in_args,
wait=False,
stdin=run.PIPE,
logger=log.getChild(daemon_name),
)))
return return_list
def start_rgw(ctx, config, on_client = None, except_client = None):
"""
Start rgw on remote sites.
"""
log.info('Starting rgw...')
log.debug('client %r', on_client)
clients_to_run = [on_client]
if on_client is None:
clients_to_run = config.keys()
testdir = teuthology.get_testdir(ctx)
for client in clients_to_run:
if client == except_client:
continue
(remote,) = ctx.cluster.only(client).remotes.iterkeys()
zone = rgw_utils.zone_for_client(ctx, client)
log.debug('zone %s', zone)
client_config = config.get(client)
if client_config is None:
client_config = {}
log.info("rgw %s config is %s", client, client_config)
id_ = client.split('.', 1)[1]
log.info('client {client} is id {id}'.format(client=client, id=id_))
cmd_prefix = [
'sudo',
'adjust-ulimits',
'ceph-coverage',
'{tdir}/archive/coverage'.format(tdir=testdir),
'daemon-helper',
'term',
]
rgw_cmd = ['radosgw']
if ctx.rgw.frontend == 'apache':
if ctx.rgw.use_fastcgi or _use_uds_with_fcgi(remote):
rgw_cmd.extend([
'--rgw-socket-path',
'{tdir}/apache/tmp.{client}/fastcgi_sock/rgw_sock'.format(
tdir=testdir,
client=client,
),
'--rgw-frontends',
'fastcgi',
])
else:
# for mod_proxy_fcgi, using tcp
rgw_cmd.extend([
'--rgw-socket-path', '',
'--rgw-print-continue', 'false',
'--rgw-frontends',
'fastcgi socket_port=9000 socket_host=0.0.0.0',
])
elif ctx.rgw.frontend == 'civetweb':
host, port = ctx.rgw.role_endpoints[client]
rgw_cmd.extend([
'--rgw-frontends',
'civetweb port={port}'.format(port=port),
])
if zone is not None:
rgw_cmd.extend(['--rgw-zone', zone])
rgw_cmd.extend([
'-n', client,
'-k', '/etc/ceph/ceph.{client}.keyring'.format(client=client),
'--log-file',
'/var/log/ceph/rgw.{client}.log'.format(client=client),
'--rgw_ops_log_socket_path',
'{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir,
client=client),
'--foreground',
run.Raw('|'),
'sudo',
'tee',
'/var/log/ceph/rgw.{client}.stdout'.format(tdir=testdir,
client=client),
run.Raw('2>&1'),
])
if client_config.get('valgrind'):
cmd_prefix = teuthology.get_valgrind_args(
testdir,
client,
cmd_prefix,
client_config.get('valgrind')
)
run_cmd = list(cmd_prefix)
run_cmd.extend(rgw_cmd)
ctx.daemons.add_daemon(
remote, 'rgw', client,
args=run_cmd,
logger=log.getChild(client),
stdin=run.PIPE,
wait=False,
)
try:
yield
finally:
teuthology.stop_daemons_of_type(ctx, 'rgw')
for client in config.iterkeys():
ctx.cluster.only(client).run(
args=[
'rm',
'-f',
'{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir,
client=client),
],
)
def start_rgw(ctx, config, on_client=None, except_client=None):
"""
Start rgw on remote sites.
"""
log.info('Starting rgw...')
log.debug('client %r', on_client)
clients_to_run = [on_client]
if on_client is None:
clients_to_run = config.keys()
log.debug('client %r', clients_to_run)
testdir = teuthology.get_testdir(ctx)
for client in clients_to_run:
if client == except_client:
continue
(remote, ) = ctx.cluster.only(client).remotes.iterkeys()
cluster_name, daemon_type, client_id = teuthology.split_role(client)
client_with_id = daemon_type + '.' + client_id
client_with_cluster = cluster_name + '.' + client_with_id
zone = rgw_utils.zone_for_client(ctx, client)
log.debug('zone %s', zone)
client_config = config.get(client)
if client_config is None:
client_config = {}
log.info("rgw %s config is %s", client, client_config)
id_ = client.split('.', 1)[1]
log.info('client {client} is id {id}'.format(client=client, id=id_))
cmd_prefix = [
'sudo',
'adjust-ulimits',
'ceph-coverage',
'{tdir}/archive/coverage'.format(tdir=testdir),
'daemon-helper',
'term',
]
rgw_cmd = ['radosgw']
if ctx.rgw.frontend == 'apache':
if ctx.rgw.use_fastcgi or _use_uds_with_fcgi(remote):
rgw_cmd.extend([
'--rgw-socket-path',
'{tdir}/apache/tmp.{client_with_cluster}/fastcgi_sock/rgw_sock'
.format(
tdir=testdir,
client_with_cluster=client_with_cluster,
),
'--rgw-frontends',
'fastcgi',
])
else:
# for mod_proxy_fcgi, using tcp
rgw_cmd.extend([
'--rgw-socket-path',
'',
'--rgw-print-continue',
'false',
'--rgw-frontends',
'fastcgi socket_port=9000 socket_host=0.0.0.0',
])
elif ctx.rgw.frontend == 'civetweb':
host, port = ctx.rgw.role_endpoints[client]
rgw_cmd.extend([
'--rgw-frontends',
'civetweb port={port}'.format(port=port),
])
if zone is not None:
rgw_cmd.extend(['--rgw-zone', zone])
rgw_cmd.extend([
'-n',
client_with_id,
'--cluster',
cluster_name,
'-k',
'/etc/ceph/{client_with_cluster}.keyring'.format(
client_with_cluster=client_with_cluster),
'--log-file',
'/var/log/ceph/rgw.{client_with_cluster}.log'.format(
client_with_cluster=client_with_cluster),
'--rgw_ops_log_socket_path',
'{tdir}/rgw.opslog.{client_with_cluster}.sock'.format(
tdir=testdir, client_with_cluster=client_with_cluster),
'--foreground',
run.Raw('|'),
'sudo',
'tee',
'/var/log/ceph/rgw.{client_with_cluster}.stdout'.format(
tdir=testdir, client_with_cluster=client_with_cluster),
run.Raw('2>&1'),
])
if client_config.get('valgrind'):
cmd_prefix = teuthology.get_valgrind_args(
testdir, client, cmd_prefix, client_config.get('valgrind'))
run_cmd = list(cmd_prefix)
run_cmd.extend(rgw_cmd)
ctx.daemons.add_daemon(
remote,
'rgw',
client,
cluster=cluster_name,
args=run_cmd,
logger=log.getChild(client),
stdin=run.PIPE,
wait=False,
)
# XXX: add_daemon() doesn't let us wait until radosgw finishes startup
# use a connection pool with retry/backoff to poll each gateway until it starts listening
http = PoolManager(retries=Retry(connect=8, backoff_factor=1))
for client in clients_to_run:
if client == except_client:
continue
host, port = ctx.rgw.role_endpoints[client]
endpoint = 'http://{host}:{port}/'.format(host=host, port=port)
log.info(
'Polling {client} until it starts accepting connections on {endpoint}'
.format(client=client, endpoint=endpoint))
http.request('GET', endpoint)
try:
yield
finally:
teuthology.stop_daemons_of_type(ctx, 'rgw')
for client in config.iterkeys():
ctx.cluster.only(client).run(args=[
'rm',
'-f',
'{tdir}/rgw.opslog.{client_with_cluster}.sock'.format(
tdir=testdir, client_with_cluster=client_with_cluster),
], )
def run_radosgw_agent(ctx, config):
"""
Run a single radosgw-agent. See task() for config format.
"""
return_list = list()
for (client, cconf) in config.items():
# don't process entries that are not clients
if not client.startswith('client.'):
log.debug(
'key {data} does not start with \'client.\', moving on'.format(
data=client))
continue
src_client = cconf['src']
dest_client = cconf['dest']
src_zone = rgw_utils.zone_for_client(ctx, src_client)
dest_zone = rgw_utils.zone_for_client(ctx, dest_client)
log.info("source is %s", src_zone)
log.info("dest is %s", dest_zone)
testdir = teuthology.get_testdir(ctx)
(remote, ) = ctx.cluster.only(client).remotes.keys()
# figure out which branch to pull from
branch = cconf.get('force-branch', None)
if not branch:
branch = cconf.get('branch', 'master')
sha1 = cconf.get('sha1')
remote.run(args=[
'cd',
testdir,
run.Raw('&&'),
'git',
'clone',
'-b',
branch,
# 'https://github.com/ceph/radosgw-agent.git',
'git://ceph.com/git/radosgw-agent.git',
'radosgw-agent.{client}'.format(client=client),
])
if sha1 is not None:
remote.run(args=[
'cd',
testdir,
run.Raw('&&'),
run.Raw('&&'),
'git',
'reset',
'--hard',
sha1,
])
remote.run(args=[
'cd',
testdir,
run.Raw('&&'),
'cd',
'radosgw-agent.{client}'.format(client=client),
run.Raw('&&'),
'./bootstrap',
])
src_host, src_port = rgw_utils.get_zone_host_and_port(
ctx, src_client, src_zone)
dest_host, dest_port = rgw_utils.get_zone_host_and_port(
ctx, dest_client, dest_zone)
src_access, src_secret = rgw_utils.get_zone_system_keys(
ctx, src_client, src_zone)
dest_access, dest_secret = rgw_utils.get_zone_system_keys(
ctx, dest_client, dest_zone)
sync_scope = cconf.get('sync-scope', None)
port = cconf.get('port', 8000)
daemon_name = '{host}.{port}.syncdaemon'.format(host=remote.name,
port=port)
in_args = [
'daemon-helper',
'kill',
'{tdir}/radosgw-agent.{client}/radosgw-agent'.format(
tdir=testdir, client=client),
'-v',
'--src-access-key',
src_access,
'--src-secret-key',
src_secret,
'--source',
"http://{addr}:{port}".format(addr=src_host, port=src_port),
'--dest-access-key',
dest_access,
'--dest-secret-key',
dest_secret,
'--max-entries',
str(cconf.get('max-entries', 1000)),
'--log-file',
'{tdir}/archive/rgw_sync_agent.{client}.log'.format(tdir=testdir,
client=client),
'--object-sync-timeout',
'30',
]
if cconf.get('metadata-only', False):
in_args.append('--metadata-only')
# the test server and full/incremental flags are mutually exclusive
if sync_scope is None:
in_args.append('--test-server-host')
in_args.append('0.0.0.0')
in_args.append('--test-server-port')
in_args.append(str(port))
log.debug('Starting a sync test server on {client}'.format(
client=client))
# Stash the radosgw-agent server / port # for use by subsequent tasks
ctx.radosgw_agent.endpoint = (client, str(port))
else:
in_args.append('--sync-scope')
in_args.append(sync_scope)
log.debug('Starting a {scope} sync on {client}'.format(
scope=sync_scope, client=client))
# positional arg for destination must come last
in_args.append("http://{addr}:{port}".format(addr=dest_host,
port=dest_port))
return_list.append((client,
remote.run(
args=in_args,
wait=False,
stdin=run.PIPE,
logger=log.getChild(daemon_name),
)))
return return_list
def start_rgw(ctx, config, on_client = None, except_client = None):
"""
Start rgw on remote sites.
"""
log.info('Starting rgw...')
log.debug('client %r', on_client)
clients_to_run = [on_client]
if on_client is None:
clients_to_run = config.keys()
log.debug('client %r', clients_to_run)
testdir = teuthology.get_testdir(ctx)
for client in clients_to_run:
if client == except_client:
continue
(remote,) = ctx.cluster.only(client).remotes.iterkeys()
cluster_name, daemon_type, client_id = teuthology.split_role(client)
client_with_id = daemon_type + '.' + client_id
client_with_cluster = cluster_name + '.' + client_with_id
zone = rgw_utils.zone_for_client(ctx, client)
log.debug('zone %s', zone)
client_config = config.get(client)
if client_config is None:
client_config = {}
log.info("rgw %s config is %s", client, client_config)
cmd_prefix = [
'sudo',
'adjust-ulimits',
'ceph-coverage',
'{tdir}/archive/coverage'.format(tdir=testdir),
'daemon-helper',
'term',
]
rgw_cmd = ['radosgw']
if ctx.rgw.frontend == 'apache':
if ctx.rgw.use_fastcgi or _use_uds_with_fcgi(remote):
rgw_cmd.extend([
'--rgw-socket-path',
'{tdir}/apache/tmp.{client_with_cluster}/fastcgi_sock/rgw_sock'.format(
tdir=testdir,
client_with_cluster=client_with_cluster,
),
'--rgw-frontends',
'fastcgi',
])
else:
# for mod_proxy_fcgi, using tcp
rgw_cmd.extend([
'--rgw-socket-path', '',
'--rgw-print-continue', 'false',
'--rgw-frontends',
'fastcgi socket_port=9000 socket_host=0.0.0.0',
])
elif ctx.rgw.frontend == 'civetweb':
host, port = ctx.rgw.role_endpoints[client]
rgw_cmd.extend([
'--rgw-frontends',
'civetweb port={port}'.format(port=port),
])
if zone is not None:
rgw_cmd.extend(['--rgw-zone', zone])
rgw_cmd.extend([
'-n', client_with_id,
'--cluster', cluster_name,
'-k', '/etc/ceph/{client_with_cluster}.keyring'.format(client_with_cluster=client_with_cluster),
'--log-file',
'/var/log/ceph/rgw.{client_with_cluster}.log'.format(client_with_cluster=client_with_cluster),
'--rgw_ops_log_socket_path',
'{tdir}/rgw.opslog.{client_with_cluster}.sock'.format(tdir=testdir,
client_with_cluster=client_with_cluster),
'--foreground',
run.Raw('|'),
'sudo',
'tee',
'/var/log/ceph/rgw.{client_with_cluster}.stdout'.format(tdir=testdir,
client_with_cluster=client_with_cluster),
run.Raw('2>&1'),
])
if client_config.get('valgrind'):
cmd_prefix = teuthology.get_valgrind_args(
testdir,
client_with_cluster,
cmd_prefix,
client_config.get('valgrind')
)
run_cmd = list(cmd_prefix)
run_cmd.extend(rgw_cmd)
ctx.daemons.add_daemon(
remote, 'rgw', client_with_id,
cluster=cluster_name,
args=run_cmd,
logger=log.getChild(client),
stdin=run.PIPE,
wait=False,
)
# XXX: add_daemon() doesn't let us wait until radosgw finishes startup
for client in clients_to_run:
if client == except_client:
continue
host, port = ctx.rgw.role_endpoints[client]
endpoint = 'http://{host}:{port}/'.format(host=host, port=port)
log.info('Polling {client} until it starts accepting connections on {endpoint}'.format(client=client, endpoint=endpoint))
wait_for_radosgw(endpoint)
try:
yield
finally:
for client in config.iterkeys():
cluster_name, daemon_type, client_id = teuthology.split_role(client)
client_with_id = daemon_type + '.' + client_id
client_with_cluster = cluster_name + '.' + client_with_id
ctx.daemons.get_daemon('rgw', client_with_id, cluster_name).stop()
ctx.cluster.only(client).run(
args=[
'rm',
'-f',
'{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir,
client=client_with_cluster),
],
)
