def run_radosgw_agent(ctx, config): """ Run a single radosgw-agent. See task() for config format. """ return_list = list() for (client, cconf) in config.items(): # don't process entries that are not clients if not client.startswith('client.'): log.debug('key {data} does not start with \'client.\', moving on'.format( data=client)) continue src_client = cconf['src'] dest_client = cconf['dest'] src_zone = rgw_utils.zone_for_client(ctx, src_client) dest_zone = rgw_utils.zone_for_client(ctx, dest_client) log.info("source is %s", src_zone) log.info("dest is %s", dest_zone) testdir = teuthology.get_testdir(ctx) (remote,) = ctx.cluster.only(client).remotes.keys() # figure out which branch to pull from branch = cconf.get('force-branch', None) if not branch: branch = cconf.get('branch', 'master') sha1 = cconf.get('sha1') remote.run( args=[ 'cd', testdir, run.Raw('&&'), 'git', 'clone', '-b', branch, # 'https://github.com/ceph/radosgw-agent.git', 'git://git.ceph.com/radosgw-agent.git', 'radosgw-agent.{client}'.format(client=client), ] ) if sha1 is not None: remote.run( args=[ 'cd', testdir, run.Raw('&&'), run.Raw('&&'), 'git', 'reset', '--hard', sha1, ] ) remote.run( args=[ 'cd', testdir, run.Raw('&&'), 'cd', 'radosgw-agent.{client}'.format(client=client), run.Raw('&&'), './bootstrap', ] ) src_host, src_port = rgw_utils.get_zone_host_and_port(ctx, src_client, src_zone) dest_host, dest_port = rgw_utils.get_zone_host_and_port(ctx, dest_client, dest_zone) src_access, src_secret = rgw_utils.get_zone_system_keys(ctx, src_client, src_zone) dest_access, dest_secret = rgw_utils.get_zone_system_keys(ctx, dest_client, dest_zone) sync_scope = cconf.get('sync-scope', None) port = cconf.get('port', 8000) daemon_name = '{host}.{port}.syncdaemon'.format(host=remote.name, port=port) in_args=[ 'daemon-helper', 'kill', '{tdir}/radosgw-agent.{client}/radosgw-agent'.format(tdir=testdir, client=client), '-v', '--src-access-key', src_access, '--src-secret-key', src_secret, '--source', "http://{addr}:{port}".format(addr=src_host, port=src_port), '--dest-access-key', dest_access, '--dest-secret-key', dest_secret, '--max-entries', str(cconf.get('max-entries', 1000)), '--log-file', '{tdir}/archive/rgw_sync_agent.{client}.log'.format( tdir=testdir, client=client), '--object-sync-timeout', '30', ] if cconf.get('metadata-only', False): in_args.append('--metadata-only') # the test server and full/incremental flags are mutually exclusive if sync_scope is None: in_args.append('--test-server-host') in_args.append('0.0.0.0') in_args.append('--test-server-port') in_args.append(str(port)) log.debug('Starting a sync test server on {client}'.format(client=client)) # Stash the radosgw-agent server / port # for use by subsequent tasks ctx.radosgw_agent.endpoint = (client, str(port)) else: in_args.append('--sync-scope') in_args.append(sync_scope) log.debug('Starting a {scope} sync on {client}'.format(scope=sync_scope,client=client)) # positional arg for destination must come last in_args.append("http://{addr}:{port}".format(addr=dest_host, port=dest_port)) return_list.append((client, remote.run( args=in_args, wait=False, stdin=run.PIPE, logger=log.getChild(daemon_name), ))) return return_list
def start_rgw(ctx, config, on_client = None, except_client = None): """ Start rgw on remote sites. """ log.info('Starting rgw...') log.debug('client %r', on_client) clients_to_run = [on_client] if on_client is None: clients_to_run = config.keys() testdir = teuthology.get_testdir(ctx) for client in clients_to_run: if client == except_client: continue (remote,) = ctx.cluster.only(client).remotes.iterkeys() zone = rgw_utils.zone_for_client(ctx, client) log.debug('zone %s', zone) client_config = config.get(client) if client_config is None: client_config = {} log.info("rgw %s config is %s", client, client_config) id_ = client.split('.', 1)[1] log.info('client {client} is id {id}'.format(client=client, id=id_)) cmd_prefix = [ 'sudo', 'adjust-ulimits', 'ceph-coverage', '{tdir}/archive/coverage'.format(tdir=testdir), 'daemon-helper', 'term', ] rgw_cmd = ['radosgw'] if ctx.rgw.frontend == 'apache': if ctx.rgw.use_fastcgi or _use_uds_with_fcgi(remote): rgw_cmd.extend([ '--rgw-socket-path', '{tdir}/apache/tmp.{client}/fastcgi_sock/rgw_sock'.format( tdir=testdir, client=client, ), '--rgw-frontends', 'fastcgi', ]) else: # for mod_proxy_fcgi, using tcp rgw_cmd.extend([ '--rgw-socket-path', '', '--rgw-print-continue', 'false', '--rgw-frontends', 'fastcgi socket_port=9000 socket_host=0.0.0.0', ]) elif ctx.rgw.frontend == 'civetweb': host, port = ctx.rgw.role_endpoints[client] rgw_cmd.extend([ '--rgw-frontends', 'civetweb port={port}'.format(port=port), ]) if zone is not None: rgw_cmd.extend(['--rgw-zone', zone]) rgw_cmd.extend([ '-n', client, '-k', '/etc/ceph/ceph.{client}.keyring'.format(client=client), '--log-file', '/var/log/ceph/rgw.{client}.log'.format(client=client), '--rgw_ops_log_socket_path', '{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir, client=client), '--foreground', run.Raw('|'), 'sudo', 'tee', '/var/log/ceph/rgw.{client}.stdout'.format(tdir=testdir, client=client), run.Raw('2>&1'), ]) if client_config.get('valgrind'): cmd_prefix = teuthology.get_valgrind_args( testdir, client, cmd_prefix, client_config.get('valgrind') ) run_cmd = list(cmd_prefix) run_cmd.extend(rgw_cmd) ctx.daemons.add_daemon( remote, 'rgw', client, args=run_cmd, logger=log.getChild(client), stdin=run.PIPE, wait=False, ) try: yield finally: teuthology.stop_daemons_of_type(ctx, 'rgw') for client in config.iterkeys(): ctx.cluster.only(client).run( args=[ 'rm', '-f', '{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir, client=client), ], )
def start_rgw(ctx, config, on_client=None, except_client=None): """ Start rgw on remote sites. """ log.info('Starting rgw...') log.debug('client %r', on_client) clients_to_run = [on_client] if on_client is None: clients_to_run = config.keys() log.debug('client %r', clients_to_run) testdir = teuthology.get_testdir(ctx) for client in clients_to_run: if client == except_client: continue (remote, ) = ctx.cluster.only(client).remotes.iterkeys() cluster_name, daemon_type, client_id = teuthology.split_role(client) client_with_id = daemon_type + '.' + client_id client_with_cluster = cluster_name + '.' + client_with_id zone = rgw_utils.zone_for_client(ctx, client) log.debug('zone %s', zone) client_config = config.get(client) if client_config is None: client_config = {} log.info("rgw %s config is %s", client, client_config) id_ = client.split('.', 1)[1] log.info('client {client} is id {id}'.format(client=client, id=id_)) cmd_prefix = [ 'sudo', 'adjust-ulimits', 'ceph-coverage', '{tdir}/archive/coverage'.format(tdir=testdir), 'daemon-helper', 'term', ] rgw_cmd = ['radosgw'] if ctx.rgw.frontend == 'apache': if ctx.rgw.use_fastcgi or _use_uds_with_fcgi(remote): rgw_cmd.extend([ '--rgw-socket-path', '{tdir}/apache/tmp.{client_with_cluster}/fastcgi_sock/rgw_sock' .format( tdir=testdir, client_with_cluster=client_with_cluster, ), '--rgw-frontends', 'fastcgi', ]) else: # for mod_proxy_fcgi, using tcp rgw_cmd.extend([ '--rgw-socket-path', '', '--rgw-print-continue', 'false', '--rgw-frontends', 'fastcgi socket_port=9000 socket_host=0.0.0.0', ]) elif ctx.rgw.frontend == 'civetweb': host, port = ctx.rgw.role_endpoints[client] rgw_cmd.extend([ '--rgw-frontends', 'civetweb port={port}'.format(port=port), ]) if zone is not None: rgw_cmd.extend(['--rgw-zone', zone]) rgw_cmd.extend([ '-n', client_with_id, '--cluster', cluster_name, '-k', '/etc/ceph/{client_with_cluster}.keyring'.format( client_with_cluster=client_with_cluster), '--log-file', '/var/log/ceph/rgw.{client_with_cluster}.log'.format( client_with_cluster=client_with_cluster), '--rgw_ops_log_socket_path', '{tdir}/rgw.opslog.{client_with_cluster}.sock'.format( tdir=testdir, client_with_cluster=client_with_cluster), '--foreground', run.Raw('|'), 'sudo', 'tee', '/var/log/ceph/rgw.{client_with_cluster}.stdout'.format( tdir=testdir, client_with_cluster=client_with_cluster), run.Raw('2>&1'), ]) if client_config.get('valgrind'): cmd_prefix = teuthology.get_valgrind_args( testdir, client, cmd_prefix, client_config.get('valgrind')) run_cmd = list(cmd_prefix) run_cmd.extend(rgw_cmd) ctx.daemons.add_daemon( remote, 'rgw', client, cluster=cluster_name, args=run_cmd, logger=log.getChild(client), stdin=run.PIPE, wait=False, ) # XXX: add_daemon() doesn't let us wait until radosgw finishes startup # use a connection pool with retry/backoff to poll each gateway until it starts listening http = PoolManager(retries=Retry(connect=8, backoff_factor=1)) for client in clients_to_run: if client == except_client: continue host, port = ctx.rgw.role_endpoints[client] endpoint = 'http://{host}:{port}/'.format(host=host, port=port) log.info( 'Polling {client} until it starts accepting connections on {endpoint}' .format(client=client, endpoint=endpoint)) http.request('GET', endpoint) try: yield finally: teuthology.stop_daemons_of_type(ctx, 'rgw') for client in config.iterkeys(): ctx.cluster.only(client).run(args=[ 'rm', '-f', '{tdir}/rgw.opslog.{client_with_cluster}.sock'.format( tdir=testdir, client_with_cluster=client_with_cluster), ], )
def run_radosgw_agent(ctx, config): """ Run a single radosgw-agent. See task() for config format. """ return_list = list() for (client, cconf) in config.items(): # don't process entries that are not clients if not client.startswith('client.'): log.debug( 'key {data} does not start with \'client.\', moving on'.format( data=client)) continue src_client = cconf['src'] dest_client = cconf['dest'] src_zone = rgw_utils.zone_for_client(ctx, src_client) dest_zone = rgw_utils.zone_for_client(ctx, dest_client) log.info("source is %s", src_zone) log.info("dest is %s", dest_zone) testdir = teuthology.get_testdir(ctx) (remote, ) = ctx.cluster.only(client).remotes.keys() # figure out which branch to pull from branch = cconf.get('force-branch', None) if not branch: branch = cconf.get('branch', 'master') sha1 = cconf.get('sha1') remote.run(args=[ 'cd', testdir, run.Raw('&&'), 'git', 'clone', '-b', branch, # 'https://github.com/ceph/radosgw-agent.git', 'git://ceph.com/git/radosgw-agent.git', 'radosgw-agent.{client}'.format(client=client), ]) if sha1 is not None: remote.run(args=[ 'cd', testdir, run.Raw('&&'), run.Raw('&&'), 'git', 'reset', '--hard', sha1, ]) remote.run(args=[ 'cd', testdir, run.Raw('&&'), 'cd', 'radosgw-agent.{client}'.format(client=client), run.Raw('&&'), './bootstrap', ]) src_host, src_port = rgw_utils.get_zone_host_and_port( ctx, src_client, src_zone) dest_host, dest_port = rgw_utils.get_zone_host_and_port( ctx, dest_client, dest_zone) src_access, src_secret = rgw_utils.get_zone_system_keys( ctx, src_client, src_zone) dest_access, dest_secret = rgw_utils.get_zone_system_keys( ctx, dest_client, dest_zone) sync_scope = cconf.get('sync-scope', None) port = cconf.get('port', 8000) daemon_name = '{host}.{port}.syncdaemon'.format(host=remote.name, port=port) in_args = [ 'daemon-helper', 'kill', '{tdir}/radosgw-agent.{client}/radosgw-agent'.format( tdir=testdir, client=client), '-v', '--src-access-key', src_access, '--src-secret-key', src_secret, '--source', "http://{addr}:{port}".format(addr=src_host, port=src_port), '--dest-access-key', dest_access, '--dest-secret-key', dest_secret, '--max-entries', str(cconf.get('max-entries', 1000)), '--log-file', '{tdir}/archive/rgw_sync_agent.{client}.log'.format(tdir=testdir, client=client), '--object-sync-timeout', '30', ] if cconf.get('metadata-only', False): in_args.append('--metadata-only') # the test server and full/incremental flags are mutually exclusive if sync_scope is None: in_args.append('--test-server-host') in_args.append('0.0.0.0') in_args.append('--test-server-port') in_args.append(str(port)) log.debug('Starting a sync test server on {client}'.format( client=client)) # Stash the radosgw-agent server / port # for use by subsequent tasks ctx.radosgw_agent.endpoint = (client, str(port)) else: in_args.append('--sync-scope') in_args.append(sync_scope) log.debug('Starting a {scope} sync on {client}'.format( scope=sync_scope, client=client)) # positional arg for destination must come last in_args.append("http://{addr}:{port}".format(addr=dest_host, port=dest_port)) return_list.append((client, remote.run( args=in_args, wait=False, stdin=run.PIPE, logger=log.getChild(daemon_name), ))) return return_list
def start_rgw(ctx, config, on_client = None, except_client = None): """ Start rgw on remote sites. """ log.info('Starting rgw...') log.debug('client %r', on_client) clients_to_run = [on_client] if on_client is None: clients_to_run = config.keys() log.debug('client %r', clients_to_run) testdir = teuthology.get_testdir(ctx) for client in clients_to_run: if client == except_client: continue (remote,) = ctx.cluster.only(client).remotes.iterkeys() cluster_name, daemon_type, client_id = teuthology.split_role(client) client_with_id = daemon_type + '.' + client_id client_with_cluster = cluster_name + '.' + client_with_id zone = rgw_utils.zone_for_client(ctx, client) log.debug('zone %s', zone) client_config = config.get(client) if client_config is None: client_config = {} log.info("rgw %s config is %s", client, client_config) cmd_prefix = [ 'sudo', 'adjust-ulimits', 'ceph-coverage', '{tdir}/archive/coverage'.format(tdir=testdir), 'daemon-helper', 'term', ] rgw_cmd = ['radosgw'] if ctx.rgw.frontend == 'apache': if ctx.rgw.use_fastcgi or _use_uds_with_fcgi(remote): rgw_cmd.extend([ '--rgw-socket-path', '{tdir}/apache/tmp.{client_with_cluster}/fastcgi_sock/rgw_sock'.format( tdir=testdir, client_with_cluster=client_with_cluster, ), '--rgw-frontends', 'fastcgi', ]) else: # for mod_proxy_fcgi, using tcp rgw_cmd.extend([ '--rgw-socket-path', '', '--rgw-print-continue', 'false', '--rgw-frontends', 'fastcgi socket_port=9000 socket_host=0.0.0.0', ]) elif ctx.rgw.frontend == 'civetweb': host, port = ctx.rgw.role_endpoints[client] rgw_cmd.extend([ '--rgw-frontends', 'civetweb port={port}'.format(port=port), ]) if zone is not None: rgw_cmd.extend(['--rgw-zone', zone]) rgw_cmd.extend([ '-n', client_with_id, '--cluster', cluster_name, '-k', '/etc/ceph/{client_with_cluster}.keyring'.format(client_with_cluster=client_with_cluster), '--log-file', '/var/log/ceph/rgw.{client_with_cluster}.log'.format(client_with_cluster=client_with_cluster), '--rgw_ops_log_socket_path', '{tdir}/rgw.opslog.{client_with_cluster}.sock'.format(tdir=testdir, client_with_cluster=client_with_cluster), '--foreground', run.Raw('|'), 'sudo', 'tee', '/var/log/ceph/rgw.{client_with_cluster}.stdout'.format(tdir=testdir, client_with_cluster=client_with_cluster), run.Raw('2>&1'), ]) if client_config.get('valgrind'): cmd_prefix = teuthology.get_valgrind_args( testdir, client_with_cluster, cmd_prefix, client_config.get('valgrind') ) run_cmd = list(cmd_prefix) run_cmd.extend(rgw_cmd) ctx.daemons.add_daemon( remote, 'rgw', client_with_id, cluster=cluster_name, args=run_cmd, logger=log.getChild(client), stdin=run.PIPE, wait=False, ) # XXX: add_daemon() doesn't let us wait until radosgw finishes startup for client in clients_to_run: if client == except_client: continue host, port = ctx.rgw.role_endpoints[client] endpoint = 'http://{host}:{port}/'.format(host=host, port=port) log.info('Polling {client} until it starts accepting connections on {endpoint}'.format(client=client, endpoint=endpoint)) wait_for_radosgw(endpoint) try: yield finally: for client in config.iterkeys(): cluster_name, daemon_type, client_id = teuthology.split_role(client) client_with_id = daemon_type + '.' + client_id client_with_cluster = cluster_name + '.' + client_with_id ctx.daemons.get_daemon('rgw', client_with_id, cluster_name).stop() ctx.cluster.only(client).run( args=[ 'rm', '-f', '{tdir}/rgw.opslog.{client}.sock'.format(tdir=testdir, client=client_with_cluster), ], )