def _authenticate(self, auth_kwargs):
"""
Helper method to turn some auth_kwargs into a set of potential storage
URLs and a token.
"""
if auth_kwargs.get('token'):
logging.debug('Using token %s at one of %r',
auth_kwargs['token'], auth_kwargs['storage_urls'])
return auth_kwargs['storage_urls'], auth_kwargs['token']
logging.debug('Authenticating to %s with %s/%s',
auth_kwargs['auth_url'], auth_kwargs['user'],
auth_kwargs['key'])
storage_url, token = client.get_auth(**auth_kwargs)
if auth_kwargs['storage_urls']:
logging.debug('Overriding auth storage url %s with '
'one of %r', storage_url,
auth_kwargs['storage_urls'])
return auth_kwargs['storage_urls'], token
return [storage_url], token
def _authenticate(self, auth_kwargs):
"""
Helper method to turn some auth_kwargs into a set of potential storage
URLs and a token.
"""
if auth_kwargs.get('token'):
logging.debug('Using token %s at one of %r', auth_kwargs['token'],
auth_kwargs['storage_urls'])
return auth_kwargs['storage_urls'], auth_kwargs['token']
logging.debug('Authenticating to %s with %s/%s',
auth_kwargs['auth_url'], auth_kwargs['user'],
auth_kwargs['key'])
storage_url, token = client.get_auth(**auth_kwargs)
if auth_kwargs['storage_urls']:
logging.debug('Overriding auth storage url %s with '
'one of %r', storage_url,
auth_kwargs['storage_urls'])
return auth_kwargs['storage_urls'], token
return [storage_url], token
def run_scenario(self, scenario, auth_url, user, key, auth_version,
os_options, cacert, insecure, storage_url, token,
noop=False, with_profiling=False, keep_objects=False,
batch_size=1):
"""
Runs a CRUD scenario, given cluster parameters and a Scenario object.
:param scenario: Scenario object describing the benchmark run
:param auth_url: Authentication URL for the Swift cluster
:param user: Account/Username to use (format is <account>:<username>)
:param key: Password for the Account/Username
:param auth_version: OpenStack auth version, default is 1.0
:param os_options: The OpenStack options which can have tenant_id,
auth_token, service_type, endpoint_type,
tenant_name, object_storage_url, region_name
:param insecure: Allow to access insecure keystone server.
The keystone's certificate will not be verified.
:param cacert: Bundle file to use in verifying SSL.
:param storage_url: Optional user-specified x-storage-url
:param token: Optional user-specified x-auth-token
:param noop: Run in no-op mode?
:param with_profiing: Profile the run?
:param keep_objects: Keep uploaded objects instead of deleting them?
:param batch_size: Send this many bench jobs per packet to workers
:param returns: Collected result records from workers
"""
run_state = RunState()
logging.info(u'Starting scenario run for "%s"', scenario.name)
soft_nofile, hard_nofile = resource.getrlimit(resource.RLIMIT_NOFILE)
nofile_target = 1024
if os.geteuid() == 0:
nofile_target = max(nofile_target, scenario.user_count + 20)
hard_nofile = nofile_target
resource.setrlimit(resource.RLIMIT_NOFILE, (nofile_target,
hard_nofile))
# Construct auth_kwargs appropriate for client.get_auth()
if not token:
auth_kwargs = dict(
auth_url=auth_url, user=user, key=key,
auth_version=auth_version, os_options=os_options,
cacert=cacert, insecure=insecure, storage_url=storage_url)
else:
auth_kwargs = dict(storage_url=storage_url, token=token)
# Ensure containers exist
if not noop:
if not token:
logging.debug('Authenticating to %s with %s/%s', auth_url,
user, key)
c_storage_url, c_token = client.get_auth(**auth_kwargs)
if storage_url:
logging.debug('Overriding auth storage url %s with %s',
c_storage_url, storage_url)
c_storage_url = storage_url
else:
c_storage_url, c_token = storage_url, token
logging.debug('Using token %s at %s', c_token, c_storage_url)
logging.info('Ensuring %d containers (%s_*) exist; '
'concurrency=%d...',
len(scenario.containers), scenario.container_base,
scenario.container_concurrency)
pool = gevent.pool.Pool(scenario.container_concurrency)
for container in scenario.containers:
pool.spawn(_container_creator, c_storage_url, c_token,
container)
pool.join()
# Enqueue initialization jobs
if not noop:
logging.info('Initializing cluster with stock data (up to %d '
'concurrent workers)', scenario.user_count)
self.do_a_run(scenario.user_count, scenario.initial_jobs(),
run_state.handle_initialization_result, auth_kwargs,
batch_size=batch_size)
logging.info('Starting benchmark run (up to %d concurrent '
'workers)', scenario.user_count)
if noop:
logging.info(' (not actually talking to Swift cluster!)')
if with_profiling:
import cProfile
prof = cProfile.Profile()
prof.enable()
self.do_a_run(scenario.user_count, scenario.bench_jobs(),
run_state.handle_run_result, auth_kwargs,
mapper_fn=run_state.fill_in_job,
label='Benchmark Run:', noop=noop, batch_size=batch_size)
if with_profiling:
prof.disable()
prof_output_path = '/tmp/do_a_run.%d.prof' % os.getpid()
prof.dump_stats(prof_output_path)
logging.info('PROFILED main do_a_run to %s', prof_output_path)
if not noop and not keep_objects:
logging.info('Deleting population objects from cluster')
self.do_a_run(scenario.user_count,
run_state.cleanup_object_infos(),
lambda *_: None,
auth_kwargs, mapper_fn=_gen_cleanup_job,
batch_size=batch_size)
elif keep_objects:
logging.info('NOT deleting any objects due to -k/--keep-objects')
return run_state.run_results
def ignoring_http_responses(self, statuses, fn, call_info, **extra_keys):
if 401 not in statuses:
statuses += (401, )
args = dict(
container=call_info['container'],
name=call_info['name'],
)
args.update(extra_keys)
if 'auth_kwargs' not in call_info:
raise ValueError('Got benchmark job without "auth_kwargs" key!')
tries = 0
while True:
# Make sure we've got a current storage_url/token
if call_info['auth_kwargs'].get('token', None):
token_key = None
args['url'] = random.choice(
call_info['auth_kwargs']['storage_urls'])
args['token'] = call_info['auth_kwargs']['token']
else:
token_key = self._token_key(call_info['auth_kwargs'])
if token_key not in self.token_data:
self.token_data_lock.acquire()
collided = False
try:
if token_key not in self.token_data:
logging.debug('Authenticating with %r',
call_info['auth_kwargs'])
storage_url, token = client.get_auth(
**call_info['auth_kwargs'])
override_urls = call_info['auth_kwargs'].get(
'storage_urls', None)
if override_urls:
logging.debug(
'Will override auth storage url %s with '
'one of %r', storage_url, override_urls)
storage_urls = override_urls
else:
storage_urls = [storage_url]
self.token_data[token_key] = (storage_urls, token)
else:
collided = True
finally:
self.token_data_lock.release()
if collided:
# Wait just a little bit if we just collided with
# another greenthread's re-auth
logging.debug('Collided on re-auth; sleeping 0.005')
gevent.sleep(0.005)
storage_urls, args['token'] = self.token_data[token_key]
args['url'] = random.choice(storage_urls)
# Check for connection pool initialization (protected by a
# semaphore)
if args['url'] not in self.conn_pools:
self._create_connection_pool(
args['url'],
call_info.get('connect_timeout',
client.DEFAULT_CONNECT_TIMEOUT),
call_info.get('network_timeout',
client.DEFAULT_NETWORK_TIMEOUT))
try:
fn_results = None
with self.connection(args['url']) as conn:
fn_results = fn(http_conn=conn, **args)
if fn_results:
if tries != 0:
logging.info('%r succeeded after %d tries', call_info,
tries)
break
tries += 1
if tries > self.max_retries:
e = Exception('No fn_results for %r after %d retires' %
(fn, self.max_retries))
e.retries = tries - 1
raise e
# XXX The name of this method does not suggest that it
# will also retry on socket-level errors. Regardless,
# sometimes Swift refuses connections (probably when it's
# way overloaded and the listen socket's connection queue
# (in the kernel) is full, so the kernel just says RST).
#
# UPDATE: connections should be handled by the ConnectionPool
# (which will trap socket.error and retry after a slight delay), so
# socket.error should NOT get raised here for connection failures.
# So hopefully this socket.error trapping code path will not get
# hit.
except socket.error as error:
tries += 1
if tries > self.max_retries:
error.retries = tries - 1
raise error
except client.ClientException as error:
tries += 1
if error.http_status in statuses and \
tries <= self.max_retries:
if error.http_status == 401 and token_key:
if token_key in self.token_data and \
self.token_data[token_key][1] == args['token']:
self.token_data_lock.acquire()
try:
if token_key in self.token_data and \
self.token_data[token_key][1] == \
args['token']:
logging.debug(
'Deleting token %s',
self.token_data[token_key][1])
del self.token_data[token_key]
finally:
self.token_data_lock.release()
logging.debug("Retrying an error: %r", error)
else:
error.retries = tries - 1
raise error
fn_results['retries'] = tries
return fn_results
def ignoring_http_responses(self, statuses, fn, call_info, **extra_keys):
if 401 not in statuses:
statuses += (401,)
args = dict(
container=call_info['container'],
name=call_info['name'],
)
args.update(extra_keys)
if 'auth_kwargs' not in call_info:
raise ValueError('Got benchmark job without "auth_kwargs" key!')
tries = 0
while True:
# Make sure we've got a current storage_url/token
if call_info['auth_kwargs'].get('token', None):
args['url'] = random.choice(
call_info['auth_kwargs']['storage_urls'])
args['token'] = call_info['auth_kwargs']['token']
else:
token_key = self._token_key(call_info['auth_kwargs'])
if token_key not in self.token_data:
self.token_data_lock.acquire()
collided = False
try:
if token_key not in self.token_data:
logging.debug('Authenticating with %r',
call_info['auth_kwargs'])
storage_url, token = client.get_auth(
**call_info['auth_kwargs'])
override_urls = call_info['auth_kwargs'].get(
'storage_urls', None)
if override_urls:
logging.debug(
'Will override auth storage url %s with '
'one of %r', storage_url, override_urls)
storage_urls = override_urls
else:
storage_urls = [storage_url]
self.token_data[token_key] = (storage_urls, token)
else:
collided = True
finally:
self.token_data_lock.release()
if collided:
# Wait just a little bit if we just collided with
# another greenthread's re-auth
logging.debug('Collided on re-auth; sleeping 0.005')
gevent.sleep(0.005)
storage_urls, args['token'] = self.token_data[token_key]
args['url'] = random.choice(storage_urls)
# Check for connection pool initialization (protected by a
# semaphore)
if args['url'] not in self.conn_pools:
self._create_connection_pool(
args['url'],
call_info.get('connect_timeout',
client.DEFAULT_CONNECT_TIMEOUT),
call_info.get('network_timeout',
client.DEFAULT_NETWORK_TIMEOUT))
try:
fn_results = None
with self.connection(args['url']) as conn:
fn_results = fn(http_conn=conn, **args)
if fn_results:
if tries != 0:
logging.info('%r succeeded after %d tries',
call_info, tries)
break
tries += 1
if tries > self.max_retries:
e = Exception('No fn_results for %r after %d retires' % (
fn, self.max_retries))
e.retries = tries - 1
raise e
# XXX The name of this method does not suggest that it
# will also retry on socket-level errors. Regardless,
# sometimes Swift refuses connections (probably when it's
# way overloaded and the listen socket's connection queue
# (in the kernel) is full, so the kernel just says RST).
#
# UPDATE: connections should be handled by the ConnectionPool
# (which will trap socket.error and retry after a slight delay), so
# socket.error should NOT get raised here for connection failures.
# So hopefully this socket.error trapping code path will not get
# hit.
except socket.error as error:
tries += 1
if tries > self.max_retries:
error.retries = tries - 1
raise error
except client.ClientException as error:
tries += 1
if error.http_status in statuses and \
tries <= self.max_retries:
if error.http_status == 401 and token_key:
if token_key in self.token_data and \
self.token_data[token_key][1] == args['token']:
self.token_data_lock.acquire()
try:
if token_key in self.token_data and \
self.token_data[token_key][1] == \
args['token']:
logging.debug(
'Deleting token %s',
self.token_data[token_key][1])
del self.token_data[token_key]
finally:
self.token_data_lock.release()
logging.debug("Retrying an error: %r", error)
else:
error.retries = tries - 1
raise error
fn_results['retries'] = tries
return fn_results
def run_scenario(self, auth_url, user, key, storage_url, token, scenario):
"""
Runs a CRUD scenario, given cluster parameters and a Scenario object.
:auth_url: Authentication URL for the Swift cluster
:user: Account/Username to use (format is <account>:<username>)
:key: Password for the Account/Username
:scenario: Scenario object describing the benchmark run
:returns: Collected result records from workers
"""
run_state = RunState()
self.drain_stats_queue()
if not storage_url or not token:
logging.debug("Authenticating to %s with %s/%s", auth_url, user, key)
storage_url, token = client.get_auth(auth_url, user, key)
else:
logging.debug("Using token %s at %s", token, storage_url)
logging.info(u'Starting scenario run for "%s"', scenario.name)
# Ensure containers exist
logging.info(
"Ensuring %d containers (%s_*) exist; concurrency=%d...",
len(scenario.containers),
scenario.container_base,
scenario.container_concurrency,
)
pool = eventlet.GreenPool(scenario.container_concurrency)
for container in scenario.containers:
pool.spawn_n(_container_creator, storage_url, token, container)
pool.waitall()
self.queue.use(ssbench.WORK_TUBE)
# Enqueue initialization jobs
logging.info("Initializing cluster with stock data (up to %d " "concurrent workers)", scenario.user_count)
self.do_a_run(
scenario.user_count,
scenario.initial_jobs(),
run_state.handle_initialization_result,
ssbench.PRIORITY_SETUP,
storage_url,
token,
)
logging.info("Starting benchmark run (up to %d concurrent " "workers)", scenario.user_count)
self.do_a_run(
scenario.user_count,
scenario.bench_jobs(),
run_state.handle_run_result,
ssbench.PRIORITY_WORK,
storage_url,
token,
mapper_fn=run_state.fill_in_job,
label="Benchmark Run:",
)
logging.info("Deleting population objects from cluster")
self.do_a_run(
scenario.user_count,
run_state.cleanup_object_infos(),
lambda *_: None,
ssbench.PRIORITY_CLEANUP,
storage_url,
token,
mapper_fn=_gen_cleanup_job,
)
return run_state.run_results