def transform(stub, request, **kwargs):
function = kwargs['function']
stage = kwargs.get('stage')
url_args = kwargs['url_args']
trace = kwargs['trace']
hooks = kwargs.pop('hooks')
module_system_date = kwargs.get('module_system_date')
try:
transformer = hooks.make_transformer(stub)
if module_system_date and isinstance(module_system_date, basestring):
module_system_date = as_date(module_system_date)
system_date = kwargs.get('system_date')
if isinstance(system_date, basestring):
system_date = as_date('system_date')
unsafevars = ('request', 'function', 'cache', 'stage',
'module_system_date', 'system_date', 'trace')
url_args.update(x for x in stub.args().iteritems() if x[0] not in ('session',))
for var in unsafevars:
url_args.pop(var, None)
return transformer.transform(request,
module_system_date=module_system_date,
system_date=system_date,
function=function,
cache=kwargs['cache'],
stage=stage,
trace=trace,
**url_args)
except Exception, e:
_, t, v, tbinfo = compact_traceback()
msg = u'error={0}, traceback is: ({1}: {2} {3})'.format(e, t, v, tbinfo)
log.error(msg)
raise TransformError(code=400, title='Error transforming during '
'{0} stage:{1}'.format(function, stage), traceback=msg)
def transform(stub, request, **kwargs):
function = kwargs['function']
stage = kwargs.get('stage')
url_args = kwargs['url_args']
trace = kwargs['trace']
hooks = kwargs.pop('hooks')
module_system_date = kwargs.get('module_system_date')
try:
transformer = hooks.make_transformer(stub)
if module_system_date and isinstance(module_system_date, basestring):
module_system_date = as_date(module_system_date)
system_date = kwargs.get('system_date')
if isinstance(system_date, basestring):
system_date = as_date('system_date')
unsafevars = ('request', 'function', 'cache', 'stage',
'module_system_date', 'system_date', 'trace')
url_args.update(x for x in stub.args().iteritems()
if x[0] not in ('session', ))
for var in unsafevars:
url_args.pop(var, None)
return transformer.transform(request,
module_system_date=module_system_date,
system_date=system_date,
function=function,
cache=kwargs['cache'],
stage=stage,
trace=trace,
**url_args)
except Exception, e:
_, t, v, tbinfo = compact_traceback()
msg = u'error={0}, traceback is: ({1}: {2} {3})'.format(
e, t, v, tbinfo)
log.error(msg)
raise TransformError(code=400,
title='Error transforming during '
'{0} stage:{1}'.format(function, stage),
traceback=msg)
def eval_text(self, templ, request, **kwargs):
stub = kwargs['stub']
kwargs['recorded'] = stub.recorded()
recorded = kwargs.get('recorded')
if recorded:
kwargs['recorded'] = as_date(recorded)
return run_template(templ,
request=request,
request_text=request.request_body(), # legacy
# utility functions
roll_date=roll_date,
today=today_str,
as_date=as_date,
parse_xml=parse_xml,
**kwargs)
def eval_text(self, templ, request, **kwargs):
request_text = request.request_body()
stub = kwargs['stub']
kwargs['recorded'] = stub.recorded()
recorded = kwargs.get('recorded')
if recorded:
kwargs['recorded'] = as_date(recorded)
return run_template(
templ,
request_text=request_text,
# utility functions
roll_date=roll_date,
today=today_str,
as_date=as_date,
parse_xml=parse_xml,
**kwargs)
def purge_stubs():
# importing helper handler from testing deps
from stubo.testing import DummyRequestHandler
parser = ArgumentParser(
description="Purge stubs older than given expiry date."
)
parser.add_argument('-l', '--list', action='store_const', const=True,
dest='list_only', help="Just list the stubs to delete.")
parser.add_argument('-e', '--expiry', default=14, dest='expiry',
help="expiry is number of days from now (default is 14).")
parser.add_argument('--host', default='all', dest='host',
help="specify the host uri to use (defaults to all)")
parser.add_argument('-c', '--config', dest='config',
help='Path to configuration file (defaults to $CWD/etc/dev.ini)',
metavar='FILE')
args = parser.parse_args()
list_only = args.list_only or False
expiry_days = args.expiry
expiry = datetime.today().date() - timedelta(int(expiry_days))
host = args.host
config = args.config or get_default_config()
logging.config.fileConfig(config)
settings = read_config(config)
dbenv = default_env.copy()
dbenv.update((k[6:], coerce_mongo_param(k[6:], v)) for k, v in \
settings.iteritems() if k.startswith('mongo.'))
log.debug('mongo params: {0}'.format(dbenv))
log.info('purge stubs whereby all sessions in the scenario were last used before {0}'.format(expiry))
db_conn = init_mongo(dbenv).connection
slave, master = start_redis(settings)
response = list_scenarios(host)
if 'error' in response:
print response['error']
sys.exit(-1)
handler = DummyRequestHandler()
session_handler = DummyRequestHandler()
for scenario_key in response['data']['scenarios']:
log.debug("*** scenario '{0}' ***".format(scenario_key))
hostname, scenario = scenario_key.split(':')
if host != 'all' and host != hostname:
continue
handler.host = hostname
handler.request.host = '{0}:8001'.format(hostname)
session_handler.host = hostname
session_handler.request.host = '{0}:8001'.format(hostname)
handler.request.arguments['scenario'] = [scenario]
status = get_status(handler)
if 'error' in status:
log.warn('get_status error: {0}'.format(status['error']))
else:
scenario_last_used = []
sessions = status['data']['sessions']
for session in zip(*sessions)[0]:
log.debug("*** -> session '{0}' ***".format(session))
session_handler.request.arguments['session'] = [session]
session_status = get_status(session_handler)
if 'error' in session_status:
log.warn('get_status error: {0}'.format(status['error']))
else:
last_used = session_status['data']['session'].get('last_used', '-')
if last_used != '-':
scenario_last_used.append(as_date(last_used[0:10]))
if scenario_last_used and (max(scenario_last_used) < expiry):
log.info("sessions in scenario '{0}' were last used '{1}' which"
" is before expiry date '{2}'".format(scenario_key,
max(scenario_last_used), expiry))
if not list_only:
response = delete_stubs(handler, scenario_name=scenario,
force=True)
if 'error' in response:
log.error('delete stubs error: {0}'.format(response['error']))
else:
log.info('deleted stubs: {0}'.format(response['data']))
def get_response(handler, session_name):
request = handler.request
stubo_request = StuboRequest(request)
request_body = stubo_request.request_body()
if not request_body:
# TODO: not an error if they match on other attrs
raise exception_response(400, title='No text in body')
cache = Cache(get_hostname(request))
if cache.blacklisted():
raise exception_response(
400,
title="Sorry the host URL '{0}' has been "
"blacklisted. Please contact Stub-O-Matic support.".format(
cache.host))
scenario_key = cache.find_scenario_key(session_name)
scenario_name = scenario_key.partition(':')[-1]
handler.track.scenario = scenario_name
request_id = compute_hash(request_body)
module_system_date = handler.get_argument('system_date', None)
url_args = handler.track.request_params
if not module_system_date:
# BA LEGACY
module_system_date = handler.get_argument('stubbedSystemDate', None)
trace_matcher = TrackTrace(handler.track, 'matcher')
user_cache = handler.settings['ext_cache']
# check cached requests
cached_request = cache.get_request(scenario_name, session_name, request_id)
if cached_request:
response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request
else:
retry_count = 5 if handler.settings.get('is_cluster', False) else 1
session, retries = cache.get_session_with_delay(
scenario_name,
session_name,
retry_count=retry_count,
retry_interval=1)
if retries > 0:
log.warn("replication was slow for session: {0} {1}, it took {2} "\
"secs!".format(scenario_key, session_name, retries+1))
if session['status'] != 'playback':
raise exception_response(
500,
title='cache status != playback. session={0}'.format(session))
system_date = session['system_date']
if not system_date:
raise exception_response(
500,
title="slave session {0} not available for scenario {1}".
format(session_name, scenario_key))
session['ext_cache'] = user_cache
results = match(stubo_request,
session,
trace_matcher,
as_date(system_date),
url_args=url_args,
hooks=handler.settings['hooks'],
module_system_date=module_system_date)
# 0 is best match
hits_info, stub = results[0]
log.debug(u'match result: (hits_info={0}, stub={1})'.format(
hits_info, stub.payload))
hits, stub_number = hits_info
if not hits:
raise exception_response(400,
title='E017:No matching response found')
response_ids = stub.response_ids()
delay_policy_name = stub.delay_policy_name()
recorded = stub.recorded()
module_info = stub.module()
request_index_key = add_request(
session, request_id, stub, system_date, stub_number,
handler.settings['request_cache_limit'])
if not stub.response_body():
stub.set_response_body(
stub.get_response_body_from_cache(request_index_key))
if delay_policy_name:
stub.load_delay_from_cache(delay_policy_name)
if cached_request:
stub = StubCache({}, scenario_key, session_name)
stub.load_from_cache(response_ids, delay_policy_name, recorded,
system_date, module_info, request_index_key)
trace_response = TrackTrace(handler.track, 'response')
if module_info:
trace_response.info('module used', str(module_info))
response_text = stub.response_body()
if not response_text:
raise exception_response(
500,
title='Unable to find response in '
'cache using session: {0}:{1}, response_ids: {2}'.format(
scenario_key, session_name, response_ids))
# get latest delay policy
delay_policy = stub.delay_policy()
if delay_policy:
delay = calculate_delay(delay_policy)
if delay:
msg = 'apply delay: {0} => {1}'.format(delay_policy, delay)
log.debug(msg)
handler.track['delay'] = delay
trace_response.info(msg)
trace_response.info('found response')
module_system_date = as_date(module_system_date) if module_system_date \
else module_system_date
stub, _ = transform(stub,
stubo_request,
module_system_date=module_system_date,
system_date=as_date(system_date),
function='get/response',
cache=user_cache,
hooks=handler.settings['hooks'],
stage='response',
trace=trace_response,
url_args=url_args)
transfomed_response_text = stub.response_body()[0]
# Note transformed_response_text can be encoded in utf8
if response_text[0] != transfomed_response_text:
trace_response.diff('response:transformed',
dict(response=response_text[0]),
dict(response=transfomed_response_text))
return transfomed_response_text
def get_response(handler, session_name):
request = handler.request
stubo_request = StuboRequest(request)
cache = Cache(get_hostname(request))
if cache.blacklisted():
raise exception_response(400, title="Sorry the host URL '{0}' has been "
"blacklisted. Please contact Stub-O-Matic support.".format(cache.host))
scenario_key = cache.find_scenario_key(session_name)
scenario_name = scenario_key.partition(':')[-1]
handler.track.scenario = scenario_name
request_id = stubo_request.id()
module_system_date = handler.get_argument('system_date', None)
url_args = handler.track.request_params
if not module_system_date:
# LEGACY
module_system_date = handler.get_argument('stubbedSystemDate', None)
trace_matcher = TrackTrace(handler.track, 'matcher')
user_cache = handler.settings['ext_cache']
# check cached requests
cached_request = cache.get_request(scenario_name, session_name, request_id)
if cached_request:
response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request
else:
retry_count = 5 if handler.settings.get('is_cluster', False) else 1
session, retries = cache.get_session_with_delay(scenario_name,
session_name,
retry_count=retry_count,
retry_interval=1)
if retries > 0:
log.warn("replication was slow for session: {0} {1}, it took {2} "\
"secs!".format(scenario_key, session_name, retries+1))
if session['status'] != 'playback':
raise exception_response(500,
title='cache status != playback. session={0}'.format(session))
system_date = session['system_date']
if not system_date:
raise exception_response(500,
title="slave session {0} not available for scenario {1}".format(
session_name, scenario_key))
session['ext_cache'] = user_cache
result = match(stubo_request, session, trace_matcher,
as_date(system_date),
url_args=url_args,
hooks=handler.settings['hooks'],
module_system_date=module_system_date)
if not result[0]:
raise exception_response(400,
title='E017:No matching response found')
_, stub_number, stub = result
response_ids = stub.response_ids()
delay_policy_name = stub.delay_policy_name()
recorded = stub.recorded()
module_info = stub.module()
request_index_key = add_request(session, request_id, stub, system_date,
stub_number,
handler.settings['request_cache_limit'])
if not stub.response_body():
_response = stub.get_response_from_cache(request_index_key)
stub.set_response_body(_response['body'])
if delay_policy_name:
stub.load_delay_from_cache(delay_policy_name)
if cached_request:
stub = StubCache({}, scenario_key, session_name)
stub.load_from_cache(response_ids, delay_policy_name, recorded,
system_date, module_info, request_index_key)
trace_response = TrackTrace(handler.track, 'response')
if module_info:
trace_response.info('module used', str(module_info))
response_text = stub.response_body()
if not response_text:
raise exception_response(500, title='Unable to find response in '
'cache using session: {0}:{1}, response_ids: {2}'.format(
scenario_key, session_name, response_ids))
# get latest delay policy
delay_policy = stub.delay_policy()
if delay_policy:
delay = Delay.parse_args(delay_policy)
if delay:
delay = delay.calculate()
msg = 'apply delay: {0} => {1}'.format(delay_policy, delay)
log.debug(msg)
handler.track['delay'] = delay
trace_response.info(msg)
trace_response.info('found response')
module_system_date = as_date(module_system_date) if module_system_date \
else module_system_date
stub, _ = transform(stub,
stubo_request,
module_system_date=module_system_date,
system_date=as_date(system_date),
function='get/response',
cache=user_cache,
hooks=handler.settings['hooks'],
stage='response',
trace=trace_response,
url_args=url_args)
transfomed_response_text = stub.response_body()[0]
# Note transformed_response_text can be encoded in utf8
if response_text[0] != transfomed_response_text:
trace_response.diff('response:transformed',
dict(response=response_text[0]),
dict(response=transfomed_response_text))
if stub.response_status() != 200:
handler.set_status(stub.response_status())
if stub.response_headers():
for k, v in stub.response_headers().iteritems():
handler.set_header(k, v)
return transfomed_response_text
def get_response(handler, session_name):
# getting request value
request = handler.request
stubo_request = StuboRequest(request)
cache = Cache(get_hostname(request))
scenario_key = cache.find_scenario_key(session_name)
scenario_name = scenario_key.partition(':')[-1]
handler.track.scenario = scenario_name
# request_id - computed hash
request_id = stubo_request.id()
module_system_date = handler.get_argument('system_date', None)
url_args = handler.track.request_params
if not module_system_date:
# LEGACY
module_system_date = handler.get_argument('stubbedSystemDate', None)
trace_matcher = TrackTrace(handler.track, 'matcher')
user_cache = handler.settings['ext_cache']
# check cached requests
cached_request = cache.get_request(scenario_name, session_name, request_id)
if cached_request:
response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request
else:
retry_count = 5 if handler.settings.get('is_cluster', False) else 1
session, retries = cache.get_session_with_delay(
scenario_name,
session_name,
retry_count=retry_count,
retry_interval=1)
if retries > 0:
log.warn(
"replication was slow for session: {0} {1}, it took {2} secs!".
format(scenario_key, session_name, retries + 1))
if session['status'] != 'playback':
raise exception_response(
500,
title='cache status != playback. session={0}'.format(session))
system_date = session['system_date']
if not system_date:
raise exception_response(
500,
title="slave session {0} not available for scenario {1}".
format(session_name, scenario_key))
session['ext_cache'] = user_cache
result = match(stubo_request,
session,
trace_matcher,
as_date(system_date),
url_args=url_args,
hooks=handler.settings['hooks'],
module_system_date=module_system_date)
if not result[0]:
raise exception_response(400,
title='E017:No matching response found')
_, stub_number, stub = result
response_ids = stub.response_ids()
delay_policy_name = stub.delay_policy_name()
recorded = stub.recorded()
module_info = stub.module()
request_index_key = add_request(
session, request_id, stub, system_date, stub_number,
handler.settings['request_cache_limit'])
if not stub.response_body():
_response = stub.get_response_from_cache(request_index_key)
stub.set_response_body(_response['body'])
if delay_policy_name:
stub.load_delay_from_cache(delay_policy_name)
if cached_request:
stub = StubCache({}, scenario_key, session_name)
stub.load_from_cache(response_ids, delay_policy_name, recorded,
system_date, module_info, request_index_key)
trace_response = TrackTrace(handler.track, 'response')
if module_info:
trace_response.info('module used', str(module_info))
response_text = stub.response_body()
if not response_text:
raise exception_response(
500,
title='Unable to find response in cache using session: {0}:{1}, '
'response_ids: {2}'.format(scenario_key, session_name,
response_ids))
# get latest delay policy
delay_policy = stub.delay_policy()
if delay_policy:
delay = Delay.parse_args(delay_policy)
if delay:
delay = delay.calculate()
msg = 'apply delay: {0} => {1}'.format(delay_policy, delay)
log.debug(msg)
handler.track['delay'] = delay
trace_response.info(msg)
trace_response.info('found response')
module_system_date = as_date(module_system_date) if module_system_date \
else module_system_date
stub, _ = transform(stub,
stubo_request,
module_system_date=module_system_date,
system_date=as_date(system_date),
function='get/response',
cache=user_cache,
hooks=handler.settings['hooks'],
stage='response',
trace=trace_response,
url_args=url_args)
transfomed_response_text = stub.response_body()[0]
# Note transformed_response_text can be encoded in utf8
if response_text[0] != transfomed_response_text:
trace_response.diff('response:transformed',
dict(response=response_text[0]),
dict(response=transfomed_response_text))
if stub.response_status() != 200:
handler.set_status(stub.response_status())
if stub.response_headers():
for k, v in stub.response_headers().iteritems():
handler.set_header(k, v)
return transfomed_response_text
def get_response_v2(handler, full_scenario_name, session_name):
# main result dictionary that will be returned
result_dict = {}
request = handler.request
stubo_request = StuboRequest(request)
cache = Cache(get_hostname(request))
scenario_name = full_scenario_name.partition(':')[-1]
handler.track.scenario = scenario_name
request_id = stubo_request.id()
module_system_date = handler.get_argument('system_date', None)
url_args = handler.track.request_params
# trace_matcher = TrackTrace(handler.track, 'matcher')
user_cache = handler.settings['ext_cache']
# check cached requests
cached_request = cache.get_request(scenario_name, session_name, request_id)
if cached_request:
response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request
else:
retry_count = 5 if handler.settings.get('is_cluster', False) else 1
session, retries = cache.get_session_with_delay(
scenario_name,
session_name,
retry_count=retry_count,
retry_interval=1)
if retries > 0:
log.warn(
"replication was slow for session: {0} {1}, it took {2} secs!".
format(full_scenario_name, session_name, retries + 1))
if session['status'] != 'playback':
result_dict[
"error"] = 'cache status != playback. session={0}'.format(
session)
result_dict["statusCode"] = 412
return result_dict
system_date = session['system_date']
if not system_date:
result_dict[
"error"] = "slave session {0} not available for scenario {1}".format(
session_name, full_scenario_name)
result_dict["statusCode"] = 412
return result_dict
trace_matcher = TrackTrace(handler.track, 'matcher')
session['ext_cache'] = user_cache
result = match(stubo_request,
session,
trace_matcher,
as_date(system_date),
url_args=url_args,
hooks=handler.settings['hooks'],
module_system_date=module_system_date)
# matching request not found
if not result[0]:
result_dict["error"] = "Not matching request found"
result_dict["statusCode"] = 404
return result_dict
_, stub_number, stub = result
response_ids = stub.response_ids()
delay_policy_name = stub.delay_policy_name()
recorded = stub.recorded()
module_info = stub.module()
request_index_key = add_request(
session, request_id, stub, system_date, stub_number,
handler.settings['request_cache_limit'])
if not stub.response_body():
_response = stub.get_response_from_cache(request_index_key)
stub.set_response_body(_response['body'])
if delay_policy_name:
stub.load_delay_from_cache(delay_policy_name)
if cached_request:
stub = StubCache({}, full_scenario_name, session_name)
stub.load_from_cache(response_ids, delay_policy_name, recorded,
system_date, module_info, request_index_key)
trace_response = TrackTrace(handler.track, 'response')
if module_info:
trace_response.info('module used', str(module_info))
response_text = stub.response_body()
if not response_text:
result_dict[
"error"] = 'Unable to find response in cache using session: {0}:{1}, '
'response_ids: {2}'.format(full_scenario_name, session_name,
response_ids)
result_dict["statusCode"] = 400
return result_dict
# get latest delay policy
delay_policy = stub.delay_policy()
if delay_policy:
delay = Delay.parse_args(delay_policy)
if delay:
delay = delay.calculate()
msg = 'apply delay: {0} => {1}'.format(delay_policy, delay)
log.debug(msg)
handler.track['delay'] = delay
trace_response.info(msg)
trace_response.info('found response')
module_system_date = as_date(module_system_date) if module_system_date \
else module_system_date
stub, _ = transform(stub,
stubo_request,
module_system_date=module_system_date,
system_date=as_date(system_date),
function='get/response',
cache=user_cache,
hooks=handler.settings['hooks'],
stage='response',
trace=trace_response,
url_args=url_args)
transfomed_response_text = stub.response_body()[0]
# Note transformed_response_text can be encoded in utf8
if response_text[0] != transfomed_response_text:
trace_response.diff('response:transformed',
dict(response=response_text[0]),
dict(response=transfomed_response_text))
result_dict["body"] = transfomed_response_text
result_dict["headers"] = stub.response_headers()
result_dict["statusCode"] = stub.response_status()
return result_dict
