def assert_valid_session(self, scenario_name, session_name):
scenario_key = self.scenario_key_name(scenario_name)
# if session exists it can only be dormant
if self.exists(scenario_key, session_name):
session = self.get_session(scenario_name, session_name, local=False)
session_status = session['status']
if session_status != 'dormant':
raise exception_response(400, title='Session already exists '
'- {0}:{1} in {2} mode.'.format(scenario_key, session_name,
session_status))
# Limitation: as the session name is not passed in as an arg to get/response
# we need a mapping to find the session for a scenario.
# host:sessions -> session_name->scenario_name
# we can therefore only have one session name per host/scenario
sessions_key = '{0}:sessions'.format(self.host)
scenario_found = self.get_cache_backend()(get_redis_master()).get_raw(sessions_key,
session_name)
if scenario_found and scenario_found != scenario_name:
raise exception_response(400, title='Session {0} can not be '
'used for scenario: {1}. This session is already being used '
'with another scenario: {2} on host: {3}.'.format(session_name,
scenario_name,
scenario_found,
self.host))
def export_playback(self, host, export_payload, files, session,
playback_session):
playback_payload = export_payload['playback']
scenario_name = playback_payload['scenario']
scenario = u'{0}:{1}'.format(host, scenario_name)
runnable_info = dict()
tracker = Tracker()
last_used = tracker.session_last_used(scenario, playback_session,
'playback')
if not last_used:
raise exception_response(400,
title="Unable to find playback session")
runnable_info['last_used'] = dict(remote_ip=last_used['remote_ip'],
start_time=str(last_used['start_time']))
playback = tracker.get_last_playback(scenario_name, playback_session,
last_used['start_time'])
playback = list(playback)
if not playback:
raise exception_response(400,
title="Unable to find a playback for scenario='{0}', playback_session='{1}'".format(scenario_name, playback_session))
number_of_requests = len(playback)
runnable_info['number_of_playback_requests'] = number_of_requests
for nrequest in range(number_of_requests):
track = playback[nrequest]
request_text = track.get('request_text')
if not request_text:
raise exception_response(400, title='Unable to obtain playback details, was full tracking enabled?')
stubo_request = StuboRequest(DummyModel(headers=track.get('request_headers'),
body=request_text))
vars = track.get('request_params')
vars.pop('session', None)
vars.pop('scenario', None)
request_payload = dict(body=stubo_request.body,
method=stubo_request.method,
host=stubo_request.host,
uri=stubo_request.uri,
path=stubo_request.path,
query=stubo_request.query,
headers=stubo_request.headers)
request_file_name = '{0}_{1}.request'.format(session, nrequest)
files.append((request_file_name, json.dumps(request_payload, indent=3)))
# export a response for comparison
stubo_response_text = track['stubo_response']
if not isinstance(stubo_response_text, basestring):
stubo_response_text = unicode(stubo_response_text)
response_payload = dict(status=track.get('return_code'),
body=stubo_response_text,
headers=track.get('response_headers'))
stubo_response_file_name = '{0}_{1}.stubo_response'.format(session,
nrequest)
playback_payload['requests'].append(dict(
file=request_file_name,
vars=vars,
response=stubo_response_file_name))
files.append((stubo_response_file_name, json.dumps(response_payload,
indent=3)))
return runnable_info
def put_bookmark(handler, session_name, name):
cache = Cache(get_hostname(handler.request))
response = dict(version=version, data = {})
if not session_name:
raise exception_response(400, title="No session provided")
scenario_key = cache.find_scenario_key(session_name)
scenario_name = scenario_key.partition(':')[-1]
# retrieve the request index state for selected session
index_state = {}
request_index_data = cache.get_request_index_data(scenario_name)
if request_index_data:
for k, v in request_index_data.iteritems():
indexed_session_name, _, stub_key = k.partition(':')
if indexed_session_name == session_name:
index_state[stub_key] = v
if not index_state:
raise exception_response(400,
title="No indexes found for session '{0}'. Is the session in "
'playback mode and have state?'.format(session_name))
log.debug("save request index state '{0}' = {1}".format(name, index_state))
cache.set_saved_request_index_data(scenario_name, name, index_state)
response['data'][name] = index_state
return response
def store_source_recording(scenario_name_key, record_session):
host, scenario_name = scenario_name_key.split(':')
# use original put/stub payload logged in tracker
tracker = Tracker()
last_used = tracker.session_last_used(scenario_name_key,
record_session, 'record')
if not last_used:
# empty recordings are currently supported!
log.debug('Unable to find a recording for session={0}, scenario={1}'.format(record_session, scenario_name_key))
return
recording = tracker.get_last_recording(scenario_name, record_session,
last_used['start_time'])
recording = list(recording)
if not recording:
raise exception_response(400,
title="Unable to find a recording for scenario='{0}', record_session='{1}'".format(scenario_name, record_session))
number_of_requests = len(recording)
scenario_db = Scenario()
for nrequest in range(number_of_requests):
track = recording[nrequest]
request_text = track.get('request_text')
if not request_text:
raise exception_response(400, title='Unable to obtain recording details, was full tracking enabled?')
priority = int(track['request_params'].get('priority', nrequest+1))
stub = parse_stub(request_text, scenario_name_key,
track['request_params'])
stub.set_priority(priority)
scenario_db.insert_pre_stub(scenario_name_key, stub)
def match(request, session, trace, system_date, url_args, hooks,
module_system_date=None):
"""Returns the stats of a request match process
:param request: source stubo request
:param session: cached session payload associated with this request
:param module_system_date: optional system date of an external module
"""
request_text = request.request_body()
scenario_key = session['scenario']
session_name = session['session']
log.debug(u'match: request_text={0}'.format(request_text))
trace.info('system_date={0}, module_system_date={1}'.format(
system_date, module_system_date))
stats = []
if 'stubs' not in session or not len(session['stubs']):
if session.get('status') != 'playback':
raise exception_response(400,
title="session {0} not in playback mode for scenario "
"{1}".format(session_name, scenario_key))
raise exception_response(500,
title="no stubs found in session {0} for {1}, status={2}".format(
session_name, scenario_key, session.get('status')))
stub_count = len(session['stubs'])
trace.info(u'matching against {0} stubs'.format(stub_count))
for stub_number in range(stub_count):
trace.info('stub ({0})'.format(stub_number))
stub = StubCache(session['stubs'][stub_number], scenario_key,
session_name)
source_stub = copy.deepcopy(stub)
request_copy = copy.deepcopy(request)
stub, request_copy = transform(stub,
request_copy,
module_system_date=module_system_date,
system_date=system_date,
function='get/response',
cache=session.get('ext_cache'),
hooks=hooks,
stage='matcher',
trace=trace,
url_args=url_args)
trace.info('finished transformation')
if source_stub != stub:
trace.diff('stub ({0}) was transformed'.format(stub_number),
source_stub.payload, stub.payload)
trace.info('stub ({0}) was transformed into'.format(stub_number),
stub.payload)
if request_copy != request:
trace.diff('request was transformed', request_copy.request_body(),
request.request_body())
trace.info('request was transformed into', request_copy.request_body())
matcher = StubMatcher(trace)
if matcher.match(request_copy, stub):
return (True, stub_number, stub)
return (False,)
def run_recording(self, recording):
scenario_args = dict(scenario=recording['scenario'],
session=recording['session'])
recordings = []
recordings.append(('delete/stubs', self.run_command('delete/stubs',
**scenario_args)))
recordings.append(('begin/session', self.run_command('begin/session',
mode='record', **scenario_args)))
for stub in recording['stubs']:
try:
if 'file' in stub:
stub_data_url = urljoin(self.parent_path, stub['file'])
stub_payload, _, _ = UrlFetch().get(stub_data_url)
elif 'json' in stub:
stub_payload = stub['json']
else:
raise exception_response(400, title="A stub definition must "
"contain either a 'file' location key or a 'json' key that "
"defines an inplace payload.")
vars = stub.get('vars', {})
vars.update(scenario_args)
url = self.get_url('put/stub', **vars)
log.debug(u'run_command: {0}'.format(url))
if not isinstance(stub_payload, dict):
stub_payload = json.loads(stub_payload)
response = UrlFetch().post(url, data=None, json=stub_payload)
recordings.append(('put/stub', response.status_code))
except Exception, e:
recordings.append(('put/stub', str(e)))
def find_scenario_key(self, session_name):
scenario_key = self.get_scenario_key(session_name)
if not scenario_key:
raise exception_response(400,
title='session not found - {0}:{1}'.format(self.host,
session_name))
return scenario_key
def put_module(handler, names):
module = Module(handler.track.host)
added = []
result = dict(version=version)
for name in names:
uri, module_name = UriLocation(handler.request)(name)
log.info('uri={0}, module_name={1}'.format(uri, module_name))
response, _, code = UrlFetch().get(uri)
module_name = module_name[:-3]
last_version = module.latest_version(module_name)
module_version_name = module.sys_module_name(module_name,
last_version+1)
if last_version and response == module.get_source(module_name,
last_version):
msg = 'Module source has not changed for {0}'.format(
module_version_name)
result['data'] = dict(message=msg)
try:
code, mod = module.add_sys_module(module_version_name, response)
log.debug('{0}, {1}'.format(mod, code))
except Exception, e:
msg = 'error={0}'.format(e)
raise exception_response(400,
title='Unable to compile {0}:{1}, {2}'.format(module.host(),
module_version_name, msg))
module.add(module_name, response)
added.append(module_version_name)
def _begin_session(self):
"""
Begins session
:raise exception_response:
Example output:
Record new session
{
"version": "0.6.3",
"data":
{"status": "record",
"scenario": "localhost:scenario_rest_api",
"scenarioRef": "/stubo/api/v2/scenarios/objects/localhost:scenario_rest_api",
"scenario_id": "55acba53fc456205eaf7e258",
"session": "new_session_rest2",
"message": "Record mode initiated...."}
}
"""
warm_cache = asbool(self.get_argument('warm_cache', False))
if not self.mode:
raise exception_response(400,
title="'mode' of playback or record required")
# passing parameters to api v2 handler, it avoids creating scenario if there is an existing one,
# since all scenarios should be existing!
response = api_v2_begin_session(self, self.scenario_name,
self.session_name,
self.mode,
self.get_argument('system_date', None),
warm_cache)
# adding scenarioRef key for easier resource access.
response['data']['scenarioRef'] = '/stubo/api/v2/scenarios/objects/%s' % response['data']['scenario']
self.write(response)
def run_commands(handler, cmds_text):
response = {
'version': version
}
host = get_hostname(handler.request)
cmd_processor = TextCommandsImporter(UriLocation(handler.request))
cmds = cmd_processor.parse(cmds_text)
if any(x for x in cmds if urlparse(x).path not in form_input_cmds):
raise exception_response(400, title='command/s not supported, must be '
'one of these: {0}'.format(form_input_cmds))
responses = cmd_processor.run_cmds(cmds)
log.debug('responses: {0}'.format(responses))
response['data'] = {
'executed_commands': responses,
'number_of_requests': len(responses['commands']),
'number_of_errors': len([x for x in responses['commands'] if x[1] > 399])
}
def get_links(cmd):
cmd_uri = urlparse(cmd)
scenario_name = cmd_uri.query.partition('=')[-1]
scenario_name_key = '{0}:{1}'.format(host, scenario_name)
files = [(scenario_name + '.zip',), (scenario_name + '.tar.gz',),
(scenario_name + '.jar',)]
links = get_export_links(handler, scenario_name_key, files)
return links
export_links = [(x, get_links(x)) for x in cmds if 'get/export' in x]
if export_links:
response['data']['export_links'] = export_links
return response
def run(self):
if not self.cmd_file_url:
raise exception_response(500,
title='run requires a cmd_file_url input to the ctor.')
cmds, _, _ = UrlFetch().get(self.location(self.cmd_file_url)[0])
cmds_expanded = run_template(cmds,
# utility functions
roll_date=roll_date,
today=today_str,
as_date=as_date,
parse_xml=parse_xml,
**self.location.request.arguments)
try:
payload = self.parse(cmds_expanded)
except Exception, e:
raise exception_response(400, title="Unable to parse '{0}', error={1}".format(self.cmd_file_url, e))
def run_recording(self, recording):
scenario_args = dict(scenario=recording["scenario"], session=recording["session"])
recordings = []
recordings.append(("delete/stubs", self.run_command("delete/stubs", **scenario_args)))
recordings.append(("begin/session", self.run_command("begin/session", mode="record", **scenario_args)))
for stub in recording["stubs"]:
try:
if "file" in stub:
stub_data_url = urljoin(self.parent_path, stub["file"])
stub_payload, _, _ = UrlFetch().get(stub_data_url)
elif "json" in stub:
stub_payload = stub["json"]
else:
raise exception_response(
400,
title="A stub definition must "
"contain either a 'file' location key or a 'json' key that "
"defines an inplace payload.",
)
vars = stub.get("vars", {})
vars.update(scenario_args)
url = self.get_url("put/stub", **vars)
log.debug(u"run_command: {0}".format(url))
if not isinstance(stub_payload, dict):
stub_payload = json.loads(stub_payload)
response = UrlFetch().post(url, data=None, json=stub_payload)
recordings.append(("put/stub", response.status_code))
except Exception, e:
recordings.append(("put/stub", str(e)))
def run_config(files):
# find the config file in the extract and run it
config_file = [x for x in files if x.endswith(config_types)]
if not config_file:
raise exception_response(400, title='Config file not'
' found in archive: {0}'.format(cmd_file_url))
return run(os.path.join('static', 'imports', temp_dir_name,
config_file[0]))
def begin_session_request(handler):
scenario = handler.track.scenario = get_scenario_arg(handler)
session = get_session_arg(handler)
mode = handler.get_argument('mode', None)
warm_cache = asbool(handler.get_argument('warm_cache', False))
if not mode:
raise exception_response(400,
title="'mode' of playback or record required")
return begin_session(handler, scenario, session, mode,
handler.get_argument('system_date', None), warm_cache)
def run_command_file(cmd_file_url, request, static_path):
def run(cmd_file_path):
response = {
'version' : version
}
is_legacy_text_format = cmd_file_path.endswith('.commands')
location = UriLocation(request)
cmd_processor = TextCommandsImporter(location, cmd_file_path) if is_legacy_text_format else YAMLImporter(location, cmd_file_path)
responses = cmd_processor.run()
log.debug('responses: {0}'.format(responses))
response['data'] = responses
return response
file_type = os.path.basename(urlparse(cmd_file_url).path).rpartition(
'.')[-1]
supported_types = ('zip', 'gz', 'tar', 'jar')
if file_type in supported_types:
# import compressed contents and run contained config file
import_dir = os.path.join(static_path, 'imports')
with make_temp_dir(dirname=import_dir) as temp_dir:
temp_dir_name = os.path.basename(temp_dir)
response, headers, status_code = UrlFetch().get(
UriLocation(request)(cmd_file_url)[0])
content_type = headers["Content-Type"]
log.debug('received {0} file.'.format(content_type))
config_types = ('.yaml', '.commands')
def run_config(files):
# find the config file in the extract and run it
config_file = [x for x in files if x.endswith(config_types)]
if not config_file:
raise exception_response(400, title='Config file not'
' found in archive: {0}'.format(cmd_file_url))
return run(os.path.join('static', 'imports', temp_dir_name,
config_file[0]))
if content_type == 'application/x-tar' or file_type == 'tar':
with closing(tarfile.open(fileobj=StringIO(response))) as tar:
tar.extractall(path=temp_dir)
response = run_config(tar.getnames())
elif content_type in ('application/zip',
'application/java-archive') or file_type in \
('zip', 'jar'):
with zipfile.ZipFile(StringIO(response)) as zipf:
zipf.extractall(path=temp_dir)
response = run_config(zipf.namelist())
else:
raise exception_response(400, title='Expected Content-Type has'
' to be one of these: {0} not {1}'.format(supported_types,
content_type))
else:
response = run(cmd_file_url)
return response
def get_stats(handler):
# cluster.host
# "cluster1.ahost or cluster1.*",
response = {
'version': version
}
cluster = handler.get_argument('cluster', handler.settings['cluster_name'])
host = handler.get_argument('host', '*')
cluster_host_server = '{0}.{1}'.format(cluster, host)
metric = handler.get_argument('metric', 'latency')
if metric == 'latency':
target = 'averageSeries(stats.timers.stubo.{0}.stuboapi.get_response.latency.mean_90)'.format(
cluster_host_server)
percent_above_value = int(handler.get_argument('percent_above_value', 50))
else:
raise exception_response(400,
title="metric '{0}' parameter not supported.".format(metric))
server = handler.settings.get('graphite.host')
auth = (handler.settings.get('graphite.user'),
handler.settings.get('graphite.passwd'))
from_str = handler.get_argument('from', '-1hours')
to_str = handler.get_argument('to', 'now')
json_response, hdrs, status_code = get_graphite_stats(server, auth,
target=target, from_str=from_str, to_str=to_str)
if status_code != 200 or (hdrs['content-type'] != 'application/json'):
raise exception_response(500,
title='unexpected response from graphite => {0}: {1}'.format(
hdrs, json_response))
ts = get_graphite_datapoints(json_response, target)
slow = [x for x in ts if x[0] > percent_above_value]
pcent = len(slow) / float(len(ts)) * 100
response['data'] = {
'target': target,
'metric': metric,
'pcent': pcent,
'percent_above_value': percent_above_value,
'from': from_str,
'to': to_str
}
return response
def get_tracks(handler, scenario_filter, session_filter, show_only_errors, skip,
limit, start_time, latency, all_hosts, function):
tracker = Tracker()
tracker_filter = {}
if start_time:
try:
start = datetime.datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
tracker_filter['start_time'] = {"$lt": start}
except ValueError, e:
raise exception_response(400,
title='start_time format error: {0}'.format(e))
def get_sys_module(self, name, version):
# lazy load if not in already in sys.modules
module_name = self.sys_module_name(name, version)
if module_name not in sys.modules:
# load from source
code = self.get_source(name, version)
if not code:
raise exception_response(500, title="module '{0}' source not "
"found.".format(module_name))
self.add_sys_module(module_name, code)
return sys.modules[module_name]
def raise_on_error(self, response, url):
status = response.status_code
if status != 200:
if status == 404:
msg = "File not found using url: {0}".format(url)
raise exception_response(response.status_code, title=msg)
else:
json_response = None
try:
json_response = response.json()
except Exception:
pass
if json_response and "error" in json_response:
# its one of ours, reconstruct the error
raise exception_response(response.status_code, title=json_response["error"].get("message"))
if response.status_code > 399:
raise exception_response(
response.status_code,
title="Error executing request '{0}', reason={1}".format(url, response.reason),
)
def get_session_with_delay(self, scenario_name, session_name, retry_count=5,
retry_interval=1):
for i in range(retry_count):
scenario_key = self.scenario_key_name(scenario_name)
session = self.get_session(scenario_name, session_name)
if not session:
raise exception_response(500,
title="session {0} not found!".format(session_name))
if session['status'] == 'playback':
# indicates session has been replicated
break
elif session['status'] == 'record':
raise exception_response(409, title="session '{0}' for scenario" \
"'{1}' in record mode, playback expected ...".format(
session_name, scenario_key))
else:
log.warn("slave session data not available! try again in {0} "
'secs'.format(retry_interval))
time.sleep(retry_interval)
return session, i
def run_playback(self, playback):
scenario_args = dict(scenario=playback["scenario"], session=playback["session"])
plays = []
plays.append(("begin/session", self.run_command("begin/session", mode="playback", **scenario_args)))
for request in playback["requests"]:
"""
{"method": "GET",
"body": '{"body": {"cmd": {"a": "b"}}',
"headers" : {
"Content-Type" : "application/json",
"X-Custom-Header" : "1234"
}}
"""
try:
if "file" in request:
request_data_url = urljoin(self.parent_path, request["file"])
payload, _, _ = UrlFetch().get(request_data_url)
elif "json" in request:
payload = request["json"]
else:
raise exception_response(
400,
title="A request definition "
"must contain either a 'file' location key or a 'json' "
"key that defines an inplace payload.",
)
if isinstance(payload, basestring):
payload = json.loads(payload)
vars = request.get("vars", {})
vars.update(scenario_args)
url = self.get_url("get/response", **vars)
log.debug(u"run_command: {0}".format(url))
body = payload["body"]
if isinstance(body, dict):
# payload is json
encoded_data = json.dumps(body)
else:
encoded_data = body.encode("utf-8")
headers = {
"Stubo-Request-URI": payload.get("uri"),
"Stubo-Request-Method": payload.get("method", "POST"),
"Stubo-Request-Headers": payload.get("headers", ""),
"Stubo-Request-Host": payload.get("host"),
"Stubo-Request-Path": payload.get("path"),
"Stubo-Request-Query": payload.get("query", ""),
}
response = UrlFetch().post(url, data=encoded_data, headers=headers)
plays.append(("get/response", response.status_code))
except Exception, e:
plays.append(("get/response", 500, str(e)))
def get_session_from_headers():
session_name = request.headers.get('Stubo-Request-Session')
if session_name:
return session_name
# NOTE: legacy support for session_name = stb_scenario + '_' + stb_session
session = [(k, v) for k, v in request.headers.iteritems() \
if 'stb_session' in k or 'Stb_session' in k]
scenario = [(k, v) for k, v in request.headers.iteritems() \
if 'stb_scenario' in k or 'Stb_scenario' in k]
if not session:
raise exception_response(400,
title="session not supplied in headers.")
if not scenario:
raise exception_response(400,
title="scenario parameter not supplied in headers.")
_, session_name = session[0]
scenario_key, scenario_name = scenario[0]
session_name = '{0}_{1}'.format(scenario_name, session_name)
# put into standard key for tracker display
handler.track.request_headers['Stubo-Request-Session'] = session_name
return session_name
def delete_stubs(handler, scenario_name=None, host=None, force=False):
"""delete all data relating to one named scenario or host/s."""
log.debug('delete_stubs')
response = {
'version': version
}
scenario_db = Scenario()
static_dir = handler.settings['static_path']
def delete_scenario(sce_name_key, frc):
log.debug(u'delete_scenario: {0}'.format(sce_name_key))
# getting host and scenario names
hst, sce_name = sce_name_key.split(':')
cache = Cache(hst)
if not frc:
active_sessions = cache.get_active_sessions(sce_name,
local=False)
if active_sessions:
raise exception_response(400,
title='Sessons in playback/record, can not delete. '
'Found the following active sessions: {0} '
'for scenario: {1}'.format(active_sessions, sce_name))
scenario_db.remove_all(sce_name_key)
cache.delete_caches(sce_name)
scenarios = []
if scenario_name:
# if scenario_name exists it takes priority
handler.track.scenario = scenario_name
hostname = host or get_hostname(handler.request)
scenarios.append(':'.join([hostname, scenario_name]))
elif host:
if host == 'all':
scenarios = [x['name'] for x in scenario_db.get_all()]
export_dir = os.path.join(static_dir, 'exports')
if os.path.exists(export_dir):
log.info('delete export dir')
shutil.rmtree(export_dir)
else:
# get all scenarios for host
scenarios = [x['name'] for x in scenario_db.get_all(
{'$regex': '{0}:.*'.format(host)})]
else:
raise exception_response(400,
title='scenario or host argument required')
for scenario_name_key in scenarios:
delete_scenario(scenario_name_key, force)
response['data'] = dict(message='stubs deleted.', scenarios=scenarios)
return response
def run_playback(self, playback):
scenario_args = dict(scenario=playback['scenario'],
session=playback['session'])
plays = []
plays.append(('begin/session', self.run_command('begin/session',
mode='playback', **scenario_args)))
for request in playback['requests']:
"""
{"method": "GET",
"body": '{"body": {"cmd": {"a": "b"}}',
"headers" : {
"Content-Type" : "application/json",
"X-Custom-Header" : "1234"
}}
"""
try:
if 'file' in request:
request_data_url = urljoin(self.parent_path, request['file'])
payload, _, _ = UrlFetch().get(request_data_url)
elif 'json' in request:
payload = request['json']
else:
raise exception_response(400, title="A request definition "
"must contain either a 'file' location key or a 'json' "
"key that defines an inplace payload.")
if isinstance(payload, basestring):
payload = json.loads(payload)
vars = request.get('vars', {})
vars.update(scenario_args)
url = self.get_url('get/response', **vars)
log.debug(u'run_command: {0}'.format(url))
body = payload['body']
if isinstance(body, dict):
# payload is json
encoded_data = json.dumps(body)
else:
encoded_data = body.encode('utf-8')
headers = {
'Stubo-Request-URI': payload.get('uri'),
'Stubo-Request-Method': payload.get('method', 'POST'),
'Stubo-Request-Headers': payload.get('headers', ''),
'Stubo-Request-Host': payload.get('host'),
'Stubo-Request-Path': payload.get('path'),
'Stubo-Request-Query': payload.get('query', '')
}
response = UrlFetch().post(url, data=encoded_data, headers=headers)
plays.append(('get/response', response.status_code))
except Exception, e:
plays.append(('get/response', 500, str(e)))
def delete_scenario(scenario_name_key, force):
log.debug(u'delete_scenario: {0}'.format(scenario_name_key))
host, scenario_name = scenario_name_key.split(':')
cache = Cache(host)
if not force:
active_sessions = cache.get_active_sessions(scenario_name,
local=False)
if active_sessions:
raise exception_response(400,
title='Sessons in playback/record, can not delete. Found th'
'e following active sessions: {0} for scenario: {1}'.format(
active_sessions, scenario_name))
scenario_db.remove_all(scenario_name_key)
cache.delete_caches(scenario_name)
def post(self, scenario_name):
"""
Gets response for specified "response contains" data. Equivalent to:
http://stubo-app.readthedocs.org/en/latest/pages/api.html#get-response
Example:
Insert stub via PUT call:
{
"request": {
"method": "POST",
"bodyPatterns": [
{ "contains": ["<status>IS_OK</status>"] }
]
},
"response": {
"status": 200,
"body": "<response>YES</response>"
}
}
after having your stub inserted just use POST calls:
header:
session: session_test
body:
data: <status>IS_OK</status>
Example response:
<response>YES</response>
:param scenario_name: string - should be used to speed up stub search
:return: :raise exception_response:
"""
session_name = self.request.headers.get('session', None)
if session_name is None:
session_name = self.request.headers.get('Stubo-Request-Session', None)
if not session_name:
raise exception_response(400, title="Session name not found in headers.")
request = self.request
self.track.function = 'get/response'
log.debug('Found session: {0}, for route: {1}'.format(session_name,
request.path))
return get_response(self, session_name)
def put_stub(handler, session_name, delay_policy, stateful, priority,
recorded=None, module_name=None, recorded_module_system_date=None):
log.debug('put_stub request: {0}'.format(handler.request))
request = handler.request
stubo_request = StuboRequest(request)
session_name = session_name.partition(',')[0]
cache = Cache(get_hostname(request))
scenario_key = cache.find_scenario_key(session_name)
trace = TrackTrace(handler.track, 'put_stub')
url_args = handler.track.request_params
err_msg = 'put/stub body format error - {0}, for session: {1}'
try:
stub = parse_stub(stubo_request.body_unicode, scenario_key, url_args)
except Exception, e:
raise exception_response(400, title=err_msg.format(e.message,
session_name))
def jump_bookmark(handler, name, sessions, index=None):
request = handler.request
cache = Cache(get_hostname(request))
response = dict(version=version, data = {})
scenario_key = cache.find_scenario_key(host, sessions[0])
scenario_name = scenario_key.partition(':')[-1]
if not all(cache.find_scenario_key(host, x) == scenario_key for x in sessions):
raise exception_response(400,
title="All sessions must belong to the same scenario")
index_state = cache.get_saved_request_index_data(scenario_name, name)
check_bookmark(host, name, index_state)
results = []
for session in sessions:
for k, v in index_state.iteritems():
v = v if index is None else index
session_key = '{0}:{1}'.format(session, k)
result = set_request_index_item(scenario_name, session_key, v)
results.append((k, v, result))
response['data']['results'] = results
return response
def command_handler_form_request(handler):
cmds = handler.get_argument('cmds', None)
cmd_file_url = handler.get_argument('cmdfile', None)
if not (cmds or cmd_file_url):
# TODO: use form validation instead
raise exception_response(400,
title="'cmds' or 'cmdFile' parameter not supplied.")
log.debug(u'command_handler_form_request: cmd_file={0},cmds={1}'.format(
cmd_file_url, cmds))
if cmd_file_url:
request = DummyModel(protocol=handler.request.protocol,
host=handler.request.host,
arguments=handler.request.arguments)
response = run_command_file(cmd_file_url, request,
handler.settings['static_path'])
elif cmds:
response = run_commands(handler, cmds)
links = dict((k,v) for k,v in response['data'].get('export_links', []))
return handler.render_string("commands.html", page_title='Commands',
executed=response['data'].get('executed_commands'),
export_links=links)
def get_response_request(handler):
session_name = handler.get_argument('session', None)
request = handler.request
def get_session_from_headers():
session_name = request.headers.get('Stubo-Request-Session')
if session_name:
return session_name
# NOTE: legacy support for session_name = stb_scenario + '_' + stb_session
session = [(k, v) for k, v in request.headers.iteritems() \
if 'stb_session' in k or 'Stb_session' in k]
scenario = [(k, v) for k, v in request.headers.iteritems() \
if 'stb_scenario' in k or 'Stb_scenario' in k]
if not session:
raise exception_response(400,
title="session not supplied in headers.")
if not scenario:
raise exception_response(400,
title="scenario parameter not supplied in headers.")
_, session_name = session[0]
scenario_key, scenario_name = scenario[0]
session_name = '{0}_{1}'.format(scenario_name, session_name)
# put into standard key for tracker display
handler.track.request_headers['Stubo-Request-Session'] = session_name
return session_name
if not session_name:
session_name = get_session_from_headers()
if not session_name:
raise exception_response(400, title="session not found in headers.")
handler.track.function = 'get/response'
log.debug('Found session: {0}, for route: {1}'.format(session_name,
request.path))
return get_response(handler, session_name)
try:
stub = parse_stub(stubo_request.body_unicode, scenario_key, url_args)
except Exception, e:
raise exception_response(400,
title=err_msg.format(e.message, session_name))
log.debug('stub: {0}'.format(stub))
if delay_policy:
stub.set_delay_policy(delay_policy)
stub.set_priority(priority)
session = cache.get_session(scenario_key.partition(':')[-1],
session_name,
local=False)
if not session:
raise exception_response(
400, title='session not found - {0}'.format(session_name))
stub.set_recorded(recorded or today_str('%Y-%m-%d'))
if module_name:
stub.set_module({
'name':
module_name,
# TODO: is module['system_date'] used?
'system_date':
today_str('%Y-%m-%d'),
'recorded_system_date':
recorded_module_system_date or today_str('%Y-%m-%d')
})
trace.info('module used', stub.module())
source_stub = copy.deepcopy(stub)
stub, _ = transform(stub,
stubo_request,
def export_playback(self, host, export_payload, files, session,
playback_session):
playback_payload = export_payload['playback']
scenario_name = playback_payload['scenario']
scenario = u'{0}:{1}'.format(host, scenario_name)
runnable_info = dict()
tracker = Tracker()
last_used = tracker.session_last_used(scenario, playback_session,
'playback')
if not last_used:
raise exception_response(400,
title="Unable to find playback session")
runnable_info['last_used'] = dict(remote_ip=last_used['remote_ip'],
start_time=str(last_used['start_time']))
playback = tracker.get_last_playback(scenario_name, playback_session,
last_used['start_time'])
playback = list(playback)
if not playback:
raise exception_response(400,
title="Unable to find a playback for scenario='{0}', "
"playback_session='{1}'".format(scenario_name, playback_session))
number_of_requests = len(playback)
runnable_info['number_of_playback_requests'] = number_of_requests
for nrequest in range(number_of_requests):
track = playback[nrequest]
request_text = track.get('request_text')
if not request_text:
raise exception_response(400, title='Unable to obtain playback details, was full tracking enabled?')
stubo_request = StuboRequest(DummyModel(headers=track.get('request_headers'),
body=request_text))
variables = track.get('request_params')
variables.pop('session', None)
variables.pop('scenario', None)
request_payload = dict(body=stubo_request.body,
method=stubo_request.method,
host=stubo_request.host,
uri=stubo_request.uri,
path=stubo_request.path,
query=stubo_request.query,
headers=stubo_request.headers)
request_file_name = '{0}_{1}.request'.format(session, nrequest)
files.append((request_file_name, json.dumps(request_payload, indent=3)))
# export a response for comparison
stubo_response_text = track['stubo_response']
if not isinstance(stubo_response_text, basestring):
stubo_response_text = unicode(stubo_response_text)
response_payload = dict(status=track.get('return_code'),
body=stubo_response_text,
headers=track.get('response_headers'))
stubo_response_file_name = '{0}_{1}.stubo_response'.format(session,
nrequest)
playback_payload['requests'].append(dict(
file=request_file_name,
vars=variables,
response=stubo_response_file_name))
files.append((stubo_response_file_name, json.dumps(response_payload,
indent=3)))
return runnable_info
def begin_session(handler,
scenario_name,
session_name,
mode,
system_date=None,
warm_cache=False):
"""
Begins session for given scenario
:param handler: request handler class
:param scenario_name: scenario name
:param session_name: session name
:param mode: mode - record, playback
:param system_date:
:param warm_cache:
:return: :raise exception_response:
"""
log.debug('begin_session')
response = {'version': version}
scenario_manager = Scenario()
# cache = Cache(get_hostname(handler.request))
# checking whether full name (with hostname) was passed, if not - getting full name
# scenario_name_key = "localhost:scenario_1"
if ":" not in scenario_name:
cache = Cache(get_hostname(handler.request))
scenario_name_key = cache.scenario_key_name(scenario_name)
else:
# setting scenario full name
scenario_name_key = scenario_name
# removing hostname from scenario name
slices = scenario_name.split(":")
scenario_name = slices[1]
cache = Cache(slices[0])
# get scenario document
scenario_doc = scenario_manager.get(scenario_name_key)
if not scenario_doc:
raise exception_response(
404,
title='Scenario not found - {0}. To begin a'
' session - create a scenario.'.format(scenario_name_key))
cache.assert_valid_session(scenario_name, session_name)
if mode == 'record':
log.debug('begin_session, mode=record')
# check if there are any existing stubs in this scenario
if scenario_manager.stub_count(scenario_name_key) > 0:
err = exception_response(
400,
title='Scenario ({0}) has existing stubs, delete them before '
'recording or create another scenario!'.format(
scenario_name_key))
raise err
scenario_id = scenario_doc['_id']
log.debug('new scenario: {0}'.format(scenario_id))
session_payload = {
'status': 'record',
'scenario': scenario_name_key,
'scenario_id': str(scenario_id),
'session': str(session_name)
}
cache.set_session(scenario_name, session_name, session_payload)
log.debug('new redis session: {0}:{1}'.format(scenario_name_key,
session_name))
response["data"] = {
'message': 'Record mode initiated....',
}
response["data"].update(session_payload)
cache.set_session_map(scenario_name, session_name)
log.debug('finish record')
elif mode == 'playback':
recordings = cache.get_sessions_status(scenario_name,
status='record',
local=False)
if recordings:
raise exception_response(400,
title='Scenario recordings taking '
'place - {0}. Found the '
'following record sessions: {1}'.format(
scenario_name_key, recordings))
cache.create_session_cache(scenario_name, session_name, system_date)
if warm_cache:
# iterate over stubs and call get/response for each stub matchers
# to build the request & request_index cache
# reset request_index to 0
log.debug("warm cache for session '{0}'".format(session_name))
scenario_manager = Scenario()
for payload in scenario_manager.get_stubs(scenario_name_key):
stub = Stub(payload['stub'], scenario_name_key)
mock_request = " ".join(stub.contains_matchers())
handler.request.body = mock_request
get_response(handler, session_name)
cache.reset_request_index(scenario_name)
response["data"] = {"message": "Playback mode initiated...."}
response["data"].update({
"status": "playback",
"scenario": scenario_name_key,
"session": str(session_name)
})
else:
raise exception_response(400,
title='Mode of playback or record required')
return response
def test_client_error(self):
from stubo.exceptions import exception_response, HTTPClientError
e = exception_response(404)
self.assertTrue(isinstance(e, HTTPClientError))
def match(request, session, trace, system_date, url_args, hooks,
module_system_date=None):
"""Returns the stats of a request match process
:param request: source stubo request
:param session: cached session payload associated with this request
:param module_system_date: optional system date of an external module
"""
request_text = request.request_body()
scenario_key = session['scenario']
session_name = session['session']
log.debug(u'match: request_text={0}'.format(request_text))
trace.info('system_date={0}, module_system_date={1}'.format(
system_date, module_system_date))
stats = []
if 'stubs' not in session or not len(session['stubs']):
if session.get('status') != 'playback':
raise exception_response(400,
title="session {0} not in playback mode for scenario "
"{1}".format(session_name, scenario_key))
raise exception_response(500,
title="no stubs found in session {0} for {1}, status={2}".format(
session_name, scenario_key, session.get('status')))
stub_count = len(session['stubs'])
trace.info(u'matching against {0} stubs'.format(stub_count))
for stub_number in range(stub_count):
trace.info('stub ({0})'.format(stub_number))
stub = StubCache(session['stubs'][stub_number], scenario_key,
session_name)
source_stub = copy.deepcopy(stub)
request_copy = copy.deepcopy(request)
stub, request_copy = transform(stub,
request_copy,
module_system_date=module_system_date,
system_date=system_date,
function='get/response',
cache=session.get('ext_cache'),
hooks=hooks,
stage='matcher',
trace=trace,
url_args=url_args)
trace.info('finished transformation')
if source_stub != stub:
trace.diff('stub ({0}) was transformed'.format(stub_number),
source_stub.payload, stub.payload)
trace.info('stub ({0}) was transformed into'.format(stub_number),
stub.payload)
if request_copy != request:
trace.diff('request was transformed', request_copy.request_body(),
request.request_body())
trace.info('request was transformed into', request_copy.request_body())
matcher = StubMatcher(trace)
hits = matcher.match(request_copy, stub)
stats.append(((hits, stub_number), stub))
if hits == stub.number_of_matchers():
# without most_matchers_win support
trace.info(u"stub '{0}' matched".format(stub_number))
break
# sort by hits (desc), stub_number (asc)
# to match against 1. greatest hits or 2. first stub to get a hit
return sorted(stats, key=lambda k: (k[0][0], -k[0][1]), reverse=True)
def run_command(self, url):
data = ''
log.debug('url.path={0}'.format(url.path))
cmd_path = url.geturl()
parent_path = self.location(os.path.dirname(
self.cmd_file_url))[0] + '/'
if url.path == 'import/bookmarks':
loc = parse_qs(url.query).get('location')
if loc:
loc = loc[0]
else:
raise exception_response(
400,
title="missing 'location' param executing import/bookmarks"
)
target_url = self.location(urljoin(parent_path, loc))[0]
log.debug('run_command: {0}'.format(target_url))
import_cmd_url = self.location(
'stubo/api/import/bookmarks?location={0}'.format(
target_url))[0]
response, _ = UrlFetch().get(import_cmd_url)
return
elif url.path == 'put/stub':
# Note: delay policy is an optional param, the text matchers &
# response start after the first ","
query, _, matchers_response = url.query.partition(',')
query_params = parse_qs(query)
delist_arguments(query_params)
if 'session' not in query_params:
raise exception_response(400,
title="Missing 'session' param in"
" query: {0}".format(url.query))
matchers_response = u''.join(matchers_response.split()).strip()
matchers_response = matchers_response.split(',')
response_fname = matchers_response[-1].strip()
matchers = matchers_response[:-1]
request_matchers = []
for matcher in matchers:
if matcher[:4] == 'url=':
matcher_data_url = matcher[4:]
matcher_text, _ = UrlFetch().get(matcher_data_url)
elif matcher[:5] == 'text=':
matcher_text = matcher[5:]
else:
matcher_data_url = urljoin(parent_path, matcher)
matcher_text, _ = UrlFetch().get(matcher_data_url)
request_matchers.append(matcher_text)
if response_fname[:4] == 'url=':
response_data_url = response_fname[4:]
response_text, _ = UrlFetch().get(response_data_url)
elif response_fname[:5] == 'text=':
response_text = response_fname[5:]
else:
response_data_url = urljoin(parent_path, response_fname)
response_text, _ = UrlFetch().get(response_data_url)
if not response_text:
raise exception_response(
400, title="put/stub response text can not be empty.")
stub_payload = create(request_matchers, response_text)
cmd_path = url.path + '?{0}'.format(urlencode(query_params))
url = self.location(urljoin(api_base, cmd_path))[0]
log.debug(u'run_command: {0}'.format(url))
UrlFetch().post(url, data=None, json=stub_payload)
return
elif url.path == 'get/response':
# get/response?session=foo_1, my.request
query, _, request_fname = url.query.partition(',')
query_params = parse_qs(query)
if 'session' not in query_params:
raise exception_response(400,
title="Missing 'session' param in"
" query: {0}".format(url.query))
request_fname = request_fname.strip()
if request_fname[:4] == 'url=':
request_data_url = request_fname[4:]
request_text, _ = UrlFetch().get(request_data_url)
elif request_fname[:5] == 'text=':
request_text = request_fname[5:]
else:
request_data_url = urljoin(parent_path, request_fname)
request_text, _ = UrlFetch().get(request_data_url)
data = request_text
cmd_path = url.path + '?{0}'.format(query)
elif url.path == 'put/delay_policy':
url = self.location(urljoin(api_base, cmd_path))[0]
log.debug('run_command: {0}, data={1}'.format(url, data))
response, _ = UrlFetch().get(url)
return
url = self.location(urljoin(api_base, cmd_path))[0]
log.debug(u'run_command: {0}'.format(url))
encoded_data = data.encode('utf-8')
UrlFetch().post(url, data=encoded_data)
def export_stubs_to_commands_format(handler, scenario_name):
"""
Exports scenario to .commands file format.
:param handler:
:param scenario_name: <string> Scenario name
:return: :raise exception_response:
"""
cache = Cache(get_hostname(handler.request))
scenario_name_key = cache.scenario_key_name(scenario_name)
# use user arg or epoch time
session_id = handler.get_argument('session_id', int(time.time()))
session = u'{0}_{1}'.format(scenario_name, session_id)
cmds = [
'delete/stubs?scenario={0}'.format(scenario_name),
'begin/session?scenario={0}&session={1}&mode=record'.format(
scenario_name, session)
]
files = []
scenario = Scenario()
# get scenario pre stubs for specified scenario
stubs = list(scenario.get_pre_stubs(scenario_name_key))
if stubs:
for i in range(len(stubs)):
entry = stubs[i]
stub = Stub(entry['stub'], scenario_name_key)
# if stub is rest - matcher may be None, checking that
if stub.contains_matchers() is None:
cmds.append(
'# Stub skipped since no matchers were found. Consider using .yaml format for additional '
'capabilities')
# skipping to next stub, this stub is not compatible with .commands format
continue
matchers = [('{0}_{1}_{2}.textMatcher'.format(session, i, x),
stub.contains_matchers()[x])
for x in range(len(stub.contains_matchers()))]
matchers_str = ",".join(x[0] for x in matchers)
url_args = stub.args()
url_args['session'] = session
module_info = stub.module()
if module_info:
# Note: not including put/module in the export, modules are shared
# by multiple scenarios.
url_args['ext_module'] = module_info['name']
url_args['stub_created_date'] = stub.recorded()
url_args['stubbedSystemDate'] = module_info.get(
'recorded_system_date')
url_args['system_date'] = module_info.get('system_date')
url_args = urlencode(url_args)
responses = stub.response_body()
assert (len(responses) == 1)
response = responses[0]
response = ('{0}_{1}.response'.format(session, i), response)
cmds.append('put/stub?{0},{1},{2}'.format(url_args, matchers_str,
response[0]))
files.append(response)
files.extend(matchers)
else:
cmds.append(
'put/stub?session={0},text=a_dummy_matcher,text=a_dummy_response'.
format(session))
cmds.append('end/session?session={0}'.format(session))
runnable = asbool(handler.get_argument('runnable', False))
runnable_info = dict()
if runnable:
playback_session = handler.get_argument('playback_session', None)
if not playback_session:
raise exception_response(
400,
title="'playback_session' argument required with 'runnable")
runnable_info['playback_session'] = playback_session
tracker = Tracker()
last_used = tracker.session_last_used(scenario_name_key,
playback_session, 'playback')
if not last_used:
raise exception_response(400,
title="Unable to find playback session")
runnable_info['last_used'] = dict(remote_ip=last_used['remote_ip'],
start_time=str(
last_used['start_time']))
playback = tracker.get_last_playback(scenario_name, playback_session,
last_used['start_time'])
playback = list(playback)
if not playback:
raise exception_response(
400,
title=
"Unable to find a playback for scenario='{0}', playback_session='{1}'"
.format(scenario_name, playback_session))
cmds.append(
'begin/session?scenario={0}&session={1}&mode=playback'.format(
scenario_name, session))
number_of_requests = len(playback)
runnable_info['number_of_playback_requests'] = number_of_requests
for nrequest in range(number_of_requests):
track = playback[nrequest]
request_text = track.get('request_text')
if not request_text:
raise exception_response(
400,
title=
'Unable to obtain playback details, was full tracking enabled?'
)
request_file_name = '{0}_{1}.request'.format(session, nrequest)
files.append((request_file_name, request_text))
stubo_response_text = track['stubo_response']
if not isinstance(stubo_response_text, basestring):
stubo_response_text = unicode(stubo_response_text)
stubo_response_file_name = '{0}_{1}.stubo_response'.format(
session, nrequest)
files.append((stubo_response_file_name, stubo_response_text))
url_args = track['request_params']
url_args['session'] = session
url_args = urlencode(url_args)
cmds.append(u'get/response?{0},{1}'.format(url_args,
request_file_name))
cmds.append('end/session?session={0}'.format(session))
files.append(('{0}.commands'.format(scenario_name), b"\r\n".join(cmds)))
static_dir = handler.settings['static_path']
export_dir = handler.get_argument('export_dir',
scenario_name_key).replace(':', '_')
export_dir_path = os.path.join(static_dir, 'exports', export_dir)
if os.path.exists(export_dir_path):
shutil.rmtree(export_dir_path)
os.makedirs(export_dir_path)
archive_name = os.path.join(export_dir_path, scenario_name)
zout = zipfile.ZipFile(archive_name + '.zip', "w")
tar = tarfile.open(archive_name + ".tar.gz", "w:gz")
for finfo in files:
fname, contents = finfo
file_path = os.path.join(export_dir_path, fname)
with codecs.open(file_path, mode='wb', encoding='utf-8') as f:
f.write(contents)
f.close()
tar.add(file_path, fname)
zout.write(file_path, fname)
tar.close()
zout.close()
shutil.copy(archive_name + '.zip', archive_name + '.jar')
files.extend([(scenario_name + '.zip', ), (scenario_name + '.tar.gz', ),
(scenario_name + '.jar', )])
# getting links
links = get_export_links(handler, scenario_name_key, files)
return links
def create_session_cache(self, scenario_name, session_name,
system_date=None):
scenario_key = self.scenario_key_name(scenario_name)
log.debug("create_session_cache: scenario_key={0}, session_name={1}".format(
scenario_key, session_name))
session = self.get(scenario_key, session_name)
if not session:
# must be using a different session name for playback than record
session = {
'session' : session_name,
'scenario' : scenario_key
}
# add to sessions map
self.set_raw('{0}:sessions'.format(self.host), session_name, scenario_name)
session['status'] = 'playback'
session['system_date'] = system_date or datetime.date.today().strftime(
'%Y-%m-%d')
session['last_used'] = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
cache_info = []
# copy mongo scenario stubs to redis cache
scenario_col = Scenario()
stubs_cursor = scenario_col.get_stubs(scenario_key)
stubs = list(stubs_cursor)
if not stubs:
raise exception_response(500,
title="found no stubs in mongo for {0}".format(scenario_key))
from stubo.ext.module import Module
for scenario_stub in stubs:
stub = Stub(scenario_stub['stub'], scenario_stub['scenario'])
if stub.module():
module_name = stub.module()['name']
# tag this stub with the latest version of the module
module = Module(self.host)
version = module.latest_version(module_name)
if not version:
raise exception_response(500,
title="module '{0}' not found in cache".format(
module.key(module_name)))
stub.module()['version'] = version
response_pairs = [(compute_hash(x), x) for x in stub.response_body()]
# cache each response id -> text
for response in response_pairs:
response_id, response_text = response
self.set_response_text(scenario_name, session_name, response_id,
response_text)
# replace response text with response hash ids for session cache
# stub.pop('response', None)
stub.response().pop('body', None)
stub.response()['ids'] = [x[0] for x in response_pairs]
delay_policy_name = stub.delay_policy()
if delay_policy_name:
# Note: the delay policy is not really cached with the session.
# The get/response call will just use the name to get the latest
# delay value from the 'delay_policy' key in redis.
delay_policy_key = '{0}:delay_policy'.format(self.host)
delay_policy = self.get(delay_policy_key, delay_policy_name)
if not delay_policy:
log.warn('unable to find delay_policy: {0}'.format(
delay_policy_name))
stub.set_delay_policy(delay_policy)
#_id = ObjectId(scenario_stub['_id'])
#stub['recorded'] = str(_id.generation_time.date())
cache_info.append(stub.payload)
session['stubs'] = cache_info
#log.debug('stubs: {0}'.format(session['stubs']))
self.set(scenario_key, session_name, session)
log.debug('created session cache: {0}:{1}'.format(session['scenario'],
session['session']))
def check_bookmark(host, bookmark_name, bookmark):
if not bookmark:
raise exception_response(
400,
title='No bookmarks have been saved {0}'.format(
self.get_saved_request_index_key()))
def test_client_error_with_title(self):
from stubo.exceptions import exception_response, HTTPClientError
e = exception_response(404, title='help!')
self.assertTrue(isinstance(e, HTTPClientError))
self.assertEqual(e.title, 'help!')
def test_server_error(self):
from stubo.exceptions import exception_response, HTTPServerError
e = exception_response(500)
self.assertTrue(isinstance(e, HTTPServerError))
def test_server_error_with_title_arg(self):
from stubo.exceptions import exception_response, HTTPServerError
e = exception_response(500, title='help!')
self.assertTrue(isinstance(e, HTTPServerError))
self.assertEqual(e.title, 'help!')
def get_response(handler, session_name):
request = handler.request
stubo_request = StuboRequest(request)
cache = Cache(get_hostname(request))
if cache.blacklisted():
raise exception_response(
400,
title="Sorry the host URL '{0}' has been "
"blacklisted. Please contact Stub-O-Matic support.".format(
cache.host))
scenario_key = cache.find_scenario_key(session_name)
scenario_name = scenario_key.partition(':')[-1]
handler.track.scenario = scenario_name
request_id = stubo_request.id()
module_system_date = handler.get_argument('system_date', None)
url_args = handler.track.request_params
if not module_system_date:
# LEGACY
module_system_date = handler.get_argument('stubbedSystemDate', None)
trace_matcher = TrackTrace(handler.track, 'matcher')
user_cache = handler.settings['ext_cache']
# check cached requests
cached_request = cache.get_request(scenario_name, session_name, request_id)
if cached_request:
response_ids, delay_policy_name, recorded, system_date, module_info, request_index_key = cached_request
else:
retry_count = 5 if handler.settings.get('is_cluster', False) else 1
session, retries = cache.get_session_with_delay(
scenario_name,
session_name,
retry_count=retry_count,
retry_interval=1)
if retries > 0:
log.warn("replication was slow for session: {0} {1}, it took {2} "\
"secs!".format(scenario_key, session_name, retries+1))
if session['status'] != 'playback':
raise exception_response(
500,
title='cache status != playback. session={0}'.format(session))
system_date = session['system_date']
if not system_date:
raise exception_response(
500,
title="slave session {0} not available for scenario {1}".
format(session_name, scenario_key))
session['ext_cache'] = user_cache
result = match(stubo_request,
session,
trace_matcher,
as_date(system_date),
url_args=url_args,
hooks=handler.settings['hooks'],
module_system_date=module_system_date)
if not result[0]:
raise exception_response(400,
title='E017:No matching response found')
_, stub_number, stub = result
response_ids = stub.response_ids()
delay_policy_name = stub.delay_policy_name()
recorded = stub.recorded()
module_info = stub.module()
request_index_key = add_request(
session, request_id, stub, system_date, stub_number,
handler.settings['request_cache_limit'])
if not stub.response_body():
_response = stub.get_response_from_cache(request_index_key)
stub.set_response_body(_response['body'])
if delay_policy_name:
stub.load_delay_from_cache(delay_policy_name)
if cached_request:
stub = StubCache({}, scenario_key, session_name)
stub.load_from_cache(response_ids, delay_policy_name, recorded,
system_date, module_info, request_index_key)
trace_response = TrackTrace(handler.track, 'response')
if module_info:
trace_response.info('module used', str(module_info))
response_text = stub.response_body()
if not response_text:
raise exception_response(
500,
title='Unable to find response in '
'cache using session: {0}:{1}, response_ids: {2}'.format(
scenario_key, session_name, response_ids))
# get latest delay policy
delay_policy = stub.delay_policy()
if delay_policy:
delay = Delay.parse_args(delay_policy)
if delay:
delay = delay.calculate()
msg = 'apply delay: {0} => {1}'.format(delay_policy, delay)
log.debug(msg)
handler.track['delay'] = delay
trace_response.info(msg)
trace_response.info('found response')
module_system_date = as_date(module_system_date) if module_system_date \
else module_system_date
stub, _ = transform(stub,
stubo_request,
module_system_date=module_system_date,
system_date=as_date(system_date),
function='get/response',
cache=user_cache,
hooks=handler.settings['hooks'],
stage='response',
trace=trace_response,
url_args=url_args)
transfomed_response_text = stub.response_body()[0]
# Note transformed_response_text can be encoded in utf8
if response_text[0] != transfomed_response_text:
trace_response.diff('response:transformed',
dict(response=response_text[0]),
dict(response=transfomed_response_text))
if stub.response_status() != 200:
handler.set_status(stub.response_status())
if stub.response_headers():
for k, v in stub.response_headers().iteritems():
handler.set_header(k, v)
return transfomed_response_text
def begin_session(handler,
scenario_name,
session_name,
mode,
system_date=None,
warm_cache=False):
log.debug('begin_session')
response = {'version': version}
scenario_col = Scenario()
cache = Cache(get_hostname(handler.request))
if cache.blacklisted():
raise exception_response(
400,
title="Sorry the host URL '{0}' has been "
"blacklisted. Please contact Stub-O-Matic support.".format(
cache.host))
scenario_name_key = cache.scenario_key_name(scenario_name)
scenario = scenario_col.get(scenario_name_key)
cache.assert_valid_session(scenario_name, session_name)
if mode == 'record':
log.debug('begin_session, mode=record')
# precond: delete/stubs?scenario={scenario_name}
if scenario:
err = exception_response(
400,
title='Duplicate scenario found - {0}'.format(
scenario_name_key))
raise err
if scenario_col.stub_count(scenario_name_key) != 0:
raise exception_response(
500,
title='stub_count !=0 for scenario: {0}'.format(
scenario_name_key))
scenario_id = scenario_col.insert(name=scenario_name_key)
log.debug('new scenario: {0}'.format(scenario_id))
session_payload = {
'status': 'record',
'scenario': scenario_name_key,
'scenario_id': str(scenario_id),
'session': str(session_name)
}
cache.set_session(scenario_name, session_name, session_payload)
log.debug('new redis session: {0}:{1}'.format(scenario_name_key,
session_name))
response["data"] = {
'message': 'Record mode initiated....',
}
response["data"].update(session_payload)
cache.set_session_map(scenario_name, session_name)
log.debug('finish record')
elif mode == 'playback':
if not scenario:
raise exception_response(
400,
title='Scenario not found - {0}'.format(scenario_name_key))
recordings = cache.get_sessions_status(scenario_name,
status=('record'),
local=False)
if recordings:
raise exception_response(400, title='Scenario recordings taking ' \
'place - {0}. Found the following record sessions: {1}'.format(
scenario_name_key, recordings))
cache.create_session_cache(scenario_name, session_name, system_date)
if warm_cache:
# iterate over stubs and call get/response for each stub matchers
# to build the request & request_index cache
# reset request_index to 0
log.debug("warm cache for session '{0}'".format(session_name))
scenario_col = Scenario()
for payload in scenario_col.get_stubs(scenario_name_key):
stub = Stub(payload['stub'], scenario_name_key)
mock_request = " ".join(stub.contains_matchers())
handler.request.body = mock_request
get_response(handler, session_name)
cache.reset_request_index(scenario_name)
response["data"] = {"message": "Playback mode initiated...."}
response["data"].update({
"status": "playback",
"scenario": scenario_name_key,
"session": str(session_name)
})
else:
raise exception_response(400,
title='Mode of playback or record required')
return response
def run_command(self, url, priority):
data = ''
log.debug('url.path={0}'.format(url.path))
cmd_path = url.geturl()
parent_path = self.location(os.path.dirname(
self.cmd_file_url))[0] + '/'
if url.path == 'import/bookmarks':
loc = parse_qs(url.query).get('location')
if loc:
loc = loc[0]
else:
raise exception_response(
400,
title="missing 'location' param executing import/bookmarks"
)
target_url = self.location(urljoin(parent_path, loc))[0]
log.debug('run_command: {0}'.format(target_url))
import_cmd_url = self.location(
'stubo/api/import/bookmarks?location={0}'.format(
target_url))[0]
response, _, status_code = UrlFetch().get(import_cmd_url)
return status_code
elif url.path == 'put/stub':
# Note: delay policy is an optional param, the text matchers &
# response start after the first ","
query, _, matchers_response = url.query.partition(',')
query_params = parse_qs(query)
delist_arguments(query_params)
if 'session' not in query_params:
raise exception_response(400,
title="Missing 'session' param in"
" query: {0}".format(url.query))
if 'priority' not in query_params:
query_params['priority'] = priority
matchers_response = u''.join(matchers_response.split()).strip()
matchers_response = matchers_response.split(',')
response_fname = matchers_response[-1].strip()
matchers = matchers_response[:-1]
request_matchers = []
for matcher in matchers:
if matcher[:4] == 'url=':
matcher_data_url = matcher[4:]
matcher_text, _, _ = UrlFetch().get(matcher_data_url)
elif matcher[:5] == 'text=':
matcher_text = matcher[5:]
else:
matcher_data_url = urljoin(parent_path, matcher)
matcher_text, _, _ = UrlFetch().get(matcher_data_url)
request_matchers.append(matcher_text)
if response_fname[:4] == 'url=':
response_data_url = response_fname[4:]
response_text, _, _ = UrlFetch().get(response_data_url)
elif response_fname[:5] == 'text=':
response_text = response_fname[5:]
else:
response_data_url = urljoin(parent_path, response_fname)
response_text, hdrs, _ = UrlFetch().get(response_data_url)
if 'application/json' in hdrs["Content-Type"]:
try:
response_text = json.dumps(response_text)
except Exception:
pass
if not response_text:
raise exception_response(
400, title="put/stub response text can not be empty.")
stub_payload = create(request_matchers, response_text)
cmd_path = url.path + '?{0}'.format(urlencode(query_params))
url = self.get_url(cmd_path)
log.debug(u'run_command: {0}'.format(url))
response = UrlFetch().post(url, data=None, json=stub_payload)
return response.status_code
elif url.path == 'get/response':
# get/response?session=foo_1, my.request
query, _, request_fname = url.query.partition(',')
query_params = parse_qs(query)
if 'session' not in query_params:
raise exception_response(400,
title="Missing 'session' param in"
" query: {0}".format(url.query))
request_fname, _, header_args = request_fname.partition(',')
request_fname = request_fname.strip()
if request_fname[:4] == 'url=':
request_data_url = request_fname[4:]
request_text, _, _ = UrlFetch().get(request_data_url)
elif request_fname[:5] == 'text=':
request_text = request_fname[5:]
else:
request_data_url = urljoin(parent_path, request_fname)
request_text, _, _ = UrlFetch().get(request_data_url)
data = request_text
cmd_path = url.path + '?{0}'.format(query)
url = self.get_url(cmd_path)
log.debug(u'run_command: {0}'.format(url))
if isinstance(data, dict):
# payload is json
encoded_data = json.dumps(data)
else:
encoded_data = data.encode('utf-8')
headers = {'Stubo-Request-Method': 'POST'}
if header_args:
headers.update(
dict(x.split('=') for x in header_args.split(',')))
response = UrlFetch().post(url, data=encoded_data, headers=headers)
return response.status_code
elif url.path == 'put/delay_policy':
url = self.get_url(cmd_path)
log.debug('run_command: {0}, data={1}'.format(url, data))
_, _, status_code = UrlFetch().get(url)
return status_code
url = self.get_url(cmd_path)
log.debug(u'run_command: {0}'.format(url))
encoded_data = data.encode('utf-8')
response = UrlFetch().post(url, data=encoded_data)
return response.status_code
def get_arg(handler, arg):
value = handler.get_argument(arg, None)
if not value:
raise exception_response(
400, title="'{0}' parameter not supplied.".format(arg))
return value
