def list_available_modules(hostname):
"""
Gets all available modules (for given host) and returns a list with them.
:param hostname:
:return:
"""
modules_list = []
module = Module(hostname)
# getting keys
keys = get_keys('{0}:modules:*'.format(module.host()))
names = [x.rpartition(':')[-1] for x in keys]
for name in names:
loaded_sys_versions = [
x for x in sys.modules.keys() if '{0}_v'.format(name) in x
]
lastest_code_version = module.latest_version(name)
source_code = Module(hostname).get_source(name)
obj = {
'name': name,
'latest_code_version': lastest_code_version,
'loaded_sys_versions': loaded_sys_versions,
'source_raw': source_code,
'href': '/api/v2/modules/objects/%s' % name
}
modules_list.append(obj)
return {'version': version, 'data': modules_list}
def put_module(handler, names):
module = Module(handler.track.host)
added = []
result = dict(version=version)
for name in names:
uri, module_name = UriLocation(handler.request)(name)
log.info('uri={0}, module_name={1}'.format(uri, module_name))
response, _ = UrlFetch().get(uri)
module_name = module_name[:-3]
last_version = module.latest_version(module_name)
module_version_name = module.sys_module_name(module_name,
last_version + 1)
if last_version and response == module.get_source(
module_name, last_version):
msg = 'Module source has not changed for {0}'.format(
module_version_name)
result['data'] = dict(message=msg)
try:
code, mod = module.add_sys_module(module_version_name, response)
log.debug('{0}, {1}'.format(mod, code))
except Exception, e:
_, t, v, tbinfo = compact_traceback()
msg = u'error={0}'.format(e)
stack_trace = u'traceback is: ({0}: {1} {2})'.format(t, v, tbinfo)
log.error(msg)
raise exception_response(
400,
title='Unable to compile {0}:{1}, {2}'.format(
module.host(), module_version_name, msg),
traceback=stack_trace)
module.add(module_name, response)
added.append(module_version_name)
def put_module(handler, names):
module = Module(handler.track.host)
added = []
result = dict(version=version)
for name in names:
uri, module_name = UriLocation(handler.request)(name)
log.info('uri={0}, module_name={1}'.format(uri, module_name))
response, _, code = UrlFetch().get(uri)
module_name = module_name[:-3]
last_version = module.latest_version(module_name)
module_version_name = module.sys_module_name(module_name,
last_version + 1)
if last_version and response == module.get_source(
module_name, last_version):
msg = 'Module source has not changed for {0}'.format(
module_version_name)
result['data'] = dict(message=msg)
else:
# code changed, adding new module
try:
code, mod = module.add_sys_module(module_version_name,
response)
log.debug('{0}, {1}'.format(mod, code))
except Exception, e:
msg = 'error={0}'.format(e)
raise exception_response(
400,
title='Unable to compile {0}:{1}, {2}'.format(
module.host(), module_version_name, msg))
module.add(module_name, response)
added.append(module_version_name)
result['data'] = dict(message='added modules: {0}'.format(added))
def put_module(handler, names):
module = Module(handler.track.host)
added = []
result = dict(version=version)
for name in names:
uri, module_name = UriLocation(handler.request)(name)
log.info('uri={0}, module_name={1}'.format(uri, module_name))
response, _, code = UrlFetch().get(uri)
module_name = module_name[:-3]
last_version = module.latest_version(module_name)
module_version_name = module.sys_module_name(module_name,
last_version+1)
if last_version and response == module.get_source(module_name,
last_version):
msg = 'Module source has not changed for {0}'.format(
module_version_name)
result['data'] = dict(message=msg)
try:
code, mod = module.add_sys_module(module_version_name, response)
log.debug('{0}, {1}'.format(mod, code))
except Exception, e:
msg = 'error={0}'.format(e)
raise exception_response(400,
title='Unable to compile {0}:{1}, {2}'.format(module.host(),
module_version_name, msg))
module.add(module_name, response)
added.append(module_version_name)
def list_module(handler, names):
module = Module(get_hostname(handler.request))
info = {}
if not names:
names = [
x.rpartition(':')[-1]
for x in get_keys('{0}:modules:*'.format(module.host()))
]
for name in names:
loaded_sys_versions = [x for x in sys.modules.keys() if \
'{0}_v'.format(name) in x]
lastest_code_version = module.latest_version(name)
info[name] = {
'latest_code_version': lastest_code_version,
'loaded_sys_versions': loaded_sys_versions
}
payload = dict(message='list modules', info=info)
return {'version': version, 'data': payload}
def list_module(handler, names):
module = Module(get_hostname(handler.request))
info = {}
if not names:
names = [x.rpartition(':')[-1] for x in get_keys(
'{0}:modules:*'.format(module.host()))]
for name in names:
loaded_sys_versions = [x for x in sys.modules.keys() if '{0}_v'.format(name) in x]
lastest_code_version = module.latest_version(name)
info[name] = {
'latest_code_version': lastest_code_version,
'loaded_sys_versions': loaded_sys_versions
}
payload = dict(message='list modules', info=info)
return {
'version': version,
'data': payload
}
def tearDown(self):
import sys
test_modules = [x for x in sys.modules.keys() \
if x.startswith('localhost_stubotest')]
for mod in test_modules:
del sys.modules[mod]
from stubo.ext.module import Module
Module('localhost').remove('dummy')
self.q_patch.stop()
def delete_module(request, names):
module = Module(get_hostname(request))
removed = []
for name in names:
loaded_versions = [x for x in sys.modules.keys() if '{0}_v'.format(name) in x]
for loaded in loaded_versions:
module.remove_sys_module(loaded)
if module.remove(name):
removed.append('{0}:{1}'.format(module.host(), name))
return {
'version': version,
'data': {'message': 'delete modules: {0}'.format(names),
'deleted': removed}
}
def setUp(self):
self.q = DummyQueue
self.q_patch = mock.patch('stubo.ext.module.Queue', self.q)
self.q_patch.start()
from stubo.ext.module import Module
Module('localhost').add('dummy', exit_code)
else:
error_message = result
except MissingArgumentError, e:
error_message = "Error: {0}".format(e)
except StuboException, e:
error_message = "Error: {0}".format(e.title)
cmdFile = handler.get_argument('cmdFile', '')
http_req = handler.request
response = dict(
host_scenarios=get_session_status(handler, all_hosts=all_hosts))
cache_loc = handler.get_argument('cache', 'master')
response['delays'] = get_delay_policy(handler, None, cache_loc).get('data')
modules = list_module(handler, None)['data'].get('info')
for name in modules.keys():
source_text = pretty_format_python(Module(cache.host).get_source(name))
modules[name]['code'] = source_text
response['modules'] = modules
response['cmdFile'] = cmdFile
response['message'] = message
response['error_message'] = error_message
response['module_info'] = module_info
response['stubo_version'] = version
response['host'] = cache.host
response['page_name'] = 'Manage'
return response
def get_session_status(handler, all_hosts=True):
scenario = Scenario()
host_scenarios = {}
def create_session_cache(self, scenario_name, session_name,
system_date=None):
scenario_key = self.scenario_key_name(scenario_name)
log.debug("create_session_cache: scenario_key={0}, session_name={1}".format(
scenario_key, session_name))
session = self.get(scenario_key, session_name)
if not session:
# must be using a different session name for playback than record
session = {
'session' : session_name,
'scenario' : scenario_key
}
# add to sessions map
self.set_raw('{0}:sessions'.format(self.host), session_name, scenario_name)
session['status'] = 'playback'
session['system_date'] = system_date or datetime.date.today().strftime(
'%Y-%m-%d')
session['last_used'] = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
cache_info = []
# copy mongo scenario stubs to redis cache
scenario_col = Scenario()
stubs_cursor = scenario_col.get_stubs(scenario_key)
stubs = list(stubs_cursor)
if not stubs:
raise exception_response(500,
title="found no stubs in mongo for {0}".format(scenario_key))
from stubo.ext.module import Module
for scenario_stub in stubs:
stub = Stub(scenario_stub['stub'], scenario_stub['scenario'])
if stub.module():
module_name = stub.module()['name']
# tag this stub with the latest version of the module
module = Module(self.host)
version = module.latest_version(module_name)
if not version:
raise exception_response(500,
title="module '{0}' not found in cache".format(
module.key(module_name)))
stub.module()['version'] = version
response_pairs = [(compute_hash(x), x) for x in stub.response_body()]
# cache each response id -> text
for response in response_pairs:
response_id, response_text = response
self.set_response_text(scenario_name, session_name, response_id,
response_text)
# replace response text with response hash ids for session cache
# stub.pop('response', None)
stub.response().pop('body', None)
stub.response()['ids'] = [x[0] for x in response_pairs]
delay_policy_name = stub.delay_policy()
if delay_policy_name:
# Note: the delay policy is not really cached with the session.
# The get/response call will just use the name to get the latest
# delay value from the 'delay_policy' key in redis.
delay_policy_key = '{0}:delay_policy'.format(self.host)
delay_policy = self.get(delay_policy_key, delay_policy_name)
if not delay_policy:
log.warn('unable to find delay_policy: {0}'.format(
delay_policy_name))
stub.set_delay_policy(delay_policy)
#_id = ObjectId(scenario_stub['_id'])
#stub['recorded'] = str(_id.generation_time.date())
cache_info.append(stub.payload)
session['stubs'] = cache_info
#log.debug('stubs: {0}'.format(session['stubs']))
self.set(scenario_key, session_name, session)
log.debug('created session cache: {0}:{1}'.format(session['scenario'],
session['session']))
def _get_module(self):
from stubo.ext.module import Module
return Module('localhost')
def create_session_cache(self, scenario_name, session_name,
system_date=None):
scenario_key = self.scenario_key_name(scenario_name)
log.debug("create_session_cache: scenario_key={0}, session_name={1}".format(
scenario_key, session_name))
session = self.get(scenario_key, session_name)
if not session:
# must be using a different session name for playback than record
session = {
'session' : session_name,
'scenario' : scenario_key
}
# add to sessions map
self.set_raw('{0}:sessions'.format(self.host), session_name, scenario_name)
session['status'] = 'playback'
session['system_date'] = system_date or datetime.date.today().strftime(
'%Y-%m-%d')
session['last_used'] = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
cache_info = []
# copy mongo scenario stubs to redis cache
scenario_col = Scenario()
stubs_cursor = scenario_col.get_stubs(scenario_key)
stubs = list(stubs_cursor)
if not stubs:
raise exception_response(500,
title="found no stubs in mongo for {0}".format(scenario_key))
from stubo.ext.module import Module
for scenario_stub in stubs:
stub = Stub(scenario_stub['stub'], scenario_stub['scenario'])
if stub.module():
module_name = stub.module()['name']
# tag this stub with the latest version of the module
module = Module(self.host)
version = module.latest_version(module_name)
if not version:
raise exception_response(500,
title="module '{0}' not found in cache".format(
module.key(module_name)))
stub.module()['version'] = version
response_ids = []
response_bodys = stub.response_body()
# cache each response id -> response (text, status) etc
for response_text in response_bodys:
stub.set_response_body(response_text)
response_id = response_hash(response_text, stub)
self.set_response(scenario_name, session_name, response_id,
stub.response())
response_ids.append(response_id)
# replace response text with response hash ids for session cache
stub.response().pop('body', None)
stub.response()['ids'] = response_ids
delay_policy_name = stub.delay_policy()
if delay_policy_name:
# Note: the delay policy is not really cached with the session.
# The get/response call will just use the name to get the latest
# delay value from the 'delay_policy' key in redis.
delay_policy_key = '{0}:delay_policy'.format(self.host)
delay_policy = self.get(delay_policy_key, delay_policy_name)
if not delay_policy:
log.warn('unable to find delay_policy: {0}'.format(
delay_policy_name))
stub.set_delay_policy(delay_policy)
#_id = ObjectId(scenario_stub['_id'])
#stub['recorded'] = str(_id.generation_time.date())
cache_info.append(stub.payload)
session['stubs'] = cache_info
#log.debug('stubs: {0}'.format(session['stubs']))
self.set(scenario_key, session_name, session)
log.debug('created session cache: {0}:{1}'.format(session['scenario'],
session['session']))
return session