def __init__(self, templates_dir="templates", static_dir="static"):
self.routes = {}
self.templates_env = Environment(
loader=FileSystemLoader(os.path.abspath(templates_dir)))
self.exception_handler = None
self.whitenoise = WhiteNoise(self._wsgi_app, root=static_dir)
self.middleware = Middleware(self)
def __init__(self, templates_directory="templates", static_dir="static"):
self.routes = {} #paths are the keys and handlers (functions or classes) are the values
self.templates_environment = Environment(
loader=FileSystemLoader(os.path.abspath(templates_directory))
)
self.exception_handler = None
self.whitenoise = WhiteNoise(self.wsgi_application, root=static_dir) #wrap wsgi application to serve static files
self.middleware = Middleware(self)
def __init__(self, templates_dir="templates", static_dir="static"):
self.templates_dir = templates_dir
self.static_dir = static_dir
# url路由
self.routes = {}
# html文件夹
self.templates_env = Environment(
loader=FileSystemLoader(os.path.abspath(self.templates_dir)))
# css、JavaScript文件夹
self.whitenoise = WhiteNoise(self.wsgi_app, root=static_dir)
# 自定义错误
self.exception_handler = None
# 请求中间件，将api对象传入
self.middleware = Middleware(self)
class middlewareTest(unittest.TestCase):
_middleware = Middleware()
def __validateRegister(self):
self.assertIsNotNone(self. _middleware.__validateRegister('Ridoy','5622#','Mehei','hasan','*****@*****.**'))
def __init__(self, config_path=None, bot_name="vex"):
self.name = bot_name
self._logger = logging.getLogger(__name__)
self.messager = Messager()
# adapters are inputs into the bot. Like a mic or shell input
adapter_manager = pluginmanager.PluginInterface()
adapter_manager.set_entry_points('vexbot.adapters')
plugins = adapter_manager.collect_entry_point_plugins()
plugin_wrapper = PluginWrapper(plugins[0])
pub_address = 'tcp://127.0.0.1:5555'
plugin_wrapper.activate(invoke_kwargs={'--pub_address': pub_address,
'--prompt_name': 'vexbot'})
self.messager.subscribe_to_address(pub_address)
self.messager.thread.start()
self.adapters = []
self.listeners = []
self.commands = []
self.receive_middleware = Middleware(bot=self)
self.listener_middleware = Middleware(bot=self)
self.catch_all = None
def manage_events(self, params):
sid = request.sid
try:
if params["streaming"]:
with self._thread_lock:
if not sid in self._threads:
mw = Middleware(self._socket, sid, params['topics'],
params['reservoirSize'])
self._middlewares[sid] = mw
self._threads[
sid] = self._socket.start_background_task(
target=mw.start_background_job)
elif not self._threads[sid].is_alive():
mw = Middleware(self._socket, sid, params['topics'],
params['reservoirSize'])
self._middlewares[sid] = mw
self._threads[
sid] = self._socket.start_background_task(
target=mw.start_background_job)
self.send_status(streaming=True)
else:
mw = self._middlewares.get(request.sid, None)
if mw:
mw.stop_backgroung_job()
self.send_status(streaming=False)
except:
print('Request Invalid')
self.send_status(streaming=False,
error=[{
'type': 'Request Invalid'
}])
def process_start_request():
# print("正在访问： "+request.path)
app.wsgi_app = Middleware(app.wsgi_app, request.path)
class Robot(object):
def __init__(self, config_path=None, bot_name="vex"):
self.name = bot_name
self._logger = logging.getLogger(__name__)
self.messager = Messager()
# adapters are inputs into the bot. Like a mic or shell input
adapter_manager = pluginmanager.PluginInterface()
adapter_manager.set_entry_points('vexbot.adapters')
plugins = adapter_manager.collect_entry_point_plugins()
plugin_wrapper = PluginWrapper(plugins[0])
pub_address = 'tcp://127.0.0.1:5555'
plugin_wrapper.activate(invoke_kwargs={'--pub_address': pub_address,
'--prompt_name': 'vexbot'})
self.messager.subscribe_to_address(pub_address)
self.messager.thread.start()
self.adapters = []
self.listeners = []
self.commands = []
self.receive_middleware = Middleware(bot=self)
self.listener_middleware = Middleware(bot=self)
self.catch_all = None
def listen(self, matcher_function, callback):
self.listeners.append(Listener(self, matcher_function, callback))
def hear(self, regex, callback):
match_function = regex.match
self.listeners.append(Listener(self, match_function, callback))
def listener_middleware(self, middleware):
self.listener_middleware.stack.append(middleware)
def _process_listeners(self, response, done=None):
for listener in self.listeners:
result, done = listener.call(response.message.command,
response.message.argument,
done)
#self.listener_middleware.execute(result, done=done)
if isinstance(done, bool) and done:
# TODO: pass back to the appropriate adapter?
print(result)
return
elif isinstance(done, types.FunctionType) and result:
done()
print(result)
return
if self.catch_all is not None:
self.catch_all()
def recieve(self, message, callback=None):
response = Response(self, message)
done = self.receive_middleware.execute(response,
self._process_listeners,
callback)
if isinstance(done, bool) and done:
return
self._process_listeners(response, done)
def run(self, event_loop=None):
loop = asyncio.get_event_loop()
for adapter in self.adapters:
asyncio.async(adapter.run())
try:
loop.run_forever()
except KeyboardInterrupt:
pass
finally:
loop.close()
sys.exit()
def shutdown(self):
pass
from flask import Flask
from flask import request
from flask import jsonify
from middleware import Middleware
app = Flask(__name__)
m = Middleware()
@app.route("/")
def hello():
return "Hello World!"
@app.route("/register", methods=['POST'])
def register():
username = request.json.get('username', None)
password = request.json.get('password', None)
return jsonify(m.register_user(username=username, password=password))
@app.route("/login", methods=['POST'])
def login():
username = request.json.get('username', None)
password = request.json.get('password', None)
return jsonify(m.login_user(username=username, password=password))
@app.route('/send_game_data', methods=['POST'])
def send_game_data():
username = request.json.get('username', None)
class API:
def __init__(self, templates_dir="templates", static_dir="static"):
self.routes = {}
self.exception_handler = None
self.templates_env = Environment(loader=FileSystemLoader(
searchpath=os.path.abspath(templates_dir)))
self.whitenoise = WhiteNoise(application=self.wsgi_app,
root=static_dir)
self.middleware = Middleware(self)
def __call__(self, environ, start_response):
""" We need to distinguish if the request if for static files and use
WhiteNoise, or for data and use the Middleware
"""
path_info = environ["PATH_INFO"]
if path_info.startswith("/static"):
environ["PATH_INFO"] = path_info[len("/static"):]
return self.whitenoise(environ, start_response)
return self.middleware(environ, start_response)
def wsgi_app(self, environ, start_response):
request = Request(environ)
response = self.handle_request(request)
return response(environ, start_response)
def test_session(self, base_url=BASE_URL):
session = RequestsSession()
session.mount(prefix=base_url, adapter=RequestsWSGIAdapter(self))
return session
def add_middleware(self, middleware_cls):
self.middleware.add(middleware_cls)
@staticmethod
def default_response(response):
response.status_code = 404
response.text = "Not found."
def add_exception_handler(self, exception_handler):
self.exception_handler = exception_handler
def find_handler(self, request_path):
""" Look for the corresponding handler for the incoming path """
for path, handler_data in self.routes.items():
parsed_result = parse(path, request_path)
if parsed_result is not None:
return handler_data, parsed_result.named
return None, None
def handle_request(self, request):
response = Response()
handler_data, kwargs = self.find_handler(request_path=request.path)
try:
if handler_data:
handler = handler_data["handler"]
allowed_methods = handler_data["allowed_methods"]
if inspect.isclass(handler):
handler = getattr(handler(), request.method.lower(), None)
if handler is None:
raise AttributeError("Method not allowed",
request.method)
else:
if request.method.lower() not in allowed_methods:
raise AttributeError("Method not allowed",
request.method)
handler(request, response, **kwargs)
else:
self.default_response(response)
except Exception as e:
if self.exception_handler is None:
raise e
else:
self.exception_handler(request, response, e)
return response
def add_route(self, path, handler, allowed_methods=None):
assert path not in self.routes, "Such route already exists."
if allowed_methods is None:
allowed_methods = [
"get", "post", "put", "patch", "delete", "options"
]
self.routes[path] = {
"handler": handler,
"allowed_methods": allowed_methods
}
def route(self, path, allowed_methods=None):
if path in self.routes.keys():
raise AssertionError("Duplicated route")
def wrapper(handler):
self.add_route(path, handler, allowed_methods)
return handler
return wrapper
def template(self, template_name, context=None):
context = {} or context
return self.templates_env.get_template(name=template_name).render(
**context)
class AppFactory:
def __init__(self, templates_dir='templates', static_dir='static'):
self.routes = {}
self.templates_env = Environment(
loader=FileSystemLoader(os.path.abspath(templates_dir)))
self.exception_handler = None
self.whitenoise = WhiteNoise(self.wsgi_app, root=static_dir)
self.middleware = Middleware(self)
def wsgi_app(self, environ, start_response):
request = Request(environ)
response = self.handle_request(request)
return response(environ, start_response)
def __call__(self, environ, start_response):
path_info = environ['PATH_INFO']
if path_info.startswith('/static'):
environ['PATH_INFO'] = path_info[len('/static'):]
return self.whitenoise(environ, start_response)
return self.middleware(environ, start_response)
def add_route(self, route, handler):
assert route not in self.routes, "Route already exists"
self.routes[route] = handler
def route(self, path):
def wrapper(handler):
self.add_route(path, handler)
return handler
return wrapper
def default_response(self, response):
response.status_code = 404
response.text = "Not Found"
def get_handler(self, request_route):
for path, handler in self.routes.items():
parse_params = parse(path, request_route)
if parse_params:
return handler, parse_params.named
return None, None
def handle_request(self, request):
response = Response()
handler, params = self.get_handler(request.path)
try:
if handler:
if inspect.isclass(handler):
handler = getattr(handler(), request.method.lower(), None)
if not handler:
raise AttributeError("Method not allowed",
request.method)
handler(request, response, **params)
else:
self.default_response(response)
except Exception as e:
if self.exception_handler is None:
raise e
else:
self.exception_handler(request, response, e)
return response
def test_client(self, base_url='http://testserv'):
session = RequestsSession()
session.mount(prefix=base_url, adapter=RequestsWSGIAdapter(self))
return session
def template(self, template_name, context=None):
context = context if context else {}
return self.templates_env.get_template(template_name).render(**context)
def add_exception_handler(self, exception_handler):
self.exception_handler = exception_handler
def add_middleware(self, middleware_cls):
self.middleware.add(middleware_cls)
class API:
"""API Class of Arche Framework"""
def __init__(self, templates_dir="templates", static_dir="static"):
self.routes = {}
self.templates_env = Environment(
loader=FileSystemLoader(os.path.abspath(templates_dir)))
self.exception_handler = None
self.whitenoise = WhiteNoise(self._wsgi_app, root=static_dir)
self.middleware = Middleware(self)
def add_route(self, path, handler):
assert path not in self.routes, "A route already exists"
self.routes[path] = handler
def route(self, path):
assert path not in self.routes, "A route already exists"
def wrapper(handler):
self.add_route(path, handler)
return handler
return wrapper
def _wsgi_app(self, env, start_response):
request = Request(env)
response = self.handle_request(request)
return response(env, start_response)
def __call__(self, env, start_response):
path_info = env["PATH_INFO"]
if path_info.startswith("/static"):
env["PATH_INFO"] = path_info[len("/static"):]
return self.whitenoise(env, start_response)
return self.middleware(env, start_response)
def handle_request(self, request):
response = Response()
try:
handler, kwargs = self.find_handler(request_path=request.path)
if handler is not None:
if inspect.isclass(handler):
handler = getattr(handler(), request.method.lower(), None)
if handler is None:
raise AttributeError("Method Not Allowed: ",
request.method)
handler(request, response, **kwargs)
else:
self.default_response(response)
except Exception as e:
if self.exception_handler is None:
raise e
else:
self.exception_handler(request, response, e)
return response
def find_handler(self, request_path):
for path, handler in self.routes.items():
parse_result = parse(path, request_path)
if parse_result is not None:
return handler, parse_result.named
return None, None
def default_response(self, response):
response.status_code = 404
response.text = "OOPS! Your requested page is not Found."
# TODO:
def add_exception_handler(self, exception_handler):
self.exception_handler = exception_handler
def template(self, template_name, context=None):
if context is None:
context = {}
return self.templates_env.get_template(template_name).render(**context)
def add_middleware(self, middleware_cls):
self.middleware.add(middleware_cls)
# Test Session
def test_session(self, base_url="http://testserver"):
session = RequestSession()
session.mount(prefix=base_url, adapter=RequestsWSGIAdapter(self))
return session
def test_9(self):
m = Middleware()
m.remove_user('test user', '123456')
m.remove_user('test user1', '123456dafgda')
m.remove_game_record('test user')
m.remove_game_record('test user1')
m.remove_game_record('test user2')
m.remove_game_record('test user null')
class API(object):
def __init__(self, templates_dir="templates", static_dir="static"):
self.templates_dir = templates_dir
self.static_dir = static_dir
# url路由
self.routes = {}
# html文件夹
self.templates_env = Environment(
loader=FileSystemLoader(os.path.abspath(self.templates_dir)))
# css、JavaScript文件夹
self.whitenoise = WhiteNoise(self.wsgi_app, root=static_dir)
# 自定义错误
self.exception_handler = None
# 请求中间件，将api对象传入
self.middleware = Middleware(self)
def template(self, template_name, context=None):
"""返回模板内容"""
if context is None:
context = {}
return self.templates_env.get_template(template_name).render(**context)
# 不用webob只能直接返回二进制数据
# def __call__(self, environ, start_response):
# response_body = b'Hello, World!'
# status = '200 OK'
# start_response(status, headers=[])
# return iter([response_body])
def wsgi_app(self, environ, start_response):
"""通过 webob 将请求的环境信息转为request对象"""
request = Request(environ)
response = self.handle_request(request)
return response(environ, start_response)
def __call__(self, environ, start_response):
path_info = environ["PATH_INFO"]
static = "/" + self.static_dir
# 以 /static 开头 或 中间件为空
if path_info.startswith(static) or not self.middleware:
# "/static/index.css" -> 只取 /index.css， /static开头只是用于判断
environ["PATH_INFO"] = path_info[len(static):]
return self.whitenoise(environ, start_response)
return self.middleware(environ, start_response)
def handle_request(self, request):
"""请求调度"""
response = Response()
handler, kwargs = self.find_handler(request.path)
try:
if handler is not None:
if inspect.isclass(handler): # 如果是类，则获取其中的方法
handler = getattr(handler(), request.method.lower(), None)
if handler is None: # 类中该方法不存在，则该类不支持该请求类型
raise AttributeError("Method now allowed",
request.method)
handler(request, response, **kwargs)
else:
self.defalut_response(response)
except Exception as e:
if self.exception_handler is None:
raise e
else:
# 自定义错误返回形式
self.exception_handler(request, response, e)
return response
def find_handler(self, request_path):
for path, handler in self.routes.items():
parse_result = parse(path, request_path)
if parse_result is not None:
return handler, parse_result.named
return None, None
def defalut_response(self, response):
response.status_code = 404
response.text = "Not Found"
def route(self, path):
# 添加路由的装饰器
def wrapper(handler):
self.add_route(path, handler)
return handler
return wrapper
def add_route(self, path, handler):
# 相同路径不可重复添加
assert path not in self.routes, "Such route already exists"
self.routes[path] = handler
def add_exception_handler(self, exception_handler):
# 添加自定义error handler
self.exception_handler = exception_handler
def add_middleware(self, middleware_cls):
# 添加中间件
self.middleware.add(middleware_cls)
def initialSetup():
global quadcopter, middleware, message_sender, pi_logger
from middleware import Middleware
from CustomLogger import pi_logger
pi_logger = pi_logger()
quadcopter = Quadcopter(pi_logger)
from message_sender import Message_sender
message_sender = Message_sender(socketio)
middleware = Middleware(quadcopter)
# TODO Implement Logging
if Constants.ENABLE_FLASK_LOGGING:
formatter = logging.Formatter(
Constants.LOG_FORMAT_FLASK)
handler = RotatingFileHandler(os.path.join(Constants.LOG_LOCATION_FLASK, Constants.LOG_FILENAME_FLASK),
maxBytes=10000000, backupCount=5)
handler.setLevel(logging.DEBUG)
handler.setFormatter(formatter)
app.logger.addHandler(handler)
import bonjour
service = bonjour.Bonjour(pi_logger)
service.publish()
global queue
from gevent import queue, Greenlet
queue = queue.Queue()
# http://www.gevent.org/gevent.queue.html
global sender, reader, senderexternal, readerexternal
reader, sender = gipc.pipe()
readerexternal, senderexternal = gipc.pipe()
global thread, thread3
if Constants.ENABLE_SERIAL:
# thread = threading.Thread(name="Serial Thread",
thread = gipc.start_process(name="Serial Thread",
daemon=True,
target=read_from_port,
kwargs={'port': Constants.ARDUINO_PORT,
'baud_rate': Constants.ARDUINO_BAUDRATE,
'sender': sender,
'logger': pi_logger,
}
)
# Greenlet.spawn(passmessagetoproc,senderexternal)
thread3 = gipc.start_process(name="PID Thread",
daemon=True,
target=speed_control,
kwargs={'reader': reader,
'quadcopter': quadcopter,
'message_sender': message_sender,
'middleware': middleware, 'logger': pi_logger,
'reader2': readerexternal}
)
# speed_control(reader=reader,quadcopter=quadcopter,message_sender=message_sender,middleware=middleware,logger=pi_logger)
# message_sender.__send_msg_to_arduino__("HELLOn\n")
@socketio.on('connect', namespace=Constants.SOCKETIO_NAMESPACE)
def test_connect():
print 'Client connected : %s' % request
return True
@socketio.on('disconnect', namespace=Constants.SOCKETIO_NAMESPACE)
def test_disconnect():
print 'Client disconnected : %s' % request
return True
@socketio.on('message', namespace=Constants.SOCKETIO_NAMESPACE)
def handle_message(data):
print data
socketio.emit('message', 'HELLOAGAIN')
global middleware
middleware.parseMessage(data)
mc_servers.append(mc_server)
mc_server_string_list.append("{}:{}".format(private_hostnames[i],
memcached_port))
if UPDATE_AND_INSTALL:
mc_server.update_and_install()
for s in mc_servers:
s.start()
# Set up middleware server
middleware_port = 11212
log.info("Setting up middleware on machine {} ({}).".format(
index_a4, vm_names[index_a4]))
mw_server = Middleware(public_hostnames[index_a4],
private_hostnames[index_a4],
middleware_port,
NUM_THREADS_IN_POOL,
REPLICATION_FACTOR,
mc_server_string_list,
ssh_username=ssh_username)
if UPDATE_AND_INSTALL or UPDATE_AND_INSTALL_ONLY_MIDDLEWARE:
mw_server.update_and_install()
mw_server.clear_logs()
mw_server.start()
# Sleep a bit so middleware has time to start
if not mw_server.is_running():
sleep_for = 5
log.info(
"Sleeping for {} seconds so middleware can start...".format(sleep_for))
time.sleep(sleep_for)
class TestMiddleware(unittest.TestCase):
def setUp(self):
self.m = Middleware()
def test_0_register_user(self):
self.assertEqual(self.m.register_user('test user', '123456'), {
'status': 'success',
'payload': ''
})
self.assertEqual(self.m.register_user('test user1', '123456dafgda'), {
'status': 'success',
'payload': ''
})
self.assertEqual(self.m.register_user('test user1', '123456dafgda'), {
'status': 'fail',
'payload': 'duplicate name'
})
def test_1_login_user(self):
self.assertEqual(self.m.login_user('testuser', '123456'), {
'status': 'fail',
'payload': 'wrong username or password'
})
self.assertEqual(self.m.login_user('test user', '123456'), {
'status': 'success',
'payload': ''
})
self.assertEqual(self.m.register_user('test user1', '123456dafgda'), {
'status': 'fail',
'payload': 'duplicate name'
})
def test_2_remove_user(self):
self.assertEqual(self.m.remove_user('tesser', '123456'), 0)
self.assertEqual(self.m.remove_user('test user', '123 456'), 0)
self.assertEqual(self.m.remove_user('test user', '123456'), 1)
self.assertEqual(self.m.remove_user('test user1', '123456dafgda'), 1)
self.assertEqual(self.m.remove_user('test user1', '123456dafgda'), 0)
def test_3_create_game_record(self):
self.assertEqual(self.m.create_game_record('test user', 59), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user', 97), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user1', 51), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user2', 28), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user1', 5), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user', 82), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user1', 73), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user2', 19), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user', 44), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user', 94), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user2', 63), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user', 8), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user1', 79), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user2', 60), {
"status": "success",
"payload": ""
})
self.assertEqual(self.m.create_game_record('test user2', 55), {
"status": "success",
"payload": ""
})
def test_4_get_game_record_by_user(self):
records = self.m.get_game_record_by_user('test user')
self.assertEqual(records['status'], 'success')
self.assertEqual(len(records['payload']), 6)
truth = [97, 94, 82, 59, 44, 8]
for i, re in enumerate(records['payload']):
self.assertEqual(re[u'username'], u'test user')
self.assertEqual(re[u'score'], truth[i])
def test_5_get_best_game_record(self):
records = self.m.get_best_game_record()
self.assertEqual(records['status'], 'success')
self.assertEqual(len(records['payload']), 10)
truth = [97, 94, 82, 79, 73, 63, 60, 59, 55, 51]
user = ["", "", "", "1", "1", "2", "2", "", "2", "1"]
for i, re in enumerate(records['payload']):
self.assertEqual(re[u'username'], 'test user' + user[i])
self.assertEqual(re[u'score'], truth[i])
def test_6_remove_game_record(self):
self.assertEqual(self.m.remove_game_record('test user'), 6)
self.assertEqual(self.m.remove_game_record('test user1'), 4)
self.assertEqual(self.m.remove_game_record('test user2'), 5)
self.assertEqual(self.m.remove_game_record('test user null'), 0)
class API:
def __init__(self, templates_directory="templates", static_dir="static"):
self.routes = {} #paths are the keys and handlers (functions or classes) are the values
self.templates_environment = Environment(
loader=FileSystemLoader(os.path.abspath(templates_directory))
)
self.exception_handler = None
self.whitenoise = WhiteNoise(self.wsgi_application, root=static_dir) #wrap wsgi application to serve static files
self.middleware = Middleware(self)
def __call__(self, environ, start_response):
"""
Compatible WSGI server will call for each client HTTP request.
Request for static files are treated differently from other request.
"""
print("Callable was triggered due to request from client application at time: " + str(time.time()))
path_info = environ["PATH_INFO"]
if path_info.startswith("/static"):
environ["PATH_INFO"] = path_info[len("/static"):]
print(environ["PATH_INFO"])
return self.whitenoise(environ, start_response)
else:
return self.middleware(environ, start_response)
def wsgi_application(self, environ, start_response):
request = Request(environ)
response = self.handle_request(request)
return response(environ, start_response)
"""
Middleware
"""
def add_middleware(self, middleware_cls):
self.middleware.add(middleware_cls)
"""
TEST CLIENT
"""
def test_session(self, base_url="http://testserver"):
"""
Creates a test client associated with the given a url.
The test session will be a requests session to be able to
emulate a client in the browser.
"""
session = RequestsSession()
session.mount(prefix=base_url, adapter=RequestsWSGIAdapter(self))
return session
"""
ROUTING
"""
def add_route(self, path, handler):
"""
Adds a route which is a key-value pair.
The key is the url path and the value is either
a function handler or a class-based route.
"""
if path in self.routes:
raise AssertionError("Failed. Such a route already exists.")
self.routes[path] = handler
def route(self, path):
"""
Adds a route via a decorator function.
"""
def wrapper(handler):
self.routes[path] = handler
return handler
return wrapper
"""
TEMPLATING
"""
def get_template(self, template_name, context=None):
"""
Gets the template based on the template name
"""
if context is None:
context = {}
return self.templates_environment.get_template(template_name).render(**context)
"""
HANDLING
"""
def handle_request(self, request):
"""
Handles the client request, which is an webob object.
"""
response = Response()
handler, kwargs = self.lookup_handler(request_path=request.path)
try:
if handler is not None:
if inspect.isclass(handler):
handler_method = self.get_class_method(handler, request)
if handler_method is None:
raise AttributeError("Method not allowed", request.method)
handler_method(request, response, **kwargs)
else:
handler(request, response, **kwargs)
else:
self.default_response(response)
except Exception as exception:
if self.exception_handler is None:
raise exception
else:
self.exception_handler(request, response, exception)
return response
def get_class_method(self, handler, request):
"""
Gets the method associated with the requested path.
This function is used with class based routes.
"""
handler_method = getattr(handler(),request.method.lower(), None)
return handler_method
def lookup_handler(self, request_path):
"""
Finds the function handler associated with the requested path
"""
for path, handler in self.routes.items():
parse_result = parse(path, request_path)
if parse_result is not None:
return handler, parse_result.named
return None, None
def add_exception_handler(self, exception_handler):
"""
Adds a function to the application that handle exceptions
"""
self.exception_handler = exception_handler
"""
RESPONSES
"""
def default_response(self, response):
response.status_code = 404
response.text = "Requested path not found."
import casbin
from middleware import Middleware
from flask import Flask
app = Flask(__name__)
enforcer = casbin.Enforcer("authz_model.conf", "policy.csv")
app.wsgi_app = Middleware(app.wsgi_app, enforcer)
@app.route("/")
def hello_world():
return "Hello World!"
if __name__ == '__main__':
app.run()
def start_experiment(self,
results_dir,
update_and_install=False,
experiment_runtime = 5, # minutes
runtime_buffer = 1, # minutes
stats_frequency ="30s",
num_threads_in_pool = 5,
replication_factor = 1,
memaslap_workload = "smallvalue.cfg",
memaslap_window_size = "10k",
hibernate_at_end = True,
ssh_username = "******",
num_memaslaps = 1,
num_memcacheds = 1,
concurrency = 64,
is_first_run=True
):
experiment_runtime_string = "{}m".format(experiment_runtime)
with fabric.api.settings(warn_only=True):
fabric.api.local("rm -r {}/*".format(results_dir))
fabric.api.local("mkdir -p {}".format(results_dir))
fabric.api.local("mkdir {}/graphs".format(results_dir))
# region ---- Parameters ----
TOTAL_MACHINE_COUNT = 11 # this is fixed by the template
resource_group_name = 'template11vms'
my_pub_ssh_key_path = '~/.ssh/id_rsa_asl.pub'
template_path = "azure-templates/template11vms.json"
pub_ssh_key_path = os.path.expanduser(my_pub_ssh_key_path)
with open(pub_ssh_key_path, 'r') as pub_ssh_file_fd:
pub_ssh_key = pub_ssh_file_fd.read().strip()
parameters = {
"virtualMachines_name": "foraslvms",
"virtualMachines_adminPassword": "******",
"networkInterfaces_name": "MyNetworkInterface",
"virtualNetworks_testeth_vnet_name": "MyVNet",
"key": pub_ssh_key,
"uniquedns": "pungast"
}
# endregion
# Initialize the deployer class
if is_first_run:
self.deployer = Deployer(resource_group_name, template_path, parameters)
self.deployer.deploy_wait()
# region ---- Extract VMs' IPs and other information ----
vms = self.deployer.compute_client.virtual_machines.list(resource_group_name)
vm_names = []
vm_types = []
public_hostnames = []
private_hostnames = []
for vm in vms:
vm_type = vm.hardware_profile.vm_size
vm_types.append(vm_type)
vm_names.append(vm.name)
self.log.info("VM {} [{}]".format(Colors.ok_blue(vm.name), vm_type))
# Get machine's public address that we can use for SSH-ing
public_ip = self.deployer.network_client.public_ip_addresses.get(resource_group_name, vm.name)
public_host_address = public_ip.dns_settings.fqdn
public_hostnames.append(public_host_address)
#self.log.info("Public host name: {}".format(Colors.ok_green(public_host_address)))
# Get machine's private IP address
network_interface_id = vm.network_profile.network_interfaces[0].id
network_interface_name = network_interface_id.split("/")[-1]
network_interface = self.deployer.network_client.network_interfaces.get(resource_group_name, network_interface_name)
private_host_address = network_interface.ip_configurations[0].private_ip_address
private_hostnames.append(private_host_address)
#self.log.info("Private host name: {}".format(Colors.ok_green(private_host_address)))
# endregion
# region ---- Set up all machines ----
index_a4 = vm_types.index("Basic_A4")
indices_smallmachines = list(range(TOTAL_MACHINE_COUNT))
indices_smallmachines.remove(index_a4)
memcached_machines = [vm_names.index("foraslvms" + str(x)) for x in Experiment.default_memcached_machines()]
memcached_machines = memcached_machines[0:num_memcacheds]
memaslap_machines = [vm_names.index("foraslvms" + str(x)) for x in Experiment.default_memaslap_machines()]
memaslap_machines = memaslap_machines[0:num_memaslaps]
self.log.info("A4 machine: " + str(index_a4))
self.log.info("A2 machines: " + str(indices_smallmachines))
self.log.info("Memcached machines: " + str(memcached_machines))
self.log.info("Memaslap machines: " + str(memaslap_machines))
# Wait for all servers to be responsive
if is_first_run:
aslutil.wait_for_servers(ssh_username, public_hostnames, "~/.ssh/id_rsa_asl", self.log, check_every_n_sec=10)
# Set up memcached servers
memcached_port = 11211
mc_servers = []
mc_server_string_list = []
for i in memcached_machines:
self.log.info("Setting up memcached on machine {} ({}).".format(i, vm_names[i]))
mc_server = Memcached(memcached_port, public_hostnames[i], ssh_username=ssh_username,
id_number=int(aslutil.server_name_to_number(vm_names[i])))
mc_servers.append(mc_server)
mc_server_string_list.append("{}:{}".format(private_hostnames[i], memcached_port))
if update_and_install:
mc_server.update_and_install()
for s in mc_servers:
s.start()
sleep_for = 15
self.log.info("Sleeping for {} seconds so memcached servers can start...".format(sleep_for))
time.sleep(sleep_for)
# Set up middleware server
middleware_port = 11212
self.log.info("Setting up middleware on machine {} ({}).".format(index_a4, vm_names[index_a4]))
mw_server = Middleware(public_hostnames[index_a4], private_hostnames[index_a4], middleware_port,
num_threads_in_pool, replication_factor, mc_server_string_list, ssh_username=ssh_username)
if update_and_install:
mw_server.update_and_install()
if is_first_run:
mw_server.upload_jar()
mw_server.clear_logs()
mw_server.start()
# Sleep a bit so middleware has time to start
while not mw_server.is_running():
sleep_for = 5
self.log.info("Sleeping for {} seconds so middleware can start...".format(sleep_for))
time.sleep(sleep_for)
time.sleep(10)
# Set up memaslap servers
ms_servers = []
first_memaslap = True
for i in memaslap_machines:
self.log.info("Setting up memaslap on machine {} ({}).".format(i, vm_names[i]))
ms_server = Memaslap(public_hostnames[i], private_hostnames[index_a4], middleware_port, ssh_username=ssh_username,
id_number=int(aslutil.server_name_to_number(vm_names[i]))) # i is zero-indexed
ms_servers.append(ms_server)
if is_first_run:
ms_server.upload_resources()
if update_and_install:
if not first_memaslap:
ms_server.upload_built_files()
ms_server.update_and_install()
if first_memaslap:
ms_server.download_built_files()
first_memaslap = False
for s in ms_servers:
s.clear_logs()
s.start(runtime=experiment_runtime_string, log_filename="memaslap{}.out".format(s.id_number),
stats_freq=stats_frequency, workload_filename=memaslap_workload, concurrency=concurrency,
window_size=memaslap_window_size)
# endregion
sleep_time = experiment_runtime + runtime_buffer
self.log.info("Waiting for the experiment to finish, sleeping for up to {} minutes.".format(sleep_time))
already_slept = 0
while True:
sleep_interval = 30
time.sleep(sleep_interval)
already_slept += sleep_interval
num_running_memaslaps = sum([s.is_running() for s in ms_servers])
self.log.info("Waiting for the experiment to finish (total {} minutes), {:.0f}/{} minutes elapsed ({:.0f}%), {} memaslaps running."
.format(sleep_time,
already_slept / 60, experiment_runtime,
100 * already_slept / 60.0 / experiment_runtime,
num_running_memaslaps))
if already_slept >= sleep_time * 60:
self.log.info("Stopping because of time limit.")
break
if num_running_memaslaps == 0:
self.log.info("Stopping because no memaslaps are left.")
break
# region ---- Kill everyone ----
# Memaslap
for ms_server in ms_servers:
ms_server.stop()
# Middleware
mw_server.stop()
# Memcached
for mc_server in mc_servers:
mc_server.stop()
# endregion
# region ---- Download logs, extract data, plot ----
mw_server.download_logs(local_path=results_dir)
for ms_server in ms_servers:
ms_server.download_logs(local_path=results_dir)
# endregion
if hibernate_at_end:
self.deployer.hibernate_wait()
self.log.info("Done.")
class API:
def __init__(self, templates_dir="templates", static_dir="static"):
self.routes = {}
self.templates_env = Environment(
loader=FileSystemLoader(os.path.abspath(templates_dir)))
self.exception_handler = None
self.whitenoise = WhiteNoise(self.wsgi_app, root=static_dir)
self.static_dir = os.path.abspath(static_dir)
self._static_root = "/static"
self.middleware = Middleware(self)
def wsgi_app(self, environ, start_response):
request = Request(environ)
response = self.handle_request(request)
return response(environ, start_response)
def __call__(self, environ, start_response):
path_info = environ["PATH_INFO"]
if request_for_static(path_info, self._static_root):
environ["PATH_INFO"] = cut_static_root(path_info,
self._static_root)
return self.whitenoise(environ, start_response)
return self.middleware(environ, start_response)
def add_middleware(self, middleware_cls):
self.middleware.add(middleware_cls)
def route(self, path):
def wrapper(handler):
self.add_route(path, handler)
return handler
return wrapper
def add_route(self, path, handler):
assert path not in self.routes, f"{path} already exists."
self.routes[path] = handler
def test_session(self, base_url="http:''testserver"):
session = RequestsSession()
session.mount(prefix=base_url, adapter=RequestsWSGIAdapter(self))
return session
def handle_request(self, request):
response = Response()
handler, kwargs = self.find_handler(request_path=request.path)
try:
if handler is not None:
if inspect.isclass(handler):
handler = getattr(handler(), request.method.lower(), None)
if handler is None:
raise AttributeError("Method in not allowed",
request.method)
handler(request, response, **kwargs)
else:
self.default_response(response)
except Exception as e:
if self.exception_handler is None:
raise e
else:
self.exception_handler(request, response, e)
return response
def default_response(self, response):
response.status_code = 404
response.text = "Not found"
def find_handler(self, request_path):
for path, handler in self.routes.items():
parse_result = parse(path, request_path)
if parse_result is not None:
return handler, parse_result.named
return None, None
def template(self, template_name, context=None):
if context is None:
context = {}
return self.templates_env.get_template(template_name).render(**context)
def add_exception_handler(self, exception_handler):
self.exception_handler = exception_handler
from flask import Flask, request
from middleware import Middleware
app = Flask('app')
app.wsgi_app = Middleware(app.wsgi_app)
@app.route('/', methods=['GET'])
def hello_world():
user = request.environ['user']['name']
return f'Hi {user}'
if __name__ == '__main__':
app.run('127.0.0.1', '5000', debug=True)
def setUp(self):
self.m = Middleware()