def __init__(self, app, config):
try:
self._state = app.events.State()
self._config = config
self._storage = DatabaseStorage(config.db_name)
self._init_event_receiver(app)
except KeyboardInterrupt:
# return gracefully.
pass
def __init__(self, app, config):
try:
self._state = app.events.State()
self._config = config
self._storage = DatabaseStorage(config.db_name)
self._init_event_receiver(app)
except KeyboardInterrupt:
# return gracefully.
pass
class EventProcessor:
"""
Class which should be executed as background task. Updates task results and keeps track of workers going on-
or offline.
"""
def __init__(self, app, config):
try:
self._state = app.events.State()
self._config = config
self._storage = DatabaseStorage(config.db_name)
self._init_event_receiver(app)
except KeyboardInterrupt:
# return gracefully.
pass
def worker_online(self, event):
"""
Event handler when worker comes online.
:param event:
"""
self._state.event(event)
name = event['hostname'].split("@")[1]
init_time = event['timestamp']
result = DBUtil.execute_command(
self._config.db_name, "SELECT started FROM Workers WHERE name = ?",
(name, ), "ONE")
if result is not None:
starttime = init_time - result[0]
print "STARTTIME: " + str(starttime)
DBUtil.execute_command(
self._config.db_name,
"UPDATE Workers SET initialized = 'true', starttime = ?, heartbeat = ?, active = ? WHERE name = ? ",
(starttime, time.time(), time.time(), name))
def worker_heartbeat(self, event):
"""
Event handler for received heartbeat from a worker
:param event:
"""
hostname = event['hostname']
if "active" in event:
print "active found: " + str(event['active'])
self._update_worker(hostname, event['active'])
else:
self._update_worker(hostname)
def worker_offline(self, event):
"""
Event handler when worker goes offline.
:param event:
"""
self._state.event(event)
name = event['hostname']
self._delete_worker_by_hostname(name)
def task_started(self, event):
self._state.event(event)
hash_key = event['uuid']
started = event['timestamp']
print "task started"
DBUtil.execute_command(
self._config.db_name,
"UPDATE Results SET started = ? WHERE name = ?",
(started, hash_key))
def task_succeeded(self, event):
"""
Event handler when task succeeds
:param event:
"""
self._state.event(event)
hash_key = event['uuid']
endtime = event['timestamp']
asyncresult = AsyncResult(hash_key)
self._storage.save_result_hash(hash_key, asyncresult.get(), None,
endtime)
print "Task succeeded:" + hash_key
def _delete_worker_by_hostname(self, hostname):
"""
Internal method to delete worker from database by using its hostname
:param hostname:
:return:
"""
name = hostname.split("@")[1]
DBUtil.execute_command(self._config.db_name,
"DELETE FROM Workers WHERE name = ?", (name, ))
def _init_event_receiver(self, app):
"""
Internal method to allow event handling.
:param app:
:return:
"""
with app.connection() as connection:
recv = app.events.Receiver(connection,
handlers={
'task-succeeded':
self.task_succeeded,
'task-started': self.task_started,
'worker-online': self.worker_online,
'worker-heartbeat':
self.worker_heartbeat,
'worker-offline':
self.worker_offline
})
recv.capture(limit=None, timeout=None, wakeup=True)
def _update_worker(self, hostname, active=0):
name = hostname.split("@")[1]
heartbeat = time.time()
if active > 0:
DBUtil.execute_command(
self._config.db_name,
"UPDATE Workers SET heartbeat = ?, active = ? WHERE name = ?",
(heartbeat, heartbeat, name))
else:
DBUtil.execute_command(
self._config.db_name,
"UPDATE Workers SET heartbeat = ? WHERE name = ?",
(heartbeat, name))
from storage.DatabaseStorage import DatabaseStorage
from celery.task.control import discard_all
from model.Config import Config
from flask import session, redirect, url_for, \
render_template, flash
import json
from utils import DBUtil
app = Flask(__name__, template_folder="web/templates", static_folder="web/static")
config = Config()
SECRET_KEY = config.secret_key
app.config.from_object(__name__)
kv_storage = DatabaseStorage(config.db_name)
worker_manager = DefaultWorkerManager(config, config.db_name)
comp_manager = DefaultComputeManager(worker_manager,kv_storage, config)
@app.route('/interface', methods=['GET'])
def web_interface():
return send_file("interface.html")
@app.route("/job", methods=['POST', 'GET'])
def create_job():
user_params = UserParameters()
user_params.naca4[0] = int(request.form["n0"])
user_params.naca4[1] = int(request.form["n1"])
user_params.naca4[2] = int(request.form["n2"])
class EventProcessor:
"""
Class which should be executed as background task. Updates task results and keeps track of workers going on-
or offline.
"""
def __init__(self, app, config):
try:
self._state = app.events.State()
self._config = config
self._storage = DatabaseStorage(config.db_name)
self._init_event_receiver(app)
except KeyboardInterrupt:
# return gracefully.
pass
def worker_online(self, event):
"""
Event handler when worker comes online.
:param event:
"""
self._state.event(event)
name = event['hostname'].split("@")[1]
init_time = event['timestamp']
result = DBUtil.execute_command(self._config.db_name, "SELECT started FROM Workers WHERE name = ?", (name,), "ONE")
if result is not None:
starttime = init_time - result[0]
print "STARTTIME: " + str(starttime)
DBUtil.execute_command(self._config.db_name, "UPDATE Workers SET initialized = 'true', starttime = ?, heartbeat = ?, active = ? WHERE name = ? ", (starttime, time.time(), time.time(), name))
def worker_heartbeat(self, event):
"""
Event handler for received heartbeat from a worker
:param event:
"""
hostname = event['hostname']
if "active" in event:
print "active found: " + str(event['active'])
self._update_worker(hostname, event['active'])
else:
self._update_worker(hostname)
def worker_offline(self, event):
"""
Event handler when worker goes offline.
:param event:
"""
self._state.event(event)
name = event['hostname']
self._delete_worker_by_hostname(name)
def task_started(self, event):
self._state.event(event)
hash_key = event['uuid']
started = event['timestamp']
print "task started"
DBUtil.execute_command(self._config.db_name, "UPDATE Results SET started = ? WHERE name = ?", (started, hash_key))
def task_succeeded(self, event):
"""
Event handler when task succeeds
:param event:
"""
self._state.event(event)
hash_key = event['uuid']
endtime = event['timestamp']
asyncresult = AsyncResult(hash_key)
self._storage.save_result_hash(hash_key, asyncresult.get(), None, endtime)
print "Task succeeded:" + hash_key
def _delete_worker_by_hostname(self, hostname):
"""
Internal method to delete worker from database by using its hostname
:param hostname:
:return:
"""
name = hostname.split("@")[1]
DBUtil.execute_command(self._config.db_name, "DELETE FROM Workers WHERE name = ?", (name,))
def _init_event_receiver(self, app):
"""
Internal method to allow event handling.
:param app:
:return:
"""
with app.connection() as connection:
recv = app.events.Receiver(connection, handlers={
'task-succeeded': self.task_succeeded,
'task-started': self.task_started,
'worker-online': self.worker_online,
'worker-heartbeat': self.worker_heartbeat,
'worker-offline': self.worker_offline
})
recv.capture(limit=None, timeout=None, wakeup=True)
def _update_worker(self, hostname, active=0):
name = hostname.split("@")[1]
heartbeat = time.time()
if active > 0:
DBUtil.execute_command(self._config.db_name, "UPDATE Workers SET heartbeat = ?, active = ? WHERE name = ?", (heartbeat, heartbeat, name))
else:
DBUtil.execute_command(self._config.db_name, "UPDATE Workers SET heartbeat = ? WHERE name = ?", (heartbeat, name))
