def sync_weibo():
# query entities needed to sync
entities = Entity.objects.filter(sync_timestamp1__range=(datetime.datetime.now(), datetime.datetime.now()+datetime.timedelta(minutes=5)), status1=1)
for entity in entities:
weibo_message = None
if entity.type == Entity.GAME:
weibo_message = weibo_message_builder.build_game_message(Game.objects.get(id=entity.id))
logger.info('sync game %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.REDIER:
weibo_message = weibo_message_builder.build_redier_message(Redier.objects.get(id=entity.id))
logger.info('sync redier %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.COLLECTION:
weibo_message = weibo_message_builder.build_collection_message(Collection.objects.get(id=entity.id))
logger.info('sync collection %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.PROBLEM:
weibo_message = weibo_message_builder.build_problem_message(Problem.objects.get(id=entity.id))
logger.info('sync problem %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.PLAYER:
weibo_message = weibo_message_builder.build_player_message(Player.objects.get(id=entity.id))
logger.info('sync player %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.PUZZLE:
weibo_message = weibo_message_builder.build_puzzle_message(Puzzle.objects.get(id=entity.id))
logger.info('sync puzzle %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.NEWS:
weibo_message = weibo_message_builder.build_news_message(News.objects.get(id=entity.id))
logger.info('sync news %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.EVALUATION:
weibo_message = weibo_message_builder.build_evaluation_message(Evaluation.objects.get(id=entity.id))
logger.info('sync evaluation %s to weibo' % weibo_message.entity_id)
if weibo_message is not None:
MessageSender.send_weibo(weibo_message)
else:
logger.info('nothing to sync to weibo')
def __init__(self):
self._runner_process = None
self._control_process = None
self._watchdog = ProcessWatchdog(config.Watchdog.CHECK_INTERVAL)
self.push_messages_receiver = PushMessageReceiver()
self.config_builder = ConfigurationBuilder(
config.Engine.CONFIGURATION_BUILDER)
self.message_handler = MessageHandler(self)
self.message_sender = MessageSender()
self.message_router = MessageRouter(
self.message_sender, self.message_handler.default_message_handler)
self.state = ManagerState.EMPTY
self._http_client = httpclient.HTTPClient()
self._alert_registered = False
self.obsi_id = getnode() # A unique identifier of this OBSI
self._engine_running = False
self._engine_running_lock = locks.Lock()
self._processing_graph_set = False
self._engine_config_builder = None
self._keep_alive_periodic_callback = None
self._avg_cpu = 0
self._avg_duration = 0
self._supported_elements_types = []
self._alert_messages_handler = None
self._log_messages_handler = None
def sync_web():
#query entities needed to sync
entities = Entity.objects.filter(
sync_timestamp3__range=(datetime.datetime.now(),
datetime.datetime.now() +
datetime.timedelta(minutes=5)),
status3=1)
for entity in entities:
web_message = None
if entity.type == Entity.GAME:
web_message = web_message_builder.build_game_message(
Game.objects.get(id=entity.id))
logger.info('sync game %s to web' % entity.id)
if entity.type == Entity.NEWS:
web_message = web_message_builder.build_news_message(
News.objects.get(id=entity.id))
logger.info('sync news %s to web' % entity.id)
if entity.type == Entity.COLLECTION:
web_message = web_message_builder.build_collection_message(
Collection.objects.get(id=entity.id))
logger.info('sync collection %s to web' % entity.id)
if entity.type == Entity.PUZZLE:
web_message = web_message_builder.build_puzzle_message(
Puzzle.objects.get(id=entity.id))
logger.info('sync puzzle %s to web' % entity.id)
if entity.type == Entity.EVALUATION:
web_message = web_message_builder.build_evaluation_message(
Evaluation.objects.get(id=entity.id))
logger.info('sync evaluation %s to web' % entity.id)
if web_message is not None:
MessageSender.send_web(web_message)
def sync_weixin():
#query weixin messages
weixin_msgs = Weixin.objects.filter(
sync_timestamp2__range=(datetime.datetime.now(),
datetime.datetime.now() +
datetime.timedelta(minutes=5)),
status2=1)
for msg in weixin_msgs:
logger.info('sync msg %s to weixin' % msg.id)
MessageSender.send_weixin(
weixin_message_builder.build_weixin_message(msg))
def main():
running = True
message_queue = queue.Queue()
print("Creating mqtt class..")
mqtt_thread = MqttConnector()
mqtt_thread.start()
## Start sensors
temp_sensor = TemperatureSensor(message_queue)
#ir_temp_sensor = IRTemperatureSensor(message_queue)
#ir2_temp_sensor = IR2TemperatureSensor(message_queue)
#gps_sensor = GpsSensor(message_queue)
humidity_sensor = HumiditySensor(message_queue)
pressure_sensor = PressureSensor(message_queue)
orientation_sensor = OrientationSensor(message_queue)
acceleration_sensor = AccelerationSensor(message_queue)
imu_sensor = UDPReceiver(cfg.ip_imu, cfg.port_imu, message_queue, "IMU")
seanav_sensor = UDPReceiver(cfg.ip_seanav, cfg.port_seanav, message_queue,
"SEANAV")
temp_sensor.start()
#ir_temp_sensor.start()
#ir2_temp_sensor.start()
#gps_sensor.start()
humidity_sensor.start()
pressure_sensor.start()
orientation_sensor.start()
acceleration_sensor.start()
imu_sensor.start()
seanav_sensor.start()
## Connect to databases
db_handler = DBHandler(local=True, cloud=True)
## Wait for connected to server
while not mqtt_thread.connected:
continue
print("Connected...")
print("Start message sender")
sender = MessageSender(message_queue, mqtt_thread, db_handler)
sender.start()
## Keep from stopping program
while (running):
continue
print("Program end")
def connect_to_next_next_hop(self):
ip, port = self.next_next_hop_info
try:
s = socket.create_connection((ip, port))
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, s)
self.message_sender.start()
self.next_hop_info = self.next_next_hop_info
# After we connect to a new client we have to check whether the dead client wasn't in posession
# of token
self.sending_queue.put(
events.TokenReceivedQuestionEvent(self.last_token))
except Exception as e:
logger.error(e)
def sync_weibo():
# query entities needed to sync
entities = Entity.objects.filter(
sync_timestamp1__range=(datetime.datetime.now(),
datetime.datetime.now() +
datetime.timedelta(minutes=5)),
status1=1)
for entity in entities:
weibo_message = None
if entity.type == Entity.GAME:
weibo_message = weibo_message_builder.build_game_message(
Game.objects.get(id=entity.id))
logger.info('sync game %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.REDIER:
weibo_message = weibo_message_builder.build_redier_message(
Redier.objects.get(id=entity.id))
logger.info('sync redier %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.COLLECTION:
weibo_message = weibo_message_builder.build_collection_message(
Collection.objects.get(id=entity.id))
logger.info('sync collection %s to weibo' %
weibo_message.entity_id)
elif entity.type == Entity.PROBLEM:
weibo_message = weibo_message_builder.build_problem_message(
Problem.objects.get(id=entity.id))
logger.info('sync problem %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.PLAYER:
weibo_message = weibo_message_builder.build_player_message(
Player.objects.get(id=entity.id))
logger.info('sync player %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.PUZZLE:
weibo_message = weibo_message_builder.build_puzzle_message(
Puzzle.objects.get(id=entity.id))
logger.info('sync puzzle %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.NEWS:
weibo_message = weibo_message_builder.build_news_message(
News.objects.get(id=entity.id))
logger.info('sync news %s to weibo' % weibo_message.entity_id)
elif entity.type == Entity.EVALUATION:
weibo_message = weibo_message_builder.build_evaluation_message(
Evaluation.objects.get(id=entity.id))
logger.info('sync evaluation %s to weibo' %
weibo_message.entity_id)
if weibo_message is not None:
MessageSender.send_weibo(weibo_message)
else:
logger.info('nothing to sync to weibo')
def inner_next_hop_broken_event(self, _):
# If we detect that the next hop connection is down we want to:
# 1.Try to reconnect to the client
# 2.If reconnect fails we want to connect to our next next hop
# 3.When we succesfully connect to our next next hop we want to send recovery token question
# in case that the dead client was holding the token the moment he died
ip, port = self.next_next_hop_info
# If we are the only client left we reset the data to the initial state
if ip == helpers.get_self_ip_address():
self.critical_section = None
self.next_hop_info = None
self.next_next_hop_info = None
if self.message_sender:
self.message_sender.stop()
self.sending_queue = None
self.message_sender = None
self.predecessor = None
self.last_token = 0
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_OPEN})
return
def connect_to_next_next_hop(self):
ip, port = self.next_next_hop_info
try:
s = socket.create_connection((ip, port))
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, s)
self.message_sender.start()
self.next_hop_info = self.next_next_hop_info
# After we connect to a new client we have to check whether the dead client wasn't in posession
# of token
self.sending_queue.put(
events.TokenReceivedQuestionEvent(self.last_token))
except Exception as e:
logger.error(e)
ip, port = self.next_hop_info
try:
s = socket.create_connection((ip, port))
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, s)
self.message_sender.start()
except ConnectionRefusedError as e:
logger.error(e)
connect_to_next_next_hop(self)
def __init__(self, address=("127.0.0.1", 7222), buffer_size=65536):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
registry = Registry()
self._message_receiver = MessageReceiver(
IncomingMessageTranslator(
[registry,
Broadcaster(registry,
MessageSender(OutgoingMessageTranslator(), sock))
]), address, buffer_size)
def main(argv):
client = MessageSender()
client.connect_messaging_socket("18.214.123.134", 5050)
client.send_message("test message")
return
def sync_web():
#query entities needed to sync
entities = Entity.objects.filter(sync_timestamp3__range=(datetime.datetime.now(), datetime.datetime.now()+datetime.timedelta(minutes=5)), status3=1)
for entity in entities:
web_message = None
if entity.type == Entity.GAME:
web_message = web_message_builder.build_game_message(Game.objects.get(id=entity.id))
logger.info('sync game %s to web' % entity.id)
if entity.type == Entity.NEWS:
web_message = web_message_builder.build_news_message(News.objects.get(id=entity.id))
logger.info('sync news %s to web' % entity.id)
if entity.type == Entity.COLLECTION:
web_message = web_message_builder.build_collection_message(Collection.objects.get(id=entity.id))
logger.info('sync collection %s to web' % entity.id)
if entity.type == Entity.PUZZLE:
web_message = web_message_builder.build_puzzle_message(Puzzle.objects.get(id=entity.id))
logger.info('sync puzzle %s to web' % entity.id)
if entity.type == Entity.EVALUATION:
web_message = web_message_builder.build_evaluation_message(Evaluation.objects.get(id=entity.id))
logger.info('sync evaluation %s to web' % entity.id)
if web_message is not None:
MessageSender.send_web(web_message)
def __init__(self):
self._runner_process = None
self._control_process = None
self._watchdog = ProcessWatchdog(config.Watchdog.CHECK_INTERVAL)
self.push_messages_receiver = PushMessageReceiver()
self.config_builder = ConfigurationBuilder(config.Engine.CONFIGURATION_BUILDER)
self.message_handler = MessageHandler(self)
self.message_sender = MessageSender()
self.message_router = MessageRouter(self.message_sender, self.message_handler.default_message_handler)
self.state = ManagerState.EMPTY
self._http_client = httpclient.HTTPClient()
self._alert_registered = False
self.obsi_id = getnode() # A unique identifier of this OBSI
self._engine_running = False
self._engine_running_lock = locks.Lock()
self._processing_graph_set = False
self._engine_config_builder = None
self._keep_alive_periodic_callback = None
self._avg_cpu = 0
self._avg_duration = 0
self._supported_elements_types = []
self._alert_messages_handler = None
self._log_messages_handler = None
def __init__(self, recipient_emails=()):
self._read_offers_ids_from_db()
self.message_sender = MessageSender(recipient_emails)
class Manager(object):
def __init__(self):
self._runner_process = None
self._control_process = None
self._watchdog = ProcessWatchdog(config.Watchdog.CHECK_INTERVAL)
self.push_messages_receiver = PushMessageReceiver()
self.config_builder = ConfigurationBuilder(config.Engine.CONFIGURATION_BUILDER)
self.message_handler = MessageHandler(self)
self.message_sender = MessageSender()
self.message_router = MessageRouter(self.message_sender, self.message_handler.default_message_handler)
self.state = ManagerState.EMPTY
self._http_client = httpclient.HTTPClient()
self._alert_registered = False
self.obsi_id = getnode() # A unique identifier of this OBSI
self._engine_running = False
self._engine_running_lock = locks.Lock()
self._processing_graph_set = False
self._engine_config_builder = None
self._keep_alive_periodic_callback = None
self._avg_cpu = 0
self._avg_duration = 0
self._supported_elements_types = []
self._alert_messages_handler = None
self._log_messages_handler = None
def start(self):
app_log.info("Starting components")
self.state = ManagerState.INITIALIZING
self._start_runner()
self._start_control()
self._start_watchdog()
self._start_push_messages_receiver()
self._start_configuration_builder()
self._start_message_router()
self._start_local_rest_server()
app_log.info("All components active")
self.state = ManagerState.INITIALIZED
self._start_sending_keep_alive()
self._send_hello_message()
self._start_io_loop()
def _start_runner(self):
app_log.info("Starting EE Runner on port {port}".format(port=config.Runner.Rest.PORT))
self._runner_process = _start_remote_rest_server(config.Runner.Rest.BIN, config.Runner.Rest.PORT,
config.Runner.Rest.DEBUG)
if self._runner_process.is_running() and self._rest_server_listening(config.Runner.Rest.BASE_URI):
app_log.info("EERunner REST Server running")
else:
app_log.error("EERunner REST Server not running")
self.exit(1)
if self._is_engine_supported(_get_full_uri(config.Runner.Rest.BASE_URI, config.Runner.Rest.Endpoints.ENGINES)):
app_log.info("{engine} supported by EE Runner.".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} is not supported by EE Runner".format(engine=config.Engine.NAME))
self.exit(1)
if self._set_engine(_get_full_uri(config.Runner.Rest.BASE_URI, config.Runner.Rest.Endpoints.ENGINES)):
app_log.info("{engine} set".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} not set by EERunner".format(engine=config.Engine.NAME))
self.exit(1)
if self._start_engine():
app_log.info("{engine} started".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} failed to start".format(engine=config.Engine.NAME))
self.exit(1)
self._engine_running = True
self._alert_registered = self._register_alert_uri()
app_log.info("Alert Registration status: {status}".format(status=self._alert_registered))
def exit(self, exit_code):
if self._runner_process:
while self._runner_process.is_running():
self._runner_process.kill()
if self._control_process:
while self._control_process.is_running():
self._control_process.kill()
exit(exit_code)
def _rest_server_listening(self, base_uri):
for _ in xrange(config.Manager.CONNECTION_RETRIES):
try:
self._http_client.fetch(base_uri)
return True
except httpclient.HTTPError:
return True
except KeyboardInterrupt:
raise
except socket.error:
time.sleep(config.Manager.INTERVAL_BETWEEN_CONNECTION_TRIES)
return False
def _is_engine_supported(self, uri):
engine_name = config.Engine.NAME
try:
response = self._http_client.fetch(uri)
return engine_name in json_decode(response.body)
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _set_engine(self, uri):
engine_name = config.Engine.NAME
try:
self._http_client.fetch(uri, method="POST", body=json_encode(engine_name))
return True
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _start_engine(self):
params = dict(processing_graph=config.Engine.BASE_EMPTY_CONFIG,
control_socket_type=config.Engine.CONTROL_SOCKET_TYPE,
control_socket_endpoint=config.Engine.CONTROL_SOCKET_ENDPOINT,
nthreads=config.Engine.NTHREADS,
push_messages_type=config.Engine.PUSH_MESSAGES_SOCKET_TYPE,
push_messages_endpoint=config.Engine.PUSH_MESSAGES_SOCKET_ENDPOINT,
push_messages_channel=config.Engine.PUSH_MESSAGES_CHANNEL)
uri = _get_full_uri(config.Runner.Rest.BASE_URI, config.Runner.Rest.Endpoints.START)
try:
self._http_client.fetch(uri, method="POST", body=json_encode(params))
return True
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _register_alert_uri(self):
uri = _get_full_uri(config.Runner.Rest.BASE_URI, config.Runner.Rest.Endpoints.REGISTER_ALERT_URL)
alert_uri = _get_full_uri(config.RestServer.BASE_URI, config.RestServer.Endpoints.RUNNER_ALERT)
try:
self._http_client.fetch(uri, method='POST', body=url_escape(alert_uri))
return True
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _start_control(self):
app_log.info("Starting EE Control on port {port}".format(port=config.Control.Rest.PORT))
self._control_process = _start_remote_rest_server(config.Control.Rest.BIN, config.Control.Rest.PORT,
config.Control.Rest.DEBUG)
if self._control_process.is_running() and self._rest_server_listening(config.Control.Rest.BASE_URI):
app_log.info("EEControl REST Server running")
else:
app_log.error("EEControl REST Server not running")
exit(1)
if self._is_engine_supported(
_get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.ENGINES)):
app_log.info("{engine} supported by EE Control".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} is not supported by EE Control".format(engine=config.Engine.NAME))
exit(1)
if self._set_engine(_get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.ENGINES)):
app_log.info("{engine} set by EE Control".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} not set by EE Control".format(engine=config.Engine.NAME))
exit(1)
if self._connect_control():
app_log.info("EE Control client connected to engine")
else:
app_log.error("EE Control client couldn't connect to engine")
exit(1)
def _connect_control(self):
params = dict(address=config.Control.SOCKET_ADDRESS, type=config.Control.SOCKET_TYPE)
uri = _get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.CONNECT)
try:
self._http_client.fetch(uri, method="POST", body=json_encode(params))
return True
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _start_watchdog(self):
app_log.info("Starting ProcessWatchdog")
self._watchdog.register_process(self._runner_process, self._process_died)
self._watchdog.register_process(self._control_process, self._process_died)
self._watchdog.start()
@gen.coroutine
def _process_died(self, process):
if process == self._runner_process:
app_log.error("EE Runner REST server has died")
with (yield self._engine_running_lock.acquire()):
self._engine_running = False
# TODO: add recovering logic or at least send error
elif process == self._control_process:
app_log.error("EE Control REST server has died")
# TODO: Add real handling
else:
app_log.error("Unknown process dies")
def _start_push_messages_receiver(self):
app_log.info("Starting PushMessagesReceiver and registering Alert handling")
url = None # this will force the message sender to use the URL based on the message type
send_alert_messages = functools.partial(self.message_sender.send_push_messages, messages.Alert, self.obsi_id,
url)
self._alert_messages_handler = PushMessageHandler(send_alert_messages,
config.PushMessages.Alert.BUFFER_SIZE,
config.PushMessages.Alert.BUFFER_TIMEOUT)
self.push_messages_receiver.register_message_handler('ALERT', self._alert_messages_handler.add)
self.push_messages_receiver.connect(config.PushMessages.SOCKET_ADDRESS,
config.PushMessages.SOCKET_FAMILY,
config.PushMessages.RETRY_INTERVAL)
def _start_configuration_builder(self):
app_log.info("Starting EE Configuration Builder")
try:
uri = _get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.SUPPORTED_ELEMENTS)
response = self._http_client.fetch(uri)
self._supported_elements_types = set(json_decode(response.body))
supported_blocks = set(self.config_builder.supported_blocks())
blocks_from_engine = set(self.config_builder.supported_blocks_from_supported_engine_elements_types(
self._supported_elements_types))
if supported_blocks != blocks_from_engine:
app_log.warning("There is a mismatched between supported blocks by OBSI "
"and supported blocks by engine")
except httpclient.HTTPError:
app_log.error("Unable to connect to EE control in order to get a list of supported elements types")
@gen.coroutine
def _start_message_router(self):
app_log.info("Starting MessageRouter")
self._register_messages_handler()
yield self.message_router.start()
def _register_messages_handler(self):
app_log.info("Registering handlers for messages")
for message, handler in self.message_handler.registered_message_handlers.iteritems():
self.message_router.register_message_handler(message, handler)
def _start_local_rest_server(self):
app_log.info("Starting local REST server on port {port}".format(port=config.RestServer.PORT))
rest_server.start(self)
def _start_sending_keep_alive(self):
self._keep_alive_periodic_callback = PeriodicCallback(self._send_keep_alive, config.KeepAlive.INTERVAL)
self._keep_alive_periodic_callback.start()
@gen.coroutine
def _send_keep_alive(self):
received = yield self.message_sender.send_message_ignore_response(messages.KeepAlive(dpid=self.obsi_id))
if not received:
app_log.error('KeepAlive message received an error response from OBC')
@gen.coroutine
def _send_hello_message(self):
app_log.info("Creating and sending Hello Message")
while True:
hello_message = messages.Hello(dpid=self.obsi_id, version=config.OPENBOX_VERSION,
capabilities=self.get_capabilities())
received = yield self.message_sender.send_message_ignore_response(hello_message)
if received:
break
else:
app_log.error("Hello message received an error response from OBC")
yield gen.sleep(config.Manager.INTERVAL_BETWEEN_CONNECTION_TRIES)
def get_capabilities(self):
proto_messages = []
if config.Engine.Capabilities.MODULE_INSTALLATION:
proto_messages.append(messages.AddCustomModuleRequest.__name__)
if config.Engine.Capabilities.MODULE_REMOVAL:
proto_messages.append(messages.RemoveCustomModuleRequest.__name__)
processing_blocks = self.config_builder.supported_blocks_from_supported_engine_elements_types(
self._supported_elements_types)
match_fields = self.config_builder.supported_match_fields()
complex_match = self.config_builder.supported_complex_match()
protocol_analyser_protocols = self.config_builder.supported_protocol_analyser_protocols()
return dict(proto_messages=proto_messages, processing_blocks=processing_blocks,
match_fields=match_fields, complex_match=complex_match,
protocol_analyser_protocols=protocol_analyser_protocols)
def _start_io_loop(self):
app_log.info("Starting the IOLoop")
IOLoop.current().start()
@gen.coroutine
def handle_runner_alert(self, errors):
with (yield self._engine_running_lock.acquire()):
self._engine_running = False
app_log.error("Engine stopped working: {errors}".format(errors=errors))
@gen.coroutine
def get_engine_global_stats(self):
client = httpclient.AsyncHTTPClient()
memory_uri = _get_full_uri(config.Runner.Rest.BASE_URI, config.Runner.Rest.Endpoints.MEMORY)
cpu_uri = _get_full_uri(config.Runner.Rest.BASE_URI, config.Runner.Rest.Endpoints.CPU)
uptime_uri = _get_full_uri(config.Runner.Rest.BASE_URI, config.Runner.Rest.Endpoints.UPTIME)
memory, cpu, uptime = yield [client.fetch(memory_uri), client.fetch(cpu_uri), client.fetch(uptime_uri)]
memory, cpu, uptime = json_decode(memory.body), json_decode(cpu.body), json_decode(uptime.body)
cpu_count = cpu['cpu_count']
current_load = cpu['cpu_percent'] / 100.0 / cpu_count
duration = cpu['measurement_time']
self._avg_cpu = (current_load * duration + self._avg_cpu * self._avg_duration) / (duration + self._avg_duration)
self._avg_duration += duration
stats = dict(memory_rss=memory['rss'], memory_vms=memory['vms'], memory_percent=memory['percent'],
cpus=cpu_count, current_load=current_load, avg_load=self._avg_cpu,
avg_minutes=self._avg_duration / 60.0, uptime=uptime['uptime'])
raise gen.Return(stats)
@gen.coroutine
def reset_engine_global_stats(self):
self._avg_cpu = 0
self._avg_duration = 0
@gen.coroutine
def read_block_value(self, block_name, handler_name):
with (yield self._engine_running_lock.acquire()):
if not self._engine_running:
raise EngineNotRunningError()
if not self._processing_graph_set or self._engine_config_builder is None:
raise ProcessingGraphNotSetError()
else:
(engine_element_name,
engine_handler_name,
transform_function) = self._engine_config_builder.translate_block_read_handler(block_name,
handler_name)
element = url_escape(engine_element_name)
handler = url_escape(engine_handler_name)
uri = _get_full_uri(config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.HANDLER_PATTERN.format(element=element,
handler=handler))
client = httpclient.AsyncHTTPClient()
response = yield client.fetch(uri)
raise gen.Return(transform_function(json_decode(response.body)))
@gen.coroutine
def write_block_value(self, block_name, handler_name, value):
with (yield self._engine_running_lock):
if not self._engine_running:
raise EngineNotRunningError()
if not self._processing_graph_set or self._engine_config_builder is None:
raise ProcessingGraphNotSetError()
else:
(engine_element_name,
engine_handler_name,
transform_function) = self._engine_config_builder.translate_block_write_handler(block_name,
handler_name)
uri = _get_full_uri(config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.HANDLER_PATTERN.format(element=engine_element_name,
handler=engine_handler_name))
body = json_encode(transform_function(value))
client = httpclient.AsyncHTTPClient()
yield client.fetch(uri, method='POST', body=body)
raise gen.Return(True)
@gen.coroutine
def set_processing_graph(self, required_modules, blocks, connections):
processing_graph = dict(requirements=required_modules, blocks=blocks, connections=connections)
self._engine_config_builder = self.config_builder.engine_config_builder_from_dict(processing_graph,
config.Engine.REQUIREMENTS)
engine_config = self._engine_config_builder.to_engine_config()
app_log.debug("Setting processing graph to:\n%s" % engine_config)
client = httpclient.AsyncHTTPClient()
uri = _get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.CONFIG)
yield client.fetch(uri, method='POST', body=json_encode(engine_config))
# make sure we are stable in the new config
# the config is in the same URI but with a GET method
response = yield client.fetch(uri)
new_config = json_decode(response.body)
if engine_config in new_config:
self._processing_graph_set = True
else:
app_log.error("Unable to set processing graph")
@gen.coroutine
def set_parameters(self, params):
config.KeepAlive.INTERVAL = params.get('keepalive_interval', config.KeepAlive.INTERVAL)
config.PushMessages.Alert.BUFFER_SIZE = params.get('alert_messages_buffer_size',
config.PushMessages.Alert.BUFFER_SIZE)
config.PushMessages.Alert.BUFFER_TIMEOUT = params.get('alert_messages_buffer_timeout',
config.PushMessages.Alert.BUFFER_TIMEOUT * 1000.0) / 1000.0
config.PushMessages.Log.BUFFER_SIZE = params.get('log_messages_buffer_size',
config.PushMessages.Log.BUFFER_SIZE)
config.PushMessages.Log.BUFFER_TIMEOUT = params.get('log_messages_buffer_timeout',
config.PushMessages.Log.BUFFER_TIMEOUT * 1000.0) / 1000.0
old_server, old_port = config.PushMessages.Log.SERVER_ADDRESS, config.PushMessages.Log.SERVER_PORT
config.PushMessages.Log.SERVER_ADDRESS = params.get('log_server_address',
config.PushMessages.Log.SERVER_ADDRESS)
config.PushMessages.Log.SERVER_PORT = params.get('log_server_port', config.PushMessages.Log.SERVER_PORT)
new_server, new_port = config.PushMessages.Log.SERVER_ADDRESS, config.PushMessages.Log.SERVER_PORT
config.PushMessages.Log._SERVER_CHANGED = new_server != old_server or new_port != old_port
self._update_components()
def _update_components(self):
# update keepalive
if self._keep_alive_periodic_callback:
self._keep_alive_periodic_callback.stop()
self._start_sending_keep_alive()
# update alert push messages
if self._alert_messages_handler:
self._alert_messages_handler.buffer_size = config.PushMessages.Alert.BUFFER_SIZE
self._alert_messages_handler.buffer_timeout = config.PushMessages.Alert.BUFFER_TIMEOUT
# update log push messages
if config.PushMessages.Log.SERVER_PORT and config.PushMessages.Log.SERVER_PORT:
url = "http://{host}:{port}/message/Log".format(host=config.PushMessages.Log.SERVER_ADDRESS,
port=config.PushMessages.Log.SERVER_PORT)
send_log_messages = functools.partial(self.message_sender.send_push_messages, messages.Log,
self.obsi_id, url)
if config.PushMessages.Log._SERVER_CHANGED:
# better close it and make it start over
if self._log_messages_handler:
self._log_messages_handler.close()
self.push_messages_receiver.unregister_message_handler('LOG')
self._log_messages_handler = PushMessageHandler(send_log_messages,
config.PushMessages.Log.BUFFER_SIZE,
config.PushMessages.Log.BUFFER_TIMEOUT)
self.push_messages_receiver.register_message_handler('LOG', self._log_messages_handler.add)
if self._log_messages_handler:
self._log_messages_handler.buffer_size = config.PushMessages.Log.BUFFER_SIZE
self._log_messages_handler.buffer_timeout = config.PushMessages.Log.BUFFER_TIMEOUT
def get_parameters(self, parameters):
result = dict(keepalive_interval=int(config.KeepAlive.INTERVAL),
alert_messages_buffer_size=config.PushMessages.Alert.BUFFER_SIZE,
alert_messages_buffer_timeout=int(config.PushMessages.Alert.BUFFER_TIMEOUT * 1000),
log_messages_buffer_size=config.PushMessages.Log.BUFFER_SIZE,
log_messages_buffer_timeout=int(config.PushMessages.Log.BUFFER_TIMEOUT * 1000),
log_server_address=config.PushMessages.Log.SERVER_ADDRESS,
log_server_port=config.PushMessages.Log.SERVER_PORT)
if not parameters:
# an empty list means they want all of them
return result
else:
try:
partial = {}
for parameter in parameters:
partial[parameter] = result[parameter]
return partial
except KeyError as e:
raise UnknownRequestedParameter("Request unknown parameter {parm}".format(parm=e.message))
@gen.coroutine
def add_custom_module(self, name, content, content_type, encoding, translation):
if content:
if encoding.lower() != 'base64':
raise UnsupportedModuleDataEncoding("Unknown encoding '{enc}' for module content".format(enc=encoding))
yield self._install_package(name, content, encoding.lower())
yield self._update_running_config_with_package(name)
yield self._update_supported_elements()
self.config_builder.add_custom_module(name, translation)
@gen.coroutine
def _install_package(self, name, content, encoding):
package = dict(name=name, data=content, encoding=encoding)
client = httpclient.AsyncHTTPClient()
uri = _get_full_uri(config.Runner.Rest.BASE_URI, config.Runner.Rest.Endpoints.INSTALL)
yield client.fetch(uri, method='POST', body=json_encode(package))
@gen.coroutine
def _update_running_config_with_package(self, name):
client = httpclient.AsyncHTTPClient()
uri = _get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.LOADED_PACKAGES)
yield client.fetch(uri, method='POST', body=json_encode(name))
@gen.coroutine
def _update_supported_elements(self):
client = httpclient.AsyncHTTPClient()
uri = _get_full_uri(config.Control.Rest.BASE_URI, config.Control.Rest.Endpoints.SUPPORTED_ELEMENTS)
response = yield client.fetch(uri)
self._supported_elements_types = set(json_decode(response.body))
class Manager(object):
def __init__(self):
self._runner_process = None
self._control_process = None
self._watchdog = ProcessWatchdog(config.Watchdog.CHECK_INTERVAL)
self.push_messages_receiver = PushMessageReceiver()
self.config_builder = ConfigurationBuilder(
config.Engine.CONFIGURATION_BUILDER)
self.message_handler = MessageHandler(self)
self.message_sender = MessageSender()
self.message_router = MessageRouter(
self.message_sender, self.message_handler.default_message_handler)
self.state = ManagerState.EMPTY
self._http_client = httpclient.HTTPClient()
self._alert_registered = False
self.obsi_id = getnode() # A unique identifier of this OBSI
self._engine_running = False
self._engine_running_lock = locks.Lock()
self._processing_graph_set = False
self._engine_config_builder = None
self._keep_alive_periodic_callback = None
self._avg_cpu = 0
self._avg_duration = 0
self._supported_elements_types = []
self._alert_messages_handler = None
self._log_messages_handler = None
def start(self):
app_log.info("Starting components")
self.state = ManagerState.INITIALIZING
self._start_runner()
self._start_control()
self._start_watchdog()
self._start_push_messages_receiver()
self._start_configuration_builder()
self._start_message_router()
self._start_local_rest_server()
app_log.info("All components active")
self.state = ManagerState.INITIALIZED
self._start_sending_keep_alive()
self._send_hello_message()
self._start_io_loop()
def _start_runner(self):
app_log.info("Starting EE Runner on port {port}".format(
port=config.Runner.Rest.PORT))
self._runner_process = _start_remote_rest_server(
config.Runner.Rest.BIN, config.Runner.Rest.PORT,
config.Runner.Rest.DEBUG)
if self._runner_process.is_running() and self._rest_server_listening(
config.Runner.Rest.BASE_URI):
app_log.info("EERunner REST Server running")
else:
app_log.error("EERunner REST Server not running")
self.exit(1)
if self._is_engine_supported(
_get_full_uri(config.Runner.Rest.BASE_URI,
config.Runner.Rest.Endpoints.ENGINES)):
app_log.info("{engine} supported by EE Runner.".format(
engine=config.Engine.NAME))
else:
app_log.error("{engine} is not supported by EE Runner".format(
engine=config.Engine.NAME))
self.exit(1)
if self._set_engine(
_get_full_uri(config.Runner.Rest.BASE_URI,
config.Runner.Rest.Endpoints.ENGINES)):
app_log.info("{engine} set".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} not set by EERunner".format(
engine=config.Engine.NAME))
self.exit(1)
if self._start_engine():
app_log.info("{engine} started".format(engine=config.Engine.NAME))
else:
app_log.error(
"{engine} failed to start".format(engine=config.Engine.NAME))
self.exit(1)
self._engine_running = True
self._alert_registered = self._register_alert_uri()
app_log.info("Alert Registration status: {status}".format(
status=self._alert_registered))
def exit(self, exit_code):
if self._runner_process:
while self._runner_process.is_running():
self._runner_process.kill()
if self._control_process:
while self._control_process.is_running():
self._control_process.kill()
exit(exit_code)
def _rest_server_listening(self, base_uri):
for _ in xrange(config.Manager.CONNECTION_RETRIES):
try:
self._http_client.fetch(base_uri)
return True
except httpclient.HTTPError:
return True
except KeyboardInterrupt:
raise
except socket.error:
time.sleep(config.Manager.INTERVAL_BETWEEN_CONNECTION_TRIES)
return False
def _is_engine_supported(self, uri):
engine_name = config.Engine.NAME
try:
response = self._http_client.fetch(uri)
return engine_name in json_decode(response.body)
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _set_engine(self, uri):
engine_name = config.Engine.NAME
try:
self._http_client.fetch(uri,
method="POST",
body=json_encode(engine_name))
return True
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _start_engine(self):
params = dict(
processing_graph=config.Engine.BASE_EMPTY_CONFIG,
control_socket_type=config.Engine.CONTROL_SOCKET_TYPE,
control_socket_endpoint=config.Engine.CONTROL_SOCKET_ENDPOINT,
nthreads=config.Engine.NTHREADS,
push_messages_type=config.Engine.PUSH_MESSAGES_SOCKET_TYPE,
push_messages_endpoint=config.Engine.PUSH_MESSAGES_SOCKET_ENDPOINT,
push_messages_channel=config.Engine.PUSH_MESSAGES_CHANNEL)
uri = _get_full_uri(config.Runner.Rest.BASE_URI,
config.Runner.Rest.Endpoints.START)
try:
self._http_client.fetch(uri,
method="POST",
body=json_encode(params))
return True
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _register_alert_uri(self):
uri = _get_full_uri(config.Runner.Rest.BASE_URI,
config.Runner.Rest.Endpoints.REGISTER_ALERT_URL)
alert_uri = _get_full_uri(config.RestServer.BASE_URI,
config.RestServer.Endpoints.RUNNER_ALERT)
try:
self._http_client.fetch(uri,
method='POST',
body=url_escape(alert_uri))
return True
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _start_control(self):
app_log.info("Starting EE Control on port {port}".format(
port=config.Control.Rest.PORT))
self._control_process = _start_remote_rest_server(
config.Control.Rest.BIN, config.Control.Rest.PORT,
config.Control.Rest.DEBUG)
if self._control_process.is_running() and self._rest_server_listening(
config.Control.Rest.BASE_URI):
app_log.info("EEControl REST Server running")
else:
app_log.error("EEControl REST Server not running")
exit(1)
if self._is_engine_supported(
_get_full_uri(config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.ENGINES)):
app_log.info("{engine} supported by EE Control".format(
engine=config.Engine.NAME))
else:
app_log.error("{engine} is not supported by EE Control".format(
engine=config.Engine.NAME))
exit(1)
if self._set_engine(
_get_full_uri(config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.ENGINES)):
app_log.info(
"{engine} set by EE Control".format(engine=config.Engine.NAME))
else:
app_log.error("{engine} not set by EE Control".format(
engine=config.Engine.NAME))
exit(1)
if self._connect_control():
app_log.info("EE Control client connected to engine")
else:
app_log.error("EE Control client couldn't connect to engine")
exit(1)
def _connect_control(self):
params = dict(address=config.Control.SOCKET_ADDRESS,
type=config.Control.SOCKET_TYPE)
uri = _get_full_uri(config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.CONNECT)
try:
self._http_client.fetch(uri,
method="POST",
body=json_encode(params))
return True
except httpclient.HTTPError as e:
app_log.error(e.response)
return False
def _start_watchdog(self):
app_log.info("Starting ProcessWatchdog")
self._watchdog.register_process(self._runner_process,
self._process_died)
self._watchdog.register_process(self._control_process,
self._process_died)
self._watchdog.start()
@gen.coroutine
def _process_died(self, process):
if process == self._runner_process:
app_log.error("EE Runner REST server has died")
with (yield self._engine_running_lock.acquire()):
self._engine_running = False
# TODO: add recovering logic or at least send error
elif process == self._control_process:
app_log.error("EE Control REST server has died")
# TODO: Add real handling
else:
app_log.error("Unknown process dies")
def _start_push_messages_receiver(self):
app_log.info(
"Starting PushMessagesReceiver and registering Alert handling")
url = None # this will force the message sender to use the URL based on the message type
send_alert_messages = functools.partial(
self.message_sender.send_push_messages, messages.Alert,
self.obsi_id, url)
self._alert_messages_handler = PushMessageHandler(
send_alert_messages, config.PushMessages.Alert.BUFFER_SIZE,
config.PushMessages.Alert.BUFFER_TIMEOUT)
self.push_messages_receiver.register_message_handler(
'ALERT', self._alert_messages_handler.add)
self.push_messages_receiver.connect(config.PushMessages.SOCKET_ADDRESS,
config.PushMessages.SOCKET_FAMILY,
config.PushMessages.RETRY_INTERVAL)
def _start_configuration_builder(self):
app_log.info("Starting EE Configuration Builder")
try:
uri = _get_full_uri(
config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.SUPPORTED_ELEMENTS)
response = self._http_client.fetch(uri)
self._supported_elements_types = set(json_decode(response.body))
supported_blocks = set(self.config_builder.supported_blocks())
blocks_from_engine = set(
self.config_builder.
supported_blocks_from_supported_engine_elements_types(
self._supported_elements_types))
if supported_blocks != blocks_from_engine:
app_log.warning(
"There is a mismatched between supported blocks by OBSI "
"and supported blocks by engine")
except httpclient.HTTPError:
app_log.error(
"Unable to connect to EE control in order to get a list of supported elements types"
)
@gen.coroutine
def _start_message_router(self):
app_log.info("Starting MessageRouter")
self._register_messages_handler()
yield self.message_router.start()
def _register_messages_handler(self):
app_log.info("Registering handlers for messages")
for message, handler in self.message_handler.registered_message_handlers.iteritems(
):
self.message_router.register_message_handler(message, handler)
def _start_local_rest_server(self):
app_log.info("Starting local REST server on port {port}".format(
port=config.RestServer.PORT))
rest_server.start(self)
def _start_sending_keep_alive(self):
self._keep_alive_periodic_callback = PeriodicCallback(
self._send_keep_alive, config.KeepAlive.INTERVAL)
self._keep_alive_periodic_callback.start()
@gen.coroutine
def _send_keep_alive(self):
received = yield self.message_sender.send_message_ignore_response(
messages.KeepAlive(dpid=self.obsi_id))
if not received:
app_log.error(
'KeepAlive message received an error response from OBC')
@gen.coroutine
def _send_hello_message(self):
app_log.info("Creating and sending Hello Message")
while True:
hello_message = messages.Hello(
dpid=self.obsi_id,
version=config.OPENBOX_VERSION,
capabilities=self.get_capabilities())
received = yield self.message_sender.send_message_ignore_response(
hello_message)
if received:
break
else:
app_log.error(
"Hello message received an error response from OBC")
yield gen.sleep(
config.Manager.INTERVAL_BETWEEN_CONNECTION_TRIES)
def get_capabilities(self):
proto_messages = []
if config.Engine.Capabilities.MODULE_INSTALLATION:
proto_messages.append(messages.AddCustomModuleRequest.__name__)
if config.Engine.Capabilities.MODULE_REMOVAL:
proto_messages.append(messages.RemoveCustomModuleRequest.__name__)
processing_blocks = self.config_builder.supported_blocks_from_supported_engine_elements_types(
self._supported_elements_types)
match_fields = self.config_builder.supported_match_fields()
complex_match = self.config_builder.supported_complex_match()
protocol_analyser_protocols = self.config_builder.supported_protocol_analyser_protocols(
)
return dict(proto_messages=proto_messages,
processing_blocks=processing_blocks,
match_fields=match_fields,
complex_match=complex_match,
protocol_analyser_protocols=protocol_analyser_protocols)
def _start_io_loop(self):
app_log.info("Starting the IOLoop")
IOLoop.current().start()
@gen.coroutine
def handle_runner_alert(self, errors):
with (yield self._engine_running_lock.acquire()):
self._engine_running = False
app_log.error("Engine stopped working: {errors}".format(errors=errors))
@gen.coroutine
def get_engine_global_stats(self):
client = httpclient.AsyncHTTPClient(
request_timeout=config.Manager.REQUEST_TIMEOUT)
memory_uri = _get_full_uri(config.Runner.Rest.BASE_URI,
config.Runner.Rest.Endpoints.MEMORY)
cpu_uri = _get_full_uri(config.Runner.Rest.BASE_URI,
config.Runner.Rest.Endpoints.CPU)
uptime_uri = _get_full_uri(config.Runner.Rest.BASE_URI,
config.Runner.Rest.Endpoints.UPTIME)
memory, cpu, uptime = yield [
client.fetch(memory_uri),
client.fetch(cpu_uri),
client.fetch(uptime_uri)
]
memory, cpu, uptime = json_decode(memory.body), json_decode(
cpu.body), json_decode(uptime.body)
cpu_count = cpu['cpu_count']
current_load = cpu['cpu_percent'] / 100.0 / cpu_count
duration = cpu['measurement_time']
self._avg_cpu = (current_load * duration + self._avg_cpu *
self._avg_duration) / (duration + self._avg_duration)
self._avg_duration += duration
stats = dict(memory_rss=memory['rss'],
memory_vms=memory['vms'],
memory_percent=memory['percent'],
cpus=cpu_count,
current_load=current_load,
avg_load=self._avg_cpu,
avg_minutes=self._avg_duration / 60.0,
uptime=uptime['uptime'])
raise gen.Return(stats)
@gen.coroutine
def reset_engine_global_stats(self):
self._avg_cpu = 0
self._avg_duration = 0
@gen.coroutine
def read_block_value(self, block_name, handler_name):
with (yield self._engine_running_lock.acquire()):
if not self._engine_running:
raise EngineNotRunningError()
if not self._processing_graph_set or self._engine_config_builder is None:
raise ProcessingGraphNotSetError()
else:
(engine_element_name, engine_handler_name, transform_function
) = self._engine_config_builder.translate_block_read_handler(
block_name, handler_name)
element = url_escape(engine_element_name)
handler = url_escape(engine_handler_name)
uri = _get_full_uri(
config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.HANDLER_PATTERN.format(
element=element, handler=handler))
client = httpclient.AsyncHTTPClient(
request_timeout=config.Manager.REQUEST_TIMEOUT)
response = yield client.fetch(uri)
raise gen.Return(transform_function(json_decode(response.body)))
@gen.coroutine
def write_block_value(self, block_name, handler_name, value):
with (yield self._engine_running_lock):
if not self._engine_running:
raise EngineNotRunningError()
if not self._processing_graph_set or self._engine_config_builder is None:
raise ProcessingGraphNotSetError()
else:
(engine_element_name, engine_handler_name, transform_function
) = self._engine_config_builder.translate_block_write_handler(
block_name, handler_name)
uri = _get_full_uri(
config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.HANDLER_PATTERN.format(
element=engine_element_name, handler=engine_handler_name))
body = json_encode(transform_function(value))
client = httpclient.AsyncHTTPClient(
request_timeout=config.Manager.REQUEST_TIMEOUT)
yield client.fetch(uri, method='POST', body=body)
raise gen.Return(True)
@gen.coroutine
def set_processing_graph(self, required_modules, blocks, connections):
processing_graph = dict(requirements=required_modules,
blocks=blocks,
connections=connections)
self._engine_config_builder = self.config_builder.engine_config_builder_from_dict(
processing_graph, config.Engine.REQUIREMENTS)
engine_config = self._engine_config_builder.to_engine_config()
app_log.debug("Setting processing graph to:\n%s" % engine_config)
client = httpclient.AsyncHTTPClient(
request_timeout=config.Manager.REQUEST_TIMEOUT)
uri = _get_full_uri(config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.CONFIG)
yield client.fetch(uri, method='POST', body=json_encode(engine_config))
# make sure we are stable in the new config
# the config is in the same URI but with a GET method
response = yield client.fetch(uri)
new_config = json_decode(response.body)
if engine_config in new_config:
self._processing_graph_set = True
else:
app_log.error("Unable to set processing graph")
@gen.coroutine
def set_parameters(self, params):
config.KeepAlive.INTERVAL = params.get('keepalive_interval',
config.KeepAlive.INTERVAL)
config.PushMessages.Alert.BUFFER_SIZE = params.get(
'alert_messages_buffer_size',
config.PushMessages.Alert.BUFFER_SIZE)
config.PushMessages.Alert.BUFFER_TIMEOUT = params.get(
'alert_messages_buffer_timeout',
config.PushMessages.Alert.BUFFER_TIMEOUT * 1000.0) / 1000.0
config.PushMessages.Log.BUFFER_SIZE = params.get(
'log_messages_buffer_size', config.PushMessages.Log.BUFFER_SIZE)
config.PushMessages.Log.BUFFER_TIMEOUT = params.get(
'log_messages_buffer_timeout',
config.PushMessages.Log.BUFFER_TIMEOUT * 1000.0) / 1000.0
old_server, old_port = config.PushMessages.Log.SERVER_ADDRESS, config.PushMessages.Log.SERVER_PORT
config.PushMessages.Log.SERVER_ADDRESS = params.get(
'log_server_address', config.PushMessages.Log.SERVER_ADDRESS)
config.PushMessages.Log.SERVER_PORT = params.get(
'log_server_port', config.PushMessages.Log.SERVER_PORT)
new_server, new_port = config.PushMessages.Log.SERVER_ADDRESS, config.PushMessages.Log.SERVER_PORT
config.PushMessages.Log._SERVER_CHANGED = new_server != old_server or new_port != old_port
self._update_components()
def _update_components(self):
# update keepalive
if self._keep_alive_periodic_callback:
self._keep_alive_periodic_callback.stop()
self._start_sending_keep_alive()
# update alert push messages
if self._alert_messages_handler:
self._alert_messages_handler.buffer_size = config.PushMessages.Alert.BUFFER_SIZE
self._alert_messages_handler.buffer_timeout = config.PushMessages.Alert.BUFFER_TIMEOUT
# update log push messages
if config.PushMessages.Log.SERVER_PORT and config.PushMessages.Log.SERVER_PORT:
url = "http://{host}:{port}/message/Log".format(
host=config.PushMessages.Log.SERVER_ADDRESS,
port=config.PushMessages.Log.SERVER_PORT)
send_log_messages = functools.partial(
self.message_sender.send_push_messages, messages.Log,
self.obsi_id, url)
if config.PushMessages.Log._SERVER_CHANGED:
# better close it and make it start over
if self._log_messages_handler:
self._log_messages_handler.close()
self.push_messages_receiver.unregister_message_handler('LOG')
self._log_messages_handler = PushMessageHandler(
send_log_messages, config.PushMessages.Log.BUFFER_SIZE,
config.PushMessages.Log.BUFFER_TIMEOUT)
self.push_messages_receiver.register_message_handler(
'LOG', self._log_messages_handler.add)
if self._log_messages_handler:
self._log_messages_handler.buffer_size = config.PushMessages.Log.BUFFER_SIZE
self._log_messages_handler.buffer_timeout = config.PushMessages.Log.BUFFER_TIMEOUT
def get_parameters(self, parameters):
result = dict(
keepalive_interval=int(config.KeepAlive.INTERVAL),
alert_messages_buffer_size=config.PushMessages.Alert.BUFFER_SIZE,
alert_messages_buffer_timeout=int(
config.PushMessages.Alert.BUFFER_TIMEOUT * 1000),
log_messages_buffer_size=config.PushMessages.Log.BUFFER_SIZE,
log_messages_buffer_timeout=int(
config.PushMessages.Log.BUFFER_TIMEOUT * 1000),
log_server_address=config.PushMessages.Log.SERVER_ADDRESS,
log_server_port=config.PushMessages.Log.SERVER_PORT)
if not parameters:
# an empty list means they want all of them
return result
else:
try:
partial = {}
for parameter in parameters:
partial[parameter] = result[parameter]
return partial
except KeyError as e:
raise UnknownRequestedParameter(
"Request unknown parameter {parm}".format(parm=e.message))
@gen.coroutine
def add_custom_module(self, name, content, content_type, encoding,
translation):
if content:
if encoding.lower() != 'base64':
raise UnsupportedModuleDataEncoding(
"Unknown encoding '{enc}' for module content".format(
enc=encoding))
yield self._install_package(name, content, encoding.lower())
yield self._update_running_config_with_package(name)
yield self._update_supported_elements()
self.config_builder.add_custom_module(name, translation)
@gen.coroutine
def _install_package(self, name, content, encoding):
package = dict(name=name, data=content, encoding=encoding)
client = httpclient.AsyncHTTPClient(
request_timeout=config.Manager.REQUEST_TIMEOUT)
uri = _get_full_uri(config.Runner.Rest.BASE_URI,
config.Runner.Rest.Endpoints.INSTALL)
yield client.fetch(uri, method='POST', body=json_encode(package))
@gen.coroutine
def _update_running_config_with_package(self, name):
client = httpclient.AsyncHTTPClient(
request_timeout=config.Manager.REQUEST_TIMEOUT)
uri = _get_full_uri(config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.LOADED_PACKAGES)
yield client.fetch(uri, method='POST', body=json_encode(name))
@gen.coroutine
def _update_supported_elements(self):
client = httpclient.AsyncHTTPClient(
request_timeout=config.Manager.REQUEST_TIMEOUT)
uri = _get_full_uri(config.Control.Rest.BASE_URI,
config.Control.Rest.Endpoints.SUPPORTED_ELEMENTS)
response = yield client.fetch(uri)
self._supported_elements_types = set(json_decode(response.body))
def sync_weixin():
#query weixin messages
weixin_msgs = Weixin.objects.filter(sync_timestamp2__range=(datetime.datetime.now(), datetime.datetime.now()+datetime.timedelta(minutes=5)), status2=1)
for msg in weixin_msgs:
logger.info('sync msg %s to weixin' % msg.id)
MessageSender.send_weixin(weixin_message_builder.build_weixin_message(msg))
def handle_new_client_request(self, event):
# At first we want to receive information to properly connect as new predecessor after sending init_data
message_size = event.data['connection'].recv(8)
message_size = int.from_bytes(message_size, byteorder='big')
message = b''
while len(message) < message_size:
packet = event.data['connection'].recv(message_size - len(message))
if not packet:
return None
message += packet
client_request = json.loads(message.decode('utf-8'))
first_client = not self.next_hop_info
# When we detect a new client connecting we want to;
# 1.Send him the initial data over the connection we already established
# 2.Connect to him as a predecessor
# Gather the initial board state (only the coloured spots)
marked_spots = [(x, y) for x in range(len(self.board_state))
for y in range(len(self.board_state[x]))
if self.board_state[x][y] == 1]
# If we have next hop information we send it, if we do not have we are the first client so we send our
# information as the first hop information
next_hop = (helpers.get_self_ip_address(),
config.getint(
'NewPredecessorListener',
'Port')) if first_client else self.next_hop_info
# If we are the first client next next hop is None
response = events.NewClientResponseEvent(next_hop,
self.next_next_hop_info,
marked_spots,
self.critical_section)
message = helpers.event_to_message(response)
message_size = (len(message)).to_bytes(8, byteorder='big')
event.data['connection'].send(message_size)
event.data['connection'].send(message)
try:
message = event.data['connection'].recv(8)
except Exception as ex:
if message == b'':
# Only case when we have a succesfull read of 0 bytes is when other socket shutdowns normally
pass
else:
logger.error(ex)
#Client did not initializ correctly so we abort the process
return
# If we are not the first client we have to update our next next hop to our previous next hop
if not first_client:
self.next_next_hop_info = self.next_hop_info
else:
# If we are the first client we update our next next hop info to self address
self.next_next_hop_info = (helpers.get_self_ip_address(),
config.getint('NewPredecessorListener',
'Port'))
# We stop current message sender if it exists
if self.message_sender:
self.message_sender.stop()
# We update our next hop info with the newest client request
self.next_hop_info = client_request['data']['address']
ip, port = self.next_hop_info
# We establish a new connection and a new message sender
connection = socket.create_connection((ip, port), 100)
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, connection)
self.message_sender.start()
if first_client and self.last_token != None:
# If we are the first client we start passing of the token
self.sending_queue.put(events.TokenPassEvent(self.last_token))
from message_sender import MessageSender
if __name__ == '__main__':
ms = MessageSender('config.ini')
ms.test_message()
# ms.send_messages()
def __init__(self,
event_queue,
paint_queue,
time_offset,
init_data=None,
init_connection=None):
super(ModelThread, self).__init__()
# Queues
self.event_queue = event_queue
self.paint_queue = paint_queue
#Event handling
self.handlers = {}
self.initialize_handlers()
# Unique uuid identifying clients in the network
self.uuid = uuid.uuid4().hex
# Flag indicating weather we want to enter critical section when the token comes
self.want_to_enter_critical_section = False
# Information about critical section like the timestamp and client uuid which is in the section
self.critical_section = None
# Time offset between ntp server and local time
self.time_offset = time_offset
# Value of the last token we have received
self.last_token = None
# Initial board state
self.board_state = [[
0 for _ in range(config.getint('Tkinter', 'CanvasY'))
] for _ in range(config.getint('Tkinter', 'CanvasX'))]
# If we are the first client
if not init_data:
self.next_hop_info = None
self.next_next_hop_info = None
self.sending_queue = None
self.message_sender = None
self.predecessor = None
self.last_token = 0
else:
self.next_hop_info = init_data['next_hop']
if not init_data['next_next_hop']:
# If there is no next_next_hop init data in the response we are the second client so we set
# next next hop as our address
self.next_next_hop_info = (helpers.get_self_ip_address(),
config.getint(
'NewPredecessorListener',
'Port'))
else:
# If there are more thant two clients we set the value from the response
self.next_next_hop_info = init_data['next_next_hop']
# Address of our predecessor
self.predecessor = init_connection.getsockname()
# We initialize connection to our next hop and we start sending queue
ip, port = init_data['next_hop']
s = socket.create_connection((ip, port))
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, s)
self.message_sender.start()
self.initialize_board(init_data['board_state'])
if init_data['critical_section_state']:
self.critical_section = init_data['critical_section_state']
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_CLOSED})
# We signal that client has initialized properly
init_connection.shutdown(1)
init_connection.close()
# We start a dummy message sender event which will create dummy messages to detect connection breaks
self.dummy_message_sender = DummyMessageSender(self.event_queue,
self.uuid)
self.dummy_message_sender.start()
class ModelThread(threading.Thread):
def __init__(self,
event_queue,
paint_queue,
time_offset,
init_data=None,
init_connection=None):
super(ModelThread, self).__init__()
# Queues
self.event_queue = event_queue
self.paint_queue = paint_queue
#Event handling
self.handlers = {}
self.initialize_handlers()
# Unique uuid identifying clients in the network
self.uuid = uuid.uuid4().hex
# Flag indicating weather we want to enter critical section when the token comes
self.want_to_enter_critical_section = False
# Information about critical section like the timestamp and client uuid which is in the section
self.critical_section = None
# Time offset between ntp server and local time
self.time_offset = time_offset
# Value of the last token we have received
self.last_token = None
# Initial board state
self.board_state = [[
0 for _ in range(config.getint('Tkinter', 'CanvasY'))
] for _ in range(config.getint('Tkinter', 'CanvasX'))]
# If we are the first client
if not init_data:
self.next_hop_info = None
self.next_next_hop_info = None
self.sending_queue = None
self.message_sender = None
self.predecessor = None
self.last_token = 0
else:
self.next_hop_info = init_data['next_hop']
if not init_data['next_next_hop']:
# If there is no next_next_hop init data in the response we are the second client so we set
# next next hop as our address
self.next_next_hop_info = (helpers.get_self_ip_address(),
config.getint(
'NewPredecessorListener',
'Port'))
else:
# If there are more thant two clients we set the value from the response
self.next_next_hop_info = init_data['next_next_hop']
# Address of our predecessor
self.predecessor = init_connection.getsockname()
# We initialize connection to our next hop and we start sending queue
ip, port = init_data['next_hop']
s = socket.create_connection((ip, port))
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, s)
self.message_sender.start()
self.initialize_board(init_data['board_state'])
if init_data['critical_section_state']:
self.critical_section = init_data['critical_section_state']
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_CLOSED})
# We signal that client has initialized properly
init_connection.shutdown(1)
init_connection.close()
# We start a dummy message sender event which will create dummy messages to detect connection breaks
self.dummy_message_sender = DummyMessageSender(self.event_queue,
self.uuid)
self.dummy_message_sender.start()
def run(self):
while True:
(e) = self.event_queue.get()
handler_function = self.handlers[type(e).__name__]
handler_function(e)
def initialize_board(self, board_state):
for counter in (len(board_state)):
x_coord, y_coord = board_state[counter]
try:
self.board_state[x_coord][y_coord] = 1
except IndexError:
return
self.paint_queue.put({
'type': DrawingQueueEvent.DRAWING,
'data': (board_state, 1)
})
def initialize_handlers(self):
# Inner Handlers
self.handlers[
'InnerNewClientRequestEvent'] = self.handle_new_client_request
self.handlers[
'InnerNewPredecessorRequestEvent'] = self.handle_new_predecessor_request_event
self.handlers[
'InnerDrawingInformationEvent'] = self.handle_inner_draw_information_event
self.handlers[
'InnerWantToEnterCriticalSection'] = self.inner_handle_want_to_enter_critical_section_event
self.handlers[
'InnerLeavingCriticalSection'] = self.inner_leaving_critical_section_event
self.handlers['InnerNextHopBroken'] = self.inner_next_hop_broken_event
# Outter handlers
self.handlers[
'DrawingInformationEvent'] = self.handle_drawing_information_event
self.handlers[
'EnteredCriticalSectionEvent'] = self.handle_entering_critical_section
self.handlers[
'LeavingCriticalSectionEvent'] = self.handle_leaving_critical_section
self.handlers['TokenPassEvent'] = self.handle_token_pass_event
self.handlers['NewNextNextHop'] = self.handle_new_next_next_hop_event
self.handlers[
'TokenReceivedQuestionEvent'] = self.handle_token_received_question_event
self.handlers['DummyMessageEvent'] = self.handle_dummy_message_event
############################################################################################
#
# Inner Event handlers
############################################################################################
def handle_inner_draw_information_event(self, event):
def draw_points(event):
color = event.data['color']
points = event.data['points']
try:
for point in points:
x, y = point
self.board_state[x][y] = color
except IndexError as e:
return
self.paint_queue.put({
'type': DrawingQueueEvent.DRAWING,
'data': (points, color)
})
if (self.sending_queue):
self.sending_queue.put(
events.DrawingInformationEvent(
self.uuid, helpers.get_current_timestamp(), points,
color))
if not self.critical_section:
draw_points(event)
elif self.critical_section['timestamp'] > event.data['timestamp']:
draw_points(event)
elif self.critical_section['client_uuid'] == self.uuid:
draw_points(event)
elif self.critical_section['client_uuid'] != self.uuid:
pass
def inner_handle_want_to_enter_critical_section_event(self, _):
self.want_to_enter_critical_section = True
def inner_leaving_critical_section_event(self, _):
self.critical_section = None
self.paint_queue.put({"type": DrawingQueueEvent.BOARD_OPEN})
if self.sending_queue:
self.sending_queue.put(
events.LeavingCriticalSectionEvent(
helpers.get_current_timestamp(), self.uuid))
self.sending_queue.put(events.TokenPassEvent(self.last_token))
else:
self.event_queue.put(
events.LeavingCriticalSectionEvent(
helpers.get_current_timestamp(), self.uuid))
self.event_queue.put(events.TokenPassEvent(self.last_token))
def inner_next_hop_broken_event(self, _):
# If we detect that the next hop connection is down we want to:
# 1.Try to reconnect to the client
# 2.If reconnect fails we want to connect to our next next hop
# 3.When we succesfully connect to our next next hop we want to send recovery token question
# in case that the dead client was holding the token the moment he died
ip, port = self.next_next_hop_info
# If we are the only client left we reset the data to the initial state
if ip == helpers.get_self_ip_address():
self.critical_section = None
self.next_hop_info = None
self.next_next_hop_info = None
if self.message_sender:
self.message_sender.stop()
self.sending_queue = None
self.message_sender = None
self.predecessor = None
self.last_token = 0
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_OPEN})
return
def connect_to_next_next_hop(self):
ip, port = self.next_next_hop_info
try:
s = socket.create_connection((ip, port))
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, s)
self.message_sender.start()
self.next_hop_info = self.next_next_hop_info
# After we connect to a new client we have to check whether the dead client wasn't in posession
# of token
self.sending_queue.put(
events.TokenReceivedQuestionEvent(self.last_token))
except Exception as e:
logger.error(e)
ip, port = self.next_hop_info
try:
s = socket.create_connection((ip, port))
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, s)
self.message_sender.start()
except ConnectionRefusedError as e:
logger.error(e)
connect_to_next_next_hop(self)
############################################################################################
#
# Event handlers
############################################################################################
def handle_new_client_request(self, event):
# At first we want to receive information to properly connect as new predecessor after sending init_data
message_size = event.data['connection'].recv(8)
message_size = int.from_bytes(message_size, byteorder='big')
message = b''
while len(message) < message_size:
packet = event.data['connection'].recv(message_size - len(message))
if not packet:
return None
message += packet
client_request = json.loads(message.decode('utf-8'))
first_client = not self.next_hop_info
# When we detect a new client connecting we want to;
# 1.Send him the initial data over the connection we already established
# 2.Connect to him as a predecessor
# Gather the initial board state (only the coloured spots)
marked_spots = [(x, y) for x in range(len(self.board_state))
for y in range(len(self.board_state[x]))
if self.board_state[x][y] == 1]
# If we have next hop information we send it, if we do not have we are the first client so we send our
# information as the first hop information
next_hop = (helpers.get_self_ip_address(),
config.getint(
'NewPredecessorListener',
'Port')) if first_client else self.next_hop_info
# If we are the first client next next hop is None
response = events.NewClientResponseEvent(next_hop,
self.next_next_hop_info,
marked_spots,
self.critical_section)
message = helpers.event_to_message(response)
message_size = (len(message)).to_bytes(8, byteorder='big')
event.data['connection'].send(message_size)
event.data['connection'].send(message)
try:
message = event.data['connection'].recv(8)
except Exception as ex:
if message == b'':
# Only case when we have a succesfull read of 0 bytes is when other socket shutdowns normally
pass
else:
logger.error(ex)
#Client did not initializ correctly so we abort the process
return
# If we are not the first client we have to update our next next hop to our previous next hop
if not first_client:
self.next_next_hop_info = self.next_hop_info
else:
# If we are the first client we update our next next hop info to self address
self.next_next_hop_info = (helpers.get_self_ip_address(),
config.getint('NewPredecessorListener',
'Port'))
# We stop current message sender if it exists
if self.message_sender:
self.message_sender.stop()
# We update our next hop info with the newest client request
self.next_hop_info = client_request['data']['address']
ip, port = self.next_hop_info
# We establish a new connection and a new message sender
connection = socket.create_connection((ip, port), 100)
self.sending_queue = queue.Queue(maxsize=0)
self.message_sender = MessageSender(self.event_queue,
self.sending_queue, connection)
self.message_sender.start()
if first_client and self.last_token != None:
# If we are the first client we start passing of the token
self.sending_queue.put(events.TokenPassEvent(self.last_token))
def handle_drawing_information_event(self, event):
def draw_point(event):
points = event.data['points']
color = event.data['color']
try:
for point in points:
x, y = point
self.board_state[x][y] = color
except IndexError:
return
self.paint_queue.put({
'type': DrawingQueueEvent.DRAWING,
'data': (points, color)
})
if self.sending_queue:
self.sending_queue.put(event)
if event.data['client_uuid'] == self.uuid:
return
if not self.critical_section:
draw_point(event)
elif self.critical_section['timestamp'] > event.data['timestamp']:
draw_point(event)
elif self.critical_section['client_uuid'] == event.data['client_uuid']:
draw_point(event)
elif self.critical_section['client_uuid'] != event.data['client_uuid']:
pass
def handle_new_predecessor_request_event(self, event):
# The moment we have a new predecessor this means that the client before our predecessor
# has a new next next hop address (which is our address) and our predecessor has new next next hop (which is
# our next hop)
self.predecessor = event.data['client_address']
self_address = (helpers.get_self_ip_address(),
config.getint('NewPredecessorListener', 'Port'))
# Special case if we have only 2 nodes left
if self.predecessor[0] == self.next_hop_info[0]:
self.sending_queue.put(
events.NewNextNextHop(self.predecessor, self_address))
self.next_next_hop_info = (helpers.get_self_ip_address(),
config.getint('NewPredecessorListener',
'Port'))
else:
# We send information to predecessor of our predecessor about his new next next hop address
self.sending_queue.put(
events.NewNextNextHop(self_address, self.predecessor))
# We send information to our predecessor about his new next next hop
self.sending_queue.put(
events.NewNextNextHop(self.next_hop_info, self_address))
def handle_entering_critical_section(self, event):
data = event.data
if (data['client_uuid']) == self.uuid:
return
self.critical_section = {
'timestamp': data['timestamp'],
'client_uuid': data['client_uuid']
}
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_CLOSED})
self.sending_queue.put(event)
def handle_leaving_critical_section(self, event):
data = event.data
if (data['client_uuid']) == self.uuid:
return
if self.critical_section and self.critical_section[
'client_uuid'] == event.data['client_uuid']:
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_OPEN})
self.critical_section = None
self.sending_queue.put(event)
def handle_token_pass_event(self, event):
token = event.data['token'] + 1
self.last_token = token
if self.critical_section:
# If we have received the token and the critical section exists we unvalidate critical secion info
self.critical_section = None
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_OPEN})
if self.want_to_enter_critical_section:
timestamp = helpers.get_current_timestamp()
self.critical_section = {
'timestamp': timestamp,
'client_uuid': self.uuid
}
leave_critical_section_deamon = CriticalSectionLeaver(
self.event_queue)
leave_critical_section_deamon.start()
self.want_to_enter_critical_section = False
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_CONTROLLED})
self.sending_queue.put(
events.EnteredCriticalSectionEvent(timestamp, self.uuid))
else:
if self.sending_queue:
self.sending_queue.put(events.TokenPassEvent(token))
def handle_new_next_next_hop_event(self, event):
post_destination_ip, _ = event.data['destination_next_hop']
next_hop_ip, _ = self.next_hop_info
# We are the recipient of the message
if post_destination_ip == next_hop_ip:
self.next_next_hop_info = event.data['new_address']
else:
self.sending_queue.put(event)
def handle_token_received_question_event(self, event):
# We check weather the last token we received is greater than
# the token from the request
# If it is, this means that the disconnected client was not in posession of the token when he disconnected
# If it was we have to unvalidate critial secion information and send token further
if self.last_token > event.data['token'] + 1:
return
else:
self.critical_section = None
self.paint_queue.put({'type': DrawingQueueEvent.BOARD_OPEN})
token = event.data['token'] + 1 if event.data[
'token'] else self.last_token + 1
self.sending_queue.put(events.TokenPassEvent(token))
def handle_dummy_message_event(self, event):
if self.uuid != event.data['uuid']:
return
else:
if self.sending_queue:
self.sending_queue.put(event)