server_lean.py

#!/usr/bin/python

#   Copyright (C) 2015 by seeedstudio
#   Author: Jack Shao (jacky.shaoxg@gmail.com)
#
#   The MIT License (MIT)
#
#   Permission is hereby granted, free of charge, to any person obtaining a copy
#   of this software and associated documentation files (the "Software"), to deal
#   in the Software without restriction, including without limitation the rights
#   to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#   copies of the Software, and to permit persons to whom the Software is
#   furnished to do so, subject to the following conditions:
#
#   The above copyright notice and this permission notice shall be included in
#   all copies or substantial portions of the Software.
#
#   THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#   IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#   FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#   OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#   THE SOFTWARE.

#   Dependences: pip install tornado
#                pip install pycrypto


########################################
#Configurations
#
#Set the OTA server which is used by the Wio Links
#This script will connect to OTA server to fetch node informations.
#Options: china, international, custom
OTA_SERVER='china'

#Set the address of the OTA server if OTA_SERVER is set to 'custom'
#Only applies when OTA_SERVER='custom'
CUSTOM_OTA_SERVER_ADDR='https://192.168.31.126'

#Set the accounts which will be used when logging in the OTA server
#Set each account with a key-value pair in which the pattern is email:password
ACCOUNTS={'jjj@jjj.com':'jjjjjj'}

########################################


from datetime import timedelta
import os
import json
import hashlib
import hmac
import binascii
import urllib
import urllib2
import ssl
from Crypto.Cipher import AES
from Crypto import Random

from tornado.httpserver import HTTPServer
from tornado.tcpserver import TCPServer
from tornado import ioloop
from tornado import gen
from tornado import iostream
from tornado import web
from tornado import websocket
from tornado.options import define, options
from tornado.log import *
from tornado.concurrent import Future
from tornado.queues import Queue
from tornado.locks import Semaphore, Condition


define("mode", default="online", help="Running mode, online: will fetch node info from OTA server at start up.")

PENDING_REQ_CNT = 10
NODES_DATABASE=[]

BS = AES.block_size
pad = lambda s: s if (len(s) % BS == 0) else (s + (BS - len(s) % BS) * chr(0) )
unpad = lambda s : s.rstrip(chr(0))

class DeviceConnection(object):

    state_waiters = {}
    state_happened = {}

    def __init__ (self, device_server, stream, address):
        self.recv_msg_cond = Condition()
        self.recv_msg = {}
        self.send_msg_sem = Semaphore(1)
        self.pending_request_cnt = 0
        self.device_server = device_server
        self.stream = stream
        self.address = address
        self.stream.set_nodelay(True)
        self.idle_time = 0;
        self.killed = False
        self.sn = ""
        self.private_key = ""
        self.node_id = 0
        self.name = ""
        self.iv = None
        self.cipher = None

        #self.state_waiters = []
        #self.state_happened = []

        self.event_waiters = []
        self.event_happened = []

        self.ota_ing = False
        self.ota_notify_done_future = None
        self.post_ota = False
        self.online_status = True

    @gen.coroutine
    def secure_write (self, data):
        if self.cipher:
            cipher_text = self.cipher.encrypt(pad(data))
            yield self.stream.write(cipher_text)

    @gen.coroutine
    def wait_hello (self):
        try:
            self._wait_hello_future = self.stream.read_bytes(64) #read 64bytes: 32bytes SN + 32bytes signature signed with private key
            str = yield gen.with_timeout(timedelta(seconds=10), self._wait_hello_future, io_loop=self.stream.io_loop)
            self.idle_time = 0  #reset the idle time counter

            if len(str) != 64:
                self.stream.write("sorry\r\n")
                yield gen.sleep(0.1)
                self.kill_myself()
                gen_log.debug("receive length != 64")
                raise gen.Return(100) # length not match 64

            sn = str[0:32]
            sig = str[32:64]

            gen_log.info("accepted sn: "+ sn)

            #query the sn from database
            node = None
            for n in NODES_DATABASE:
                if n['node_sn'] == sn:
                    node = n
                    break

            if not node:
                self.stream.write("sorry\r\n")
                yield gen.sleep(0.1)
                self.kill_myself()
                gen_log.info("node sn not found")
                raise gen.Return(101) #node not found

            key = node['node_key']
            key = key.encode("ascii")

            sig0 = hmac.new(key, msg=sn, digestmod=hashlib.sha256).digest()
            gen_log.debug("sig:     "+ binascii.hexlify(sig))
            gen_log.debug("sig calc:"+ binascii.hexlify(sig0))

            if sig0 == sig:
                #send IV + AES Key
                self.sn = sn
                self.private_key = key
                self.node_id = node['node_sn']
                self.name = node['name']
                gen_log.info("valid hello packet from node %s" % self.name)
                #remove the junk connection of the same sn
                self.stream.io_loop.add_callback(self.device_server.remove_junk_connection, self)
                #init aes
                self.iv = Random.new().read(AES.block_size)
                self.cipher = AES.new(key, AES.MODE_CFB, self.iv, segment_size=128)
                cipher_text = self.iv + self.cipher.encrypt(pad("hello"))
                gen_log.debug("cipher text: "+ cipher_text.encode('hex'))
                self.stream.write(cipher_text)
                raise gen.Return(0)
            else:
                self.stream.write("sorry\r\n")
                yield gen.sleep(0.1)
                self.kill_myself()
                gen_log.error("signature not match: %s %s" % (sig, sig0))
                raise gen.Return(102) #sig not match
        except gen.TimeoutError:
            self.kill_myself()
            raise gen.Return(1)
        except iostream.StreamClosedError:
            self.kill_myself()
            raise gen.Return(2)

        #self.stream.io_loop.add_future(self._serving_future, lambda future: future.result())

    @gen.coroutine
    def _loop_reading_input (self):
        line = ""
        piece = ""
        while not self.killed:
            msg = ""
            try:
                msg = yield self.stream.read_bytes(16)
                msg = unpad(self.cipher.decrypt(msg))

                self.idle_time = 0 #reset the idle time counter

                line += msg

                while line.find('\r\n') > -1:
                    index = line.find('\r\n')
                    piece = line[:index+2]
                    line = line[index+2:]
                    piece = piece.strip("\r\n")
                    json_obj = json.loads(piece)
                    gen_log.info('Node %s: recv json: %s' % (self.name, str(json_obj)))

                    try:
                        state = None
                        event = None
                        if json_obj['msg_type'] == 'online_status':
                            if json_obj['msg'] in ['1',1,True]:
                                self.online_status = True
                            else:
                                self.online_status = False
                            continue
                        elif json_obj['msg_type'] == 'ota_trig_ack':
                            state = ('going', 'Node has been notified...')
                            self.ota_ing = True
                            if self.ota_notify_done_future:
                                self.ota_notify_done_future.set_result(1)
                        elif json_obj['msg_type'] == 'ota_status':
                            if json_obj['msg'] == 'started':
                                state = ('going', 'Downloading the firmware...')
                            else:
                                state = ('error', 'Failed to start the downloading.')
                                self.post_ota = True
                        elif json_obj['msg_type'] == 'ota_result':
                            if json_obj['msg'] == 'success':
                                state = ('done', 'Firmware updated.')
                            else:
                                state = ('error', 'Update failed. Please reboot the node and retry.')
                            self.post_ota = True
                            self.ota_ing = False
                        elif json_obj['msg_type'] == 'event':
                            event = json_obj
                            event.pop('msg_type')
                        gen_log.debug(state)
                        gen_log.debug(event)
                        if state:
                            #print self.state_waiters
                            #print self.state_happened
                            if self.state_waiters and self.sn in self.state_waiters and len(self.state_waiters[self.sn]) > 0:
                                f = self.state_waiters[self.sn].pop(0)
                                f.set_result(state)
                                if len(self.state_waiters[self.sn]) == 0:
                                    del self.state_waiters[self.sn]
                            elif self.state_happened and self.sn in self.state_happened:
                                self.state_happened[self.sn].append(state)
                            else:
                                self.state_happened[self.sn] = [state]
                        elif event:
                            if len(self.event_waiters) == 0:
                                self.event_happened.append(event)
                            else:
                                for future in self.event_waiters:
                                    future.set_result(event)
                                self.event_waiters = []
                        else:
                            self.recv_msg = json_obj
                            self.recv_msg_cond.notify()
                            yield gen.moment
                    except Exception,e:
                        gen_log.warn("Node %s: %s" % (self.name ,str(e)))

            except iostream.StreamClosedError:
                gen_log.error("StreamClosedError when reading from node %s" % self.name)
                self.kill_myself()
                return
            except ValueError:
                gen_log.warn("Node %s: %s can not be decoded into json" % (self.name, piece))
            except Exception,e:
                gen_log.error("Node %s: %s" % (self.name ,str(e)))
                self.kill_myself()
                return

            yield gen.moment

            #if self.post_ota:
            #    gen_log.error("Node %s OTA finished, will reboot, kill the connection" % self.name )
            #    self.kill_myself()
            #    return

    @gen.coroutine
    def _online_check (self):
        while not self.killed:
            yield gen.sleep(1)
            if self.ota_ing:
                continue 
            self.idle_time += 1
            if self.idle_time == 60:
                gen_log.info("heartbeat sent to node %s" % self.name)
                try:
                    yield self.secure_write("##PING##")
                except iostream.StreamClosedError:
                    gen_log.error("StreamClosedError when send ping to node %s" % self.name)
                    self.kill_myself()
            if self.idle_time == 70:
                gen_log.error("no answer from node %s, kill" % self.name)
                self.kill_myself()


    @gen.coroutine
    def start_serving (self):
        ret = yield self.wait_hello()
        if ret == 0:
            #gen_log.info("waited hello")
            pass
        elif ret == 1:
            gen_log.info("timeout waiting hello, kill this connection")
            return
        elif ret == 2:
            gen_log.info("connection is closed by client")
            return
        elif ret >= 100:
            return

        ## loop reading the stream input
        self._loop_reading_input()
        self._online_check()

    def kill_myself (self):
        if self.killed:
            return
        self.sn = ""
        self.stream.io_loop.add_callback(self.device_server.remove_connection, self)
        self.stream.close()
        self.killed = True

    @gen.coroutine
    def submit_cmd (self, cmd):
        yield self.send_msg_sem.acquire()
        try:
            yield self.secure_write(cmd)
        finally:
            self.send_msg_sem.release()

    @gen.coroutine
    def submit_and_wait_resp (self, cmd, target_resp, timeout_sec=5):

        self.pending_request_cnt += 1
        if self.pending_request_cnt > PENDING_REQ_CNT:
            self.pending_request_cnt = PENDING_REQ_CNT
            gen_log.warn('Node %s: request too fast' % self.name)
            raise gen.Return((False, {"status":403, "msg":"request too fast"}))

        yield self.send_msg_sem.acquire()
        try:
            yield self.secure_write(cmd)
            ok = yield self.recv_msg_cond.wait(timeout=timedelta(seconds=timeout_sec))
            if not ok:
                gen_log.error("timeout when waiting response from Node %s" % self.name)
                raise gen.Return((False, {"status":205, "msg":"timeout when waiting response from Node %s" % self.name}))

            msg = self.recv_msg
            if msg['msg_type'] == target_resp:
                tmp = msg
                del tmp['msg_type']
                raise gen.Return((True, tmp))
            else:
                raise gen.Return((False, {"status":205, "msg":"bad response"}))
        except gen.Return:
            raise            
        except Exception,e:
            gen_log.error(e)
            raise gen.Return((False, {"status":205, "msg":"Node %s: %s" % (self.name, str(e))}))
        finally:
            self.send_msg_sem.release()  #inc the semaphore value to 1
            self.pending_request_cnt -= 1


class DeviceServer(TCPServer):

    accepted_xchange_conns = []

    def __init__ (self):
        self.role = 'xchange'
        TCPServer.__init__(self)


    def handle_stream(self, stream, address):
        conn = DeviceConnection(self, stream,address)

        self.accepted_xchange_conns.append(conn)
        gen_log.info("%s device server accepted conns: %d"% (self.role, len(self.accepted_xchange_conns)))

        conn.start_serving()

    def remove_connection (self, conn):
        gen_log.info("%s device server will remove connection: %s" % (self.role, str(conn)))
        try:
            self.accepted_xchange_conns.remove(conn)
            del conn
        except:
            pass

    def remove_junk_connection (self, conn):
        try:
            connections = self.accepted_xchange_conns
            for c in connections:
                if c.sn == conn.sn and c != conn :
                    c.killed = True
                    #clear waiting futures
                    gen_log.info("%s device server removed one junk connection of same sn: %s"% (self.role, c.sn))
                    connections.remove(c)
                    del c
                    break
        except:
            pass


class NodeBaseHandler(web.RequestHandler):

    def initialize (self, conns):
        self.conns = conns

    def get_current_user(self):
        return None

    def get_node (self):
        node = None
        token = self.get_argument("access_token","")
        if not token:
            try:
                token_str = self.request.headers.get("Authorization")
                token = token_str.replace("token ","")
            except:
                token = None
        gen_log.debug("node token:"+ str(token))
        if token and len(token) == 32:
            try:
                for n in NODES_DATABASE:
                    if n['node_key'] == token:
                        node = n
                        node['private_key'] = token
                        node['node_id'] = node['node_sn']
                        break
            except:
                node = None
        else:
            node = None

        if not node:
            self.resp(403,"Node is offline or invalid node token (not the user token).")
        else:
            gen_log.info("get current node, sn: %s, name: %s" % (node['node_sn'],node["name"]))

        return node

    '''
    205 - custom, get response from node but with error
    400 - bad variables / format
    401 - unauthorized
    403 - forbidden
    404 - not found
    500 - server internal error
    '''
    def resp (self, status, msg="",meta={}):
        response = {"status":status,"msg":msg}
        response = dict(response, **meta)
        self.write(response)


class IndexHandler(NodeBaseHandler):
    def get(self):
        self.resp(400,msg = "Please specify the url as this format: /v1/node/grove_name/property")


class NodeReadWriteHandler(NodeBaseHandler):

    @gen.coroutine
    def get(self, ignore, uri):

        gen_log.debug(ignore)

        uri = uri.split("?")[0]
        gen_log.debug("get:"+ str(uri))


        node = self.get_node()
        if not node:
            return

        for conn in self.conns:
            if conn.sn == node['node_sn'] and not conn.killed:
                try:
                    cmd = "GET /%s\r\n"%(uri)
                    cmd = cmd.encode("ascii")
                    ok, resp = yield conn.submit_and_wait_resp (cmd, "resp_get")
                    if 'status' in resp:
                        self.resp(resp['status'],resp['msg'])
                    else:
                        self.resp(200,resp['msg'])
                except Exception,e:
                    gen_log.error(e)
                return
        self.resp(404, "Node is offline")

    @gen.coroutine
    def post (self, ignore, uri):

        uri = uri.split("?")[0].rstrip("/")
        gen_log.info("post to:"+ str(uri))

        node = self.get_node()
        if not node:
            return

        try:
            json_obj = json.loads(self.request.body)
        except:
            json_obj = None

        cmd_args = ""

        if json_obj:
            gen_log.info("post request:"+ str(json_obj))

            if not isinstance(json_obj, dict):
                self.resp(400, "Bad format of json: must be key/value pair.")
                return
            for k,v in json_obj.items():
                cmd_args += str(v)
                cmd_args += "/"
        else:

            arg_list = self.request.body.split('&')

            gen_log.info("post request:"+ str(arg_list))

            if len(arg_list) == 1 and arg_list[0] == "":
                arg_list = []

            for arg in arg_list:
                if arg.find('=') > -1:
                    value = arg.split('=')[1]
                    cmd_args += value
                    cmd_args += "/"
                else:
                    cmd_args += arg
                    cmd_args += "/"

        for conn in self.conns:
            if conn.sn == node['node_sn'] and not conn.killed:
                try:
                    cmd = ("/%s/%s"%(uri, cmd_args)).rstrip("/")
                    cmd = "POST %s\r\n"%(cmd)
                    cmd = cmd.encode("ascii")
                    ok, resp = yield conn.submit_and_wait_resp (cmd, "resp_post")
                    if 'status' in resp:
                        self.resp(resp['status'],resp['msg'])
                    else:
                        self.resp(200,resp['msg'])
                except Exception,e:
                    gen_log.error(e)
                return

        self.resp(404, "Node is offline")

class NodeEventHandler(websocket.WebSocketHandler):
    def initialize (self, conns):
        self.conns = conns
        self.cur_conn = None
        self.node_key = None
        self.connected = False
        self.future = None

    def check_origin(self, origin):
        return True

    def open(self):
        gen_log.info("websocket open")
        self.connected = True

    def on_close(self):
        gen_log.info("websocket close")
        if self.connected and self.cur_conn:
            cur_waiters = self.cur_conn.event_waiters
            if self.future in cur_waiters:
                self.future.set_result(None)
                cur_waiters.remove(self.future)
                # cancel yield
                pass

        self.connected = False

    def find_node(self, key):
        for c in self.conns:
            if c.private_key == key and not c.killed:
                return c
        return None

    @gen.coroutine
    def on_message(self, message):
        self.node_key = message
        if len(message) != 32:
            self.write_message({"error":"invalid node sn "})
            self.connected = False
            self.close()

        self.cur_conn = self.find_node(message)

        if not self.cur_conn:
            self.node_offline()
            return

        #clear the events buffered before any websocket client connected
        self.cur_conn.event_happened = []

        while self.connected:
            self.future = self.wait_event_post()
            event = None
            try:
                event = yield gen.with_timeout(timedelta(seconds=5), self.future, io_loop=ioloop.IOLoop.current())
            except gen.TimeoutError:
                if not self.cur_conn or self.cur_conn.killed:
                    gen_log.debug("node %s is offline" % message)
                    self.cur_conn = self.find_node(message)
                    if not self.cur_conn:
                        self.node_offline()
            if event:
                self.write_message(event)
            yield gen.moment

    def wait_event_post(self):
        result_future = Future()

        if len(self.cur_conn.event_happened) > 0:
            result_future.set_result(self.cur_conn.event_happened.pop(0))
        else:
            self.cur_conn.event_waiters.append(result_future)

        return result_future

    def node_offline(self):
        try:
            self.write_message({"error":"node is offline"})
        except:
            pass
        self.connected = False
        self.close()


class myApplication(web.Application):

    def __init__(self):
        handlers = [
        (r"/v1[/]?", IndexHandler, dict(conns=None)),
        (r"(?!.*(event|config|resources))/v1/node/(.+)", NodeReadWriteHandler, dict(conns=DeviceServer.accepted_xchange_conns)),
        (r"/v1/node/event[/]?", NodeEventHandler,dict(conns=DeviceServer.accepted_xchange_conns)),
        ]

        web.Application.__init__(self, handlers, debug=True)


def fetch_node_info():

    global NODES_DATABASE

    ota_server_addr = CUSTOM_OTA_SERVER_ADDR
    if OTA_SERVER == 'china':
        ota_server_addr = 'https://cn.iot.seeed.cc'
    elif OTA_SERVER == 'international':
        ota_server_addr = 'https://iot.seeed.cc'

    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE

    if len(ACCOUNTS) == 0:
        gen_log.error("Please configure the user account!!!")
        sys.exit()

    gen_log.info("Fetching the list of nodes...")

    for a in ACCOUNTS:
        acc = {'email':a, 'password': ACCOUNTS[a]}
        data = urllib.urlencode(acc)
        login_result_obj = {'token':'', }
        try:
            login_result = urllib2.urlopen(ota_server_addr.rstrip('/')+'/v1/user/login', data, context=ctx)
            login_result_obj = json.load(login_result)

        except urllib2.URLError,e:
            gen_log.error(e.reason)
            sys.exit()

        #print login_result_obj
        if 'status' in login_result_obj and login_result_obj['status'] != 200:
            gen_log.error('error happened when logging in %s' % a)
            gen_log.info(login_result_obj['msg'])
            sys.exit()


        try:
            list_result = urllib2.urlopen(ota_server_addr.rstrip('/')+'/v1/nodes/list?access_token='+login_result_obj['token'], context=ctx)
            list_result_obj = json.load(list_result)
        except urllib2.URLError,e:
            gen_log.error(e.reason)
            sys.exit()

        if 'status' in list_result_obj and list_result_obj['status'] != 200:
            gen_log.error('error happened when listing nodes for %s' % a)
            gen_log.info(list_result_obj['msg'])
            sys.exit()

        NODES_DATABASE += list_result_obj['nodes']

    if len(NODES_DATABASE) == 0:
        gen_log.error("No node!!!")
        sys.exit()

    gen_log.info("fetch nodes done!")
    gen_log.debug(NODES_DATABASE)
    cur_dir = os.path.split(os.path.realpath(__file__))[0]
    open("%s/nodes_db_lean.json"%cur_dir,"w").write(json.dumps(NODES_DATABASE))

def main():

    global NODES_DATABASE

    ###--log_file_prefix=./server.log
    ###--logging=debug
    enable_pretty_logging()
    options.parse_command_line()

    cur_dir = os.path.split(os.path.realpath(__file__))[0]

    if options.mode == 'online':
        fetch_node_info()
    else:
        db_file_path = '%s/nodes_db_lean.json' % cur_dir

        if not os.path.exists(db_file_path):
            fetch_node_info()
        else:
            NODES_DATABASE = json.load(open(db_file_path))
            gen_log.info('load nodes database from json done!')
            gen_log.debug(NODES_DATABASE)


    app = myApplication()
    http_server = HTTPServer(app)
    http_server.listen(8080)


    tcp_server = DeviceServer()
    tcp_server.listen(8000)

    gen_log.info("server's running in lean mode ...")

    ioloop.IOLoop.current().start()


if __name__ == '__main__':
    main()