Пример #1
0
    def Subscribe(self, queue=None, callback=None):
        #
        # the queue and callback being subscribed to is not passed in here
        # but passed in through via "self"
        # the question is how to make this scale?
        #
        #
        #

        self.lockObj.acquire()
        if queue == None:  # done for compatibility with "simple" version
            queue = self.myqueue
        if callback == None:
            callback = self._mycallback

        #
        # communication parameters within thread
        #
        self.thread_params.append(queue)  # 1
        self.thread_params.append(callback)  # 2
        #
        # control param for thread
        #
        self.ev_obj = Event()
        self.thread_params.append(self.ev_obj)  # 3 - top of stack

        subscribe = KThread(target=self._SubscribeThread)
        subscribe.start()
        self.lockObj.release()
        #
        # return thread and its control object
        #
        return subscribe, self.ev_obj
Пример #2
0
 def Subscribe(self,queue=None,callback=None):
    #
    # the queue and callback being subscribed to is not passed in here
    # but passed in through via "self"
    # the question is how to make this scale?
    #
    #
    #
    
    self.lockObj.acquire()     
    if (queue == None):    # done for compatibility with "simple" version
      queue = self.myqueue
    if (callback == None):
      callback = self._mycallback
      
    #
    # communication parameters within thread
    #
    self.thread_params.append(queue)       # 1
    self.thread_params.append(callback)    # 2
    #
    # control param for thread
    #
    self.ev_obj = Event()
    self.thread_params.append(self.ev_obj)      # 3 - top of stack
     
    subscribe = KThread(target = self._SubscribeThread)
    subscribe.start() 
    self.lockObj.release()    
    #
    # return thread and its control object
    #
    return subscribe, self.ev_obj
class HttpServer(threading.Thread):
    """Thread class with a stop() method. The thread itself has to check
    regularly for the stopped() condition."""
    def stop(self):
        self.server.kill()
        print "HTTP server stopped"
        pass

    pass

    def stopped(self):
        return self._stop.isSet()
        pass

    pass

    def start(self):
        self.server = KThread(target=self.start_thread)
        self.server.start()
        pass

    pass

    def start_thread(self):
        PORT = 8000  # port used by the http server

        Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
        httpd = SocketServer.TCPServer(('', PORT), Handler)

        print "HTTP serving at port", PORT
        httpd.serve_forever()
        pass

    pass
Пример #4
0
def main():
    try:
        with open('config.json', 'r') as f:
            config = json.load(f)
        ports = config['AddressBook']
        num_ports = len(ports)
    except Exception as e:
        raise e

    while True:
        customer = client()
        server_id = input('Whom to connect ? 1-%d: ' % num_ports )
        request = raw_input('How can we help you? --')
        if request == 'buy':
            uuid_ = uuid.uuid1()
            requestThread = KThread(target = customer.buyTickets, args =  (ports[server_id - 1], request, uuid_))
            timeout = 5
        else:
            requestThread = KThread(target = customer.show_state, args = (ports[server_id - 1],))
            timeout = 5
        start_time = time.time()
        requestThread.start()
        while time.time() - start_time < timeout:
            if not requestThread.is_alive():
                break
        if requestThread.is_alive():
            requestThread.kill()
            msgP = 'Timeout! Try again'
            print msgP
Пример #5
0
 def listen(self, on_accept):
     srv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     srv.bind(("", self.port))
     print 'start listenning'
     while True:
         data, addr = srv.recvfrom(1024)
         thr = KThread(target=on_accept, args=(self, data, addr))
         thr.start()
     srv.close()
 def listen(self, on_accept):
     logger.debug('Server Listen Method')
     srv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
     srv.bind(("", self.port))
     print 'start listenning ', self.id, " ", self.currentTerm
     while True:
         data,addr = srv.recvfrom(1024)
         #print 'listening ', self.id
         thr = KThread(target=on_accept, args=(self,data,addr))
         thr.start()
     srv.close()
Пример #7
0
 def initiateVote(self):
     # Todo: mcip, make sure the term is the same while request vote????
     req_term = self.currentTerm
     for idx, addr in enumerate(self.addresses):
         if idx == self.id:
             continue
         # Create a thread for each request vote
         # Todo: mcip: req_term should be the same
         election_thread = KThread(target=self.thread_election,
                                   args=(
                                       idx,
                                       addr,
                                       req_term,
                                   ))
         election_thread.start()
Пример #8
0
def main():
    try:
        with open('config.json', 'r') as f:
            config = json.load(f)
        ports = config['AddressBook']
        num_ports = len(ports)
    except Exception as e:
        raise e

    while True:
        customer = client()
        server_id = input(
            'Which datacenter do you want to connect to? 1-%d: ' % num_ports)
        request = raw_input('How can we help you? --')
        if request == 'show':
            requestThread = KThread(target=customer.show_state,
                                    args=(ports[server_id - 1], ))
            timeout = 5
        elif request.split()[0] == 'change':
            uuid_ = uuid.uuid1()
            msg_split = request.split()
            new_config_msg = msg_split[1:]
            new_config = [int(item) for item in new_config_msg]
            print new_config
            requestThread = KThread(target=customer.config_change,
                                    args=(ports[server_id - 1], new_config,
                                          uuid_))
            timeout = 20
        else:
            uuid_ = uuid.uuid1()
            requestThread = KThread(target=customer.buyTickets,
                                    args=(ports[server_id - 1], request,
                                          uuid_))
            timeout = 5
        start_time = time.time()
        requestThread.start()
        while time.time() - start_time < timeout:
            if not requestThread.is_alive():
                break
        if requestThread.is_alive():
            print 'Timeout! Try again'
            requestThread.kill()
Пример #9
0
 def run(self):
     # Create a thread to run as follower
     leader_state = KThread(target=self.leader, args=())
     leader_state.start()
     candidate_state = KThread(target=self.candidate, args=())
     candidate_state.start()
     follower_state = KThread(target=self.follower, args=())
     follower_state.start()
Пример #10
0
 def append_entries(self):
     while self.role == KVServer.leader:
         # Todo: for debugging only
         # # self.debug1 += 1
         # self.logModify([self.debug1, "aa", "bb"], LogMod.APPEND)
         # self.logModify([self.debug1, "bb", "cc"], LogMod.APPEND)
         app_ent_term = self.currentTerm
         for idx, addr in enumerate(self.addresses):
             if idx == self.id:
                 continue
             # Create a thread for each append_entry message
             # Todo: mcip: append entries term is the same
             append_thread = KThread(target=self.thread_append_entry,
                                     args=(
                                         idx,
                                         addr,
                                         app_ent_term,
                                     ))
             append_thread.start()
         # Send append entry every following seconds, or be notified and wake up
         # Todo: will release during wait
         with self.appendEntriesCond:
             self.appendEntriesCond.wait(timeout=self.appendEntriesTimeout)
Пример #11
0
class mp3GUI(wx.Frame):
    fileList = []
    def __init__(self, parent, id, title,playerInstance):
        root = wx.App()
        wx.Frame.__init__(self, parent, id, title, size=(650, 380))

        hbox = wx.BoxSizer(wx.VERTICAL)

        panel = wx.Panel(self, -1)

        self.list = SortedListCtrl(panel,self.fileList)
        self.list.InsertColumn(0, 'file', 635)

        self.Bind(wx.EVT_LIST_ITEM_ACTIVATED, self.playClickedFile)

        self.updateFileList()

        gauge = wx.Gauge(panel, range=100, size=(100, 10))
        hbox.Add(self.list, 1, wx.EXPAND)
        hbox.Add(gauge, 0, wx.EXPAND)
        panel.SetSizer(hbox)

        
        #panel2 = wx.Panel(self, 0)
        #hbox2 = wx.BoxSizer(wx.HORIZONTAL)
        #gauge = wx.Gauge(panel2, range=1000, size=(200, 15))
        #hbox2.Add(gauge,1,wx.EXPAND)
        #panel2.SetSizer(hbox2)

        self.CreateStatusBar()
        self.SetStatusText("This is the statusbar")

        MN_ADD_DIR  =   101
        MN_ADD_FILE =   102
        MN_QUIT     =   100

        file_menu = wx.Menu()
        menubar = wx.MenuBar()
        file_menu.Append(MN_ADD_DIR, "Add &Directory ...", "Select a directory to add to the list")
        file_menu.Append(MN_ADD_FILE, "Add &Files ...", "Select files add to the list")
        file_menu.AppendSeparator()
        file_menu.Append(MN_QUIT, "&Quit", "Quit the application")

        self.Bind(wx.EVT_MENU, self.pickDirectory, id=MN_ADD_DIR)
        self.Bind(wx.EVT_MENU, self.pickFile, id=MN_ADD_FILE)
        self.Bind(wx.EVT_MENU, self.onExit, id=MN_QUIT)

        self.Bind(wx.EVT_CLOSE, self.onExit)    # user / app quits
        self.Bind(wx.EVT_QUERY_END_SESSION, self.onExit) # system shutdown, veto
        self.Bind(wx.EVT_END_SESSION, self.onExit) # system shutdown, no veto

        menubar.Append(file_menu, '&File')
        self.SetMenuBar(menubar)

        self.Centre()
        self.Show(True)
        self.playing = False
        self.root = root
        self.playerInstance = playerInstance
        self.gauge = gauge        

    def onExit(self,event):
        self.need_abort = True
        if self.playing or self.playerInstance.cur_song.playing:
            self.playerInstance.cur_song.stop()
            self.songGaugeThread.kill()
            print "\nStopped music."
        
        print "Closing application..."
        self.Destroy()

    def start(self):
        self.root.MainLoop()

    def updateFileList(self):
        items = self.list.itemDataMap

        for i in xrange(len(items)):
            index = self.list.InsertStringItem(sys.maxint, items[i])
            #self.list.SetStringItem(index, 1, data[1])
            #self.list.SetStringItem(index, 2, data[2])
            self.list.SetItemData(index, i)

    def GetItemText(self, item):
        return self.list.itemDataMap[item]

    def pickFile(self, event):             # Call "Choose file..." dialog
        temp_location = wx.EmptyString
        # for multiple formats: wildcard="*.mp3;*.wav"
        
        tfiles = wx.FileDialog(self, "Choose one or more .mp3 files", temp_location, wildcard="*.mp3", style=wx.FD_MULTIPLE)
        if (tfiles.ShowModal() == wx.ID_OK):
            filepaths = tfiles.GetPaths()
        print filepaths
        self.list.itemDataMap = list(set(filepaths))
        # append selected file and update the list
        #self.fileList.append(tfile)
        self.updateFileList()

    def pickDirectory(self, event):             # Call "Choose directory..." dialog
        temp_location = wx.EmptyString
        tdir = wx.DirDialog(self, "Choose a directory with mp3 files", temp_location, style=wx.DD_DIR_MUST_EXIST)
        if (tdir.ShowModal() == wx.ID_OK):
            path = tdir.GetPath()
        
            tfileList = []
            for (dirpath, dirnames, filenames) in os.walk(path):
                for tfile in filenames:
                    if tfile.endswith(".mp3") or tfile.endswith(".wav"):
                        tfileList.append(dirpath+"/"+tfile)
        #print tfileList
        self.list.itemDataMap = list(set(tfileList))
        # append selected file and update the list
        #self.fileList.append(tfile)
        self.updateFileList()

    def playClickedFile(self, event):   # play selected file in listbox with playerInstance
        try:
            if self.playerInstance.cur_song.playing:
                self.playerInstance.cur_song.stop()
                self.songGaugeThread.kill()
        except AttributeError:
            pass

        self.playerInstance.playFile(str(event.GetItem().GetText()))
        
        if self.playerInstance.cur_song.playing:
            #thread.start_new_thread(self.playingSongLoop,())
            self.songGaugeThread = KThread(target=self.playingSongLoop)
            self.songGaugeThread.start()
            

    def playingSongLoop(self):
        self.gauge.SetRange(self.playerInstance.cur_song.length)
        while self.playerInstance.cur_song.playing:
            self.gauge.SetValue(self.playerInstance.cur_song.position)
            self.playing = self.playerInstance.cur_song.playing
            time.sleep(0.8)
Пример #12
0
                while True :
                    try :
                        Awhatsapp.selecting_from_recent_contact(element['whatsapp_id'])
                        Awhatsapp.send_message(element['msg'])
                        break
                    except :
                        #print "IN except"
                        pass
           # A.start()
            flag =1
        time.sleep(10)
# thread.start_new_thread(f1,())
# thread.start_new_thread(f2,())
A= KThread(target=f1)
B= KThread(target=f2)
A.start()
B.start()
# f1()
# def a1():
#     print "in a1"
#     # while True :
#     #     pass
#     print "Func stopped"
# def b1():
#     print "in b1"
#     global A
#     i=0
#     while i<5 :
#         print i
#         i+=1
#     A.kill()
Пример #13
0
class CommObject:
    def __init__(self, host="localhost", userid="guest", password="******"):
        self.error = False
        self.debug = False
        self.host = host
        self.userid = userid
        self.password = password
        self.configList = []  # scalable config for rabbitmq
        self.chanList = []  # [id,obj] ordered pairs
        self.lockObj = Lock()  # used for thread safety
        self.thread_params = list()  # used for passing params into thread

        self.exchange = "myexchange"
        self.uuid = str(uuid.uuid1())  # make each queue unique
        self.myqueue = "myqueue" + self.uuid
        self.mykey = "myq.myx"
        self.running = False
        self.subscribe = None  # self.subscribe.start() to start thread for listening on exchange

        try:
            self.connection = amqp.Connection(
                userid=self.userid, password=self.password, host=self.host, virtualhost="/", ssl=False
            )
        except Exception as eobj:
            print eobj
            print "Is the server running? (sh rabbitmq-server-2.1.1/scripts/rabbitmq-server)"
            self.error = True
            #
            # an object is returned, not None
            #

    # end __init__
    # -----------------------------------------------------------
    def SimpleConfigure(self):
        try:
            self.channel = self.connection.channel()
        except Exception as eobj:
            print eobj
            self.connection.close()
            return -1

        print "channel = ", self.channel.channel_id

        try:
            self.channel.access_request("/data", active=True, write=True, read=True)
        except Exception as eobj:
            print eobj
            self.channel.close()
            return -1

        #
        # Note: When configuring/reconfiguring the server may not like changes
        #       to existing exchanges. Closing channel on exception clears server.

        #
        # with mulitiple listeners 'direct' ping-pongs 1:1 across n listeners
        #

        #
        # defaults on exchange_declare:are same as queue_declare:
        # passive=False, durable=False,
        # auto_delete=True, internal=False, nowait=False
        #
        # if you mix defaults between the two, then you will raise an exception
        #
        # auto_delete -> queue is deleted when last consumer closes
        # durable -> survives server reboot
        # passive -> when set tests presence of queue, if not present then "access refused" returned
        # internal -> special purpose, keep false
        # nowait -> server doesn't reply, keep false
        #
        queue_type = "fanout"  # 'direct' #
        try:
            self.channel.exchange_declare(self.exchange, type=queue_type, durable=False, auto_delete=True)
        except Exception as eobj:
            print eobj
            print "The server is unhappy."
            print "You re-declared an exchange and changed a parameter."
            print "Kill and restart the server."
            self.channel.close()
            return -1

        try:
            self.channel.queue_declare(self.myqueue, durable=False, auto_delete=True)
        except Exception as eobj:
            print eobj
            self.channel.close()
            return -1

        try:
            self.channel.queue_bind(self.myqueue, self.exchange, routing_key=self.mykey)
        except Exception as eobj:
            print eobj
            self.channel.close()
            return -1

    # end SimpleConfigure
    # -----------------------------------------------------------
    #
    # private
    #
    def _getChannelObject(self, channelList, chid):
        # channelList format:
        #   [ [<channel-id>, <channelObject>] ]
        ii = 0
        while ii < len(channelList):
            chpair = channelList[ii]
            if chid == chpair[0]:
                return chpair[1]

            ii = ii + 1

            # end while

        return None

    # end _getChannelObject
    # -----------------------------------------------------------

    def Configure(self, config=[1, "myexchange", "direct", "myqueue", "myq.myx"]):
        #
        # in: config format:
        #   [<channel>, <exchange>, <type>, <optional_queue>, <optional_routing_key>]
        #        0          1         2            3                4
        #
        # queue is not necessary for writes, only for reads
        # routing_key is used for filtering reads
        #
        # out: self.configList, self.chanList
        #   chanlList format:
        #   [ [<channel-id>, <channelObject>] ]
        #          0            1
        #
        #
        # entity-relationship for channels:
        #
        # 1 configuration : multiple channels
        # 1 channel : multiple exchanges
        # 1 exchange : 0,1 queue
        # 1 queue : 0,1 message tag
        #
        err_f = False
        channelList = self.chanList

        #
        # if channel does not exist then create channel
        # else skip create, use pre-existing channel in channelList
        #
        chobj = self._getChannelObject(channelList, config[0])
        if chobj == None:
            try:
                chobj = self.connection.channel()
            except Exception as eobj:
                print eobj
                err_f = True

        if not err_f:
            try:
                chobj.access_request("/data", active=True, write=True, read=True)
            except Exception as eobj:
                print eobj
                err_f = True
                channel.close()

        if not err_f:
            try:
                chobj.exchange_declare(exchange=config[1], type=config[2], durable=False, auto_delete=True)
            except Exception as eobj:
                print eobj
                print "The server is unhappy."
                print "You re-used an exchange: %s and changed the type parameter: %s." % (config[1], config[2])
                print "Closing the channel. Rename the exchange or kill and restart the server (server has memory)."
                chobj.close()
                err_f = True
        #
        # (type, queue, publish) = ('direct', routing key, match routing key)             -> yes delivery
        # (type, queue, publish) = ('direct', routing key, no or mismatched routing key) -> no
        # (type, queue, publish) = ('fanout', routing key, no or mismatched routing key) -> yes
        #
        if not err_f:
            if config[3] != None:  # queues are for reading from exchange
                try:
                    chobj.queue_declare(queue=config[3], durable=False, auto_delete=True)
                except Exception as eobj:
                    print eobj
                    err_f = True

                if not err_f:
                    try:
                        chobj.queue_bind(queue=config[3], exchange=config[1], routing_key=config[4])
                    except Exception as eobj:
                        print eobj
                        err_f = True

        newElement = []

        #
        # build object's channel list out of channel id from configuration list plus channel object
        # build objects's configuration list out of input list
        #
        if not err_f:
            newElement = [config[0]] + [chobj]
            self.chanList = self.chanList + [newElement]  # building list of lists

            self.configList = self.configList + [config]  # building list of lists

        return err_f

    # end Configure

    # -----------------------------------------------------------

    def ConfigureList(self, configSet):
        #
        # config format:
        #   [[<channel>, <exchange>, <type>, <optional_queue>, <optional_routing_key>], ...]
        #
        # channelList format:
        #   [ <channel-id>, <channelObject> ]
        #

        err_f = False

        ii = 0  # channelList is poplulated as channels are created
        while ii < len(configSet) and (not err_f):
            err_f = self.Configure(configSet[ii])
            if err_f:
                break

            ii = ii + 1
        # end while

        return err_f

    # end ConfigureList
    # -----------------------------------------------------------
    def SimplePut(self, msg_in):

        #
        # routing key is used for 'direct' exchanges
        #
        base_time = time.time()
        self.channel.basic_publish(amqp.Message(msg_in), self.exchange, self.mykey)
        #
        # (type, queue, publish) = ('direct', routing key, routing key)                  -> yes delivery
        # (type, queue, publish) = ('direct', routing key, no or mismatched routing key) -> no
        # (type, queue, publish) = ('fanout', routing key, no or mismatched routing key) -> yes
        #

        print "sec:", time.time() - base_time

    # end SimplePut
    # -----------------------------------------------------------
    def _searchListOfLists(
        self, listOfLists, index, key  # form: [ [...], [...],...]  # index into unwrapped list
    ):  # what to match
        match_f = False
        ii = 0

        while ii < len(listOfLists):
            theList = listOfLists[ii]
            if theList[index] == key:
                match_f = True
                break
            ii = ii + 1
        # end while

        return match_f, ii

    # end _searchListOfLists

    # -----------------------------------------------------------
    def Put(self, msg_in, exchange=None, routing_key=None):
        #
        # the purpose with the defaults is to allow the creation of simple communication links
        # with the minimal amount of programmatic effort (for testing), in parallel with
        # complex links that scale for large networks
        #
        err_f = False
        if exchange == None:
            exchange = self.exchange
            routing_key = self.mykey  # routing key is used for 'direct' exchanges
            ch_obj = self.channel
        else:
            # search the configure list for exchange, match index yields channel id and routing_key
            # search the channel list for channel id, match yields channel object

            # there are many ways for this function to fail by
            # passing in bad parameters
            try:
                match_f, e_index = self._searchListOfLists(self.configList, 1, exchange)
            except Exception as eobj:
                print eobj
                err_f = True
                return err_f

            if not match_f:
                print "Put(), exchange: %s not found in %s" % (exchange, self.configList)
                err_f = True
                return err_f

            chan_id = self.configList[e_index][0]
            routing_key = self.configList[e_index][4]

            # there are many ways for this function to fail by
            # passing in bad parameters
            try:
                match_f, c_index = self._searchListOfLists(self.chanList, 0, chan_id)
            except Exception as eobj:
                print eobj
                err_f = True
                return err_f

            if not match_f:
                print "Put(), channel %d not found in: %s" % (chan_id, self.chanList)
                err_f = True
                return err_f

            ch_obj = self.chanList[c_index][1]

        ch_obj.basic_publish(amqp.Message(msg_in), exchange, routing_key)

        return err_f

    # end Put

    # -----------------------------------------------------------
    def SimpleGet(self):
        try:
            response = self.channel.basic_get(self.myqueue, no_ack=False)
        except Exception as eobj:
            print self.myqueue, eobj
            response = None

        if response is not None:
            print 'ch: %d, body = "%s"' % (self.channel.channel_id, response.body)
            self.channel.basic_ack(response.delivery_tag)
        else:
            print "no message"

    # end SimpleGet
    # -----------------------------------------------------------
    def _queue2chobj(self, queue):
        ch_obj = None
        err_f = False
        #
        # given queue, search for channel id
        #
        try:
            match_f, q_index = self._searchListOfLists(self.configList, 3, queue)
        except Exception as eobj:
            print eobj
            err_f = True

        if not err_f:
            if not match_f:
                print "_queue2chobj(), queue %s not found in: %s" % (queue, self.configList)
                err_f = True

        if not err_f:
            chan_id = self.configList[q_index][0]

            # there are many ways for this function to fail by
            # passing in bad parameters
            try:
                match_f, c_index = self._searchListOfLists(self.chanList, 0, chan_id)
            except Exception as eobj:
                print eobj
                err_f = True

        if not err_f:
            if not match_f:
                print "_queue2chobj(), channel %d not found in: %s" % (chan_id, self.chanList)
                err_f = True

        if not err_f:
            ch_obj = self.chanList[c_index][1]

        return err_f, ch_obj

    # end _queue2chobj
    # -----------------------------------------------------------
    def Get(self, queue=None):
        err_f = False
        msg_f = False
        msg = None
        if queue == None:  # for simple configuration case
            queue = self.myqueue
            ch_obj = self.channel
        else:
            #
            # given queue, search for channel id
            #
            err_f, ch_obj = self._queue2chobj(queue)

            if not err_f:
                response = None
                try:
                    response = ch_obj.basic_get(queue, no_ack=False)
                except amqp.Timeout:
                    pass  # not an error
                except:
                    print "queue=", queue
                    print eobj
                    err_f = True

            if not err_f:
                if response is not None:
                    msg_f = True
                    msg = response.body
                    ch_obj.basic_ack(response.delivery_tag)
                else:
                    print "no data"

        return err_f, msg_f, msg

    # end Get
    # -----------------------------------------------------------
    #
    # private
    #
    def _mycallback(self, msg):
        print
        for key, val in msg.properties.items():
            print "%s: %s" % (key, str(val))
        for key, val in msg.delivery_info.items():
            print "> %s: %s" % (key, str(val))
        print "received <", msg.body, "> from channel #", msg.channel.channel_id
        msg.channel.basic_ack(msg.delivery_tag)

    # end __mycallback
    # -----------------------------------------------------------
    #
    # private
    #
    def _SimpleSubscribeThread(self):
        self.channel.basic_consume(queue=self.myqueue, callback=self._mycallback, no_ack=False)
        #
        # init loop
        #
        self.running = True
        #
        # loop - need a shared object (event) to determine external stop
        #
        while self.running:
            try:
                self.channel.wait(timeout=5)
            except amqp.Timeout:
                print "timeout"

    # end _SimpleSubscribeThread
    # -----------------------------------------------------------
    #
    # private
    #
    def _SubscribeThread(self):
        #
        # do once: get loop invarients
        #
        self.lockObj.acquire()
        if len(self.thread_params):
            #
            # note: it is very easy to mess up order when
            #       adding/changing parameters
            #
            evobj = self.thread_params.pop()  # 3
            callback = self.thread_params.pop()  # 2
            queue = self.thread_params.pop()  # 1

        self.lockObj.release()

        #
        # given queue, return channel
        #
        err_f, ch_obj = self._queue2chobj(queue)
        if err_f:
            print "Subscribe failed"
        else:
            ch_obj.basic_consume(queue=queue, callback=callback, no_ack=False)
            #
            # init loop
            #
            self.running = True
            #
            # loop - need a shared object (event) to determine external stop
            #
            while not self.ev_obj.is_set():  # while not signaled halt
                try:
                    ch_obj.wait(timeout=5)
                except amqp.Timeout:
                    if self.debug:
                        print "timeout"

    # end _SubscribeThread
    # -----------------------------------------------------------
    #
    def SimpleSubscribe(self):

        self.subscribe = KThread(target=self._SimpleSubscribeThread)
        self.subscribe.start()

    # end SimpleSubscribe

    # -----------------------------------------------------------
    #
    def Subscribe(self, queue=None, callback=None):
        #
        # the queue and callback being subscribed to is not passed in here
        # but passed in through via "self"
        # the question is how to make this scale?
        #
        #
        #

        self.lockObj.acquire()
        if queue == None:  # done for compatibility with "simple" version
            queue = self.myqueue
        if callback == None:
            callback = self._mycallback

        #
        # communication parameters within thread
        #
        self.thread_params.append(queue)  # 1
        self.thread_params.append(callback)  # 2
        #
        # control param for thread
        #
        self.ev_obj = Event()
        self.thread_params.append(self.ev_obj)  # 3 - top of stack

        subscribe = KThread(target=self._SubscribeThread)
        subscribe.start()
        self.lockObj.release()
        #
        # return thread and its control object
        #
        return subscribe, self.ev_obj

    # end Subscribe
    # -----------------------------------------------------------
    #
    def Unsubscribe(self, thread_object, event_object):
        #
        # object may no longer exists, so bound by "try"
        # also, belts and suspenders, i.e. signal before kill
        #
        try:
            event_object.set()  # signals to stop running
        except:
            print "attempting to signal a non-existent event"
        try:
            thread_object.kill()
        except Exception as eobj:
            print "attempting to signal a non-existent thread"

    # end SimpleUnsubscribe
    # -----------------------------------------------------------
    #
    def SimpleUnsubscribe(self):
        self.running = False
        try:
            self.subscribe.kill()  # might not have ever subscribed
        except Exception as eobj:
            pass

    # end SimpleUnsubscribe
    # -----------------------------------------------------------
    #
    def SimpleClose(self):
        self.SimpleUnsubscribe()
        # self.channel.close()     # sometimes hangs at sock.recv
        self.connection.close()  # connection close kills socket

    # end SimpleClose
    # -----------------------------------------------------------
    #
    def CloseChannel(self, chobj):
        # self.Unsubscribe(chobj)
        self.chobj.close()  # connection close kills socket

    # end Close
    # -----------------------------------------------------------
    #
    def CloseAll(self):
        ii = 0
        while ii < len(self.chanList):
            self.Close(self.chanList[ii][1])  # connection close kills socket
            ii = ii + 1
        # end while
        self.connection.close()
Пример #14
0
                            element['whatsapp_id'])
                        Awhatsapp.send_message(element['msg'])
                        break
                    except:
                        #print "IN except"
                        pass
        # A.start()
            flag = 1
        time.sleep(10)


# thread.start_new_thread(f1,())
# thread.start_new_thread(f2,())
A = KThread(target=f1)
B = KThread(target=f2)
A.start()
B.start()
# f1()
# def a1():
#     print "in a1"
#     # while True :
#     #     pass
#     print "Func stopped"
# def b1():
#     print "in b1"
#     global A
#     i=0
#     while i<5 :
#         print i
#         i+=1
#     A.kill()
Пример #15
0
class Server(object):
	def __init__(self, id_):
		self.id = id_
		self.config_file = 'config-%d' % self.id

		self.role = 'follower'
		self.commitIndex, self.lastApplied, self.leaderID = 0, 0, 0

		address = json.load(file('config.json'))
		self.initial_state = address['initial_state']
		self.addressbook = {}
		for id_ in address['running']:
			self.addressbook[id_] = address['AddressBook'][id_ - 1]

		# need to put it into file later on
		self.load()

		self.lastLogIndex = 0
		self.lastLogTerm, self.oldVotes, self.newVotes, self.numVotes = 0, 0, 0, 0

		self.port = self.addressbook[self.id]
		self.request_votes = self.peers[:]


		self.listener = KThread(target = self.listen, args= (acceptor,))
		self.listener.start()
		self.newPeers = []

	def listen(self, on_accept):
		print 'start listenning on port '+str(self.port)
		srv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
		srv.bind(("", self.port))
		while True:
			data, addr = srv.recvfrom(1024)
			KThread(target=on_accept, args=(self, data, addr)).start()
		srv.close()

	def follower(self):
		print 'Running as a follower'
		self.role = 'follower'
		first = True
		while True:
			self.last_update = time.time()
			election_timeout = 5 * random.random() + 5
			while time.time() - self.last_update <= election_timeout:
				pass
			if not first and self.election.is_alive():
				self.election.kill()
			first = False
			self.start_election()

	def start_election(self):
		self.role = 'candidate'
		self.election = KThread(target =self.thread_election,args = ())
		if len(self.peers) == 0:
			return
		self.currentTerm += 1
		self.votedFor = self.id
		serverConfig = ServerConfig(self.poolsize, self.currentTerm, self.votedFor, self.log, self.peers)
		with open(self.config_file, 'w') as f:
			pickle.dump(serverConfig, f)

		self.numVotes = 1
		self.election.start()

	def thread_election(self):
		print 'timouts, start a new election with term %d' % self.currentTerm
		self.role = 'candidate'
		self.request_votes = self.peers[:]

		while True:
			for peer in self.peers:
	 			if peer in self.request_votes:
	 				Msg = str(self.lastLogTerm) + ' ' + str(self.lastLogIndex)
	 				msg = BaseMessage(self.id, peer, self.currentTerm, Msg, reqtype="RequestVote")
	 				data = pickle.dumps(msg)
	 				sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
	 				sock.sendto(data, ("", self.addressbook[peer]))
 			time.sleep(1) # wait for servers to receive

 	def leader(self):
 		print 'Running as a leader'
 		self.role = 'leader'
 		self.nextIndex = {}
 		self.matchIndex = {}
 		for peer in self.peers:
 			self.nextIndex[peer] = len(self.log) + 1
 			self.matchIndex[peer] = 0
		self.append_entries()

	def append_entries(self):
		receipts = self.peers[:]
		while True:
			receipts = self.peers[:]
			for peer in receipts:
				if len(self.log) < self.nextIndex[peer]:
					entries = []
					prevLogIndex = len(self.log)
					prevLogTerm = 0
					if prevLogIndex != 0:
						prevLogTerm = self.log[prevLogIndex-1].term
				else:
					prevLogIndex = self.nextIndex[peer] - 1
					prevLogTerm = 0
					if prevLogIndex != 0:
						prevLogTerm = self.log[prevLogIndex-1].term
					entries = [self.log[prevLogIndex]]


				Msg = AppendEntriesMsg(self.id, peer, self.currentTerm, entries, self.commitIndex, prevLogIndex, prevLogTerm)
				data = pickle.dumps(Msg)
				sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
				sock.sendto(data, ("", self.addressbook[peer]))
			time.sleep(0.5)

	def step_down(self):
		if self.role == 'leader':
			self.leader_state.kill()
			self.follower_state = KThread(target = self.follower, args = ())
			self.follower_state.start()
		if self.role == 'candidate':
			print 'candidate step down when higher term'
			self.election.kill()
			self.last_update = time.time()
			self.role = 'follower'

	def load(self):
		try:
			with open(self.config_file) as f:
				serverConfig = pickle.load(f)
		except Exception as e:
			initial_running = [i+1 for i in range(5)]
			initial_running.remove(self.id)
			serverConfig = ServerConfig(100, 0, -1, [], initial_running)

		self.currentTerm = serverConfig.currentTerm
		self.poolsize = serverConfig.poolsize
		self.votedFor = serverConfig.votedFor
		self.peers = serverConfig.peers
		self.majority = (len(self.peers) + 1)/2 + 1
		self.log = serverConfig.log

	def run(self):
		time.sleep(1)
		self.follower_state = KThread(target = self.follower, args = ())
		self.follower_state.start()
Пример #16
0
class Server(object):
    def __init__(self, id_):
        self.id = id_
        self.config_file = 'config-%d' % self.id

        #self.load()
        self.role = 'follower'
        self.commitIndex = 0
        self.lastApplied = 0

        self.leaderID = 0

        address = json.load(file('config.json'))
        port_list = address['AddressBook']
        running = address['running']
        self.initial_state = address['initial_state']
        self.addressbook = {}
        for id_ in running:
            self.addressbook[id_] = port_list[id_ - 1]

        # need to put it into file later on
        self.load()

        self.port = self.addressbook[self.id]

        # self.nextIndex = {}
        # 	self.matchIndex = {}
        # 	for peer in self.peers:
        # 		self.nextIndex[peer] = len(self.log) + 1
        # 		self.matchIndex[peer] = 0

        self.request_votes = self.peers[:]

        self.numVotes = 0
        self.oldVotes = 0
        self.newVotes = 0

        self.lastLogIndex = 0
        self.lastLogTerm = 0

        self.listener = KThread(target=self.listen, args=(acceptor, ))
        self.listener.start()

        self.during_change = 0
        self.newPeers = []
        self.new = None
        self.old = None

    def listen(self, on_accept):
        srv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        srv.bind(("", self.port))
        print 'start listenning'
        while True:
            data, addr = srv.recvfrom(1024)
            thr = KThread(target=on_accept, args=(self, data, addr))
            thr.start()
        srv.close()

    def follower(self):
        print 'Running as a follower'
        self.role = 'follower'
        self.last_update = time.time()
        election_timeout = 5 * random.random() + 5
        while time.time() - self.last_update <= election_timeout:
            pass
        self.start_election()
        while True:
            self.last_update = time.time()
            election_timeout = 5 * random.random() + 5
            while time.time() - self.last_update <= election_timeout:
                pass

            if self.election.is_alive():
                self.election.kill()
            self.start_election()

    def start_election(self):
        self.role = 'candidate'
        self.election = KThread(target=self.thread_election, args=())
        if len(self.peers) != 0:
            self.currentTerm += 1
            self.votedFor = self.id
            self.save()
            self.numVotes = 1
            if self.during_change == 1:
                self.newVotes = 0
                self.oldVotes = 0
                if self.id in self.new:
                    self.newVotes = 1
                if self.id in self.old:
                    self.oldVotes = 1
            elif self.during_change == 2:
                self.newVotes = 0
                if self.id in self.new:
                    self.newVotes = 1
            self.election.start()

    def thread_election(self):
        print 'timouts, start a new election with term %d' % self.currentTerm
        self.role = 'candidate'
        self.request_votes = self.peers[:]
        sender = self.id

        while 1:
            # print 'Send vote request to ', self.request_votes
            for peer in self.peers:
                if peer in self.request_votes:
                    Msg = str(self.lastLogTerm) + ' ' + str(self.lastLogIndex)
                    msg = RequestVoteMsg(sender, peer, self.currentTerm, Msg)
                    data = pickle.dumps(msg)
                    sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                    sock.sendto(data, ("", self.addressbook[peer]))
            time.sleep(1)  # wait for servers to receive

    def leader(self):
        print 'Running as a leader'
        self.role = 'leader'
        self.nextIndex = {}
        self.matchIndex = {}
        for peer in self.peers:
            self.nextIndex[peer] = len(self.log) + 1
            self.matchIndex[peer] = 0
        self.append_entries()

    def append_entries(self):

        receipts = self.peers[:]
        while 1:
            receipts = self.peers[:]
            if self.during_change != 0:
                for peer in receipts:
                    if peer not in self.nextIndex:
                        self.nextIndex[peer] = len(self.log) + 1
                        self.matchIndex[peer] = 0
            for peer in receipts:
                if len(self.log) >= self.nextIndex[peer]:
                    prevLogIndex = self.nextIndex[peer] - 1
                    if prevLogIndex != 0:
                        prevLogTerm = self.log[prevLogIndex - 1].term
                    else:
                        prevLogTerm = 0
                    entries = [self.log[self.nextIndex[peer] - 1]]
                else:
                    entries = []
                    prevLogIndex = len(self.log)
                    if prevLogIndex != 0:
                        prevLogTerm = self.log[prevLogIndex - 1].term
                    else:
                        prevLogTerm = 0

                Msg = AppendEntriesMsg(self.id, peer, self.currentTerm,
                                       entries, self.commitIndex, prevLogIndex,
                                       prevLogTerm)
                data = pickle.dumps(Msg)
                sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
                sock.sendto(data, ("", self.addressbook[peer]))
            time.sleep(0.5)

    def step_down(self):
        if self.role == 'candidate':
            print 'candidate step down when higher term'
            self.election.kill()
            self.last_update = time.time()
            self.role = 'follower'
        elif self.role == 'leader':
            self.leader_state.kill()
            self.follower_state = KThread(target=self.follower, args=())
            self.follower_state.start()

    def load(self):
        initial_running = [1, 2, 3]
        # new_quorom = []

        try:
            with open(self.config_file) as f:
                serverConfig = pickle.load(f)
        except Exception as e:
            if self.id not in initial_running:
                serverConfig = ServerConfig(100, 0, -1, [], [])
            else:
                initial_running.remove(self.id)
                serverConfig = ServerConfig(
                    100, 0, -1, [], initial_running
                )  #Server config = obiect de stocare a datelor despre server

        self.poolsize = serverConfig.poolsize
        self.currentTerm = serverConfig.currentTerm
        self.votedFor = serverConfig.votedFor
        self.log = serverConfig.log
        self.peers = serverConfig.peers
        self.majority = (len(self.peers) + 1) / 2 + 1
        # self.new_quorom = new_quorom
        #self.majority_1 = (len(self.new_quorom) + 1)/2 + 1

    def save(self):
        serverConfig = ServerConfig(self.poolsize, self.currentTerm,
                                    self.votedFor, self.log, self.peers)
        with open(self.config_file, 'w') as f:
            pickle.dump(serverConfig, f)

    def run(self):
        time.sleep(1)
        self.follower_state = KThread(target=self.follower, args=())
        self.follower_state.start()
Пример #17
0
import os
from KThread import *

def func():
  print 'Function started'
  os.system('java -jar ' + 'TournamentServer-GUI/gui/test/lib/jstestdriver/JsTestDriver.jar --port 9876')
  print 'Function finished'

A = KThread(target=func)
A.start()
A.kill()

print 'End of main program'
Пример #18
0
class MyForm(wx.Frame):


    #----------------------------------------------------------------------
    def __init__(self):
        wx.Frame.__init__(self, None, wx.ID_ANY, "NVIDIA LTE Throughput Tester",size=(1250,800))

        self.panel = wx.Panel(self, wx.ID_ANY)
        self.pl = wx.Panel(self.panel)
        #self.pl = wx.Pane()
        self.pl.SetBackgroundColour("#d8d8bf")

        self.Font_Result = wx.Font(12,wx.FONTFAMILY_DEFAULT,wx.FONTSTYLE_NORMAL,wx.FONTWEIGHT_BOLD)

        #self.pl2 = wx.Panel(self, wx.ID_ANY)

        #self.pl = wx.Panel(self, -1)
        #self.pl.SetBackgroundColour("#EFFEFE")

        # TestType Part
        sbTestType = wx.StaticBox(self.pl, -1, 'Test Type', size=(-1, -1))
        #sbTestType.SetForegroundColour(wx.BLUE)
        sbsTestType = wx.StaticBoxSizer(sbTestType, wx.HORIZONTAL)
        #self.stTestType = wx.StaticText(self.pl,-1,'Test Type')
        #sbsTestType.Add(self.stTestType, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsTestType = wx.BoxSizer ( wx.HORIZONTAL )
        self.rbNotest = wx.RadioButton(self.pl, -1, 'No Test')
        self.Bind(wx.EVT_RADIOBUTTON, self.TestType, id=self.rbNotest.GetId())
        self.rbAuto   = wx.RadioButton(self.pl, -1, 'Auto Test')
        self.Bind(wx.EVT_RADIOBUTTON, self.TestType, id=self.rbAuto.GetId())
        self.rbUnit = wx.RadioButton(self.pl, -1, 'Unit Test')
        self.Bind(wx.EVT_RADIOBUTTON, self.TestType, id=self.rbUnit.GetId())

       # bsTestType.Add(self.stTestType,0, wx.ALL,10)
        bsTestType.Add(self.rbAuto,0, wx.ALL,10)
        bsTestType.Add(self.rbUnit,0, wx.ALL,10)
        bsTestType.Add(self.rbNotest,0, wx.ALL,10)
        sbsTestType.Add(bsTestType, 0, wx.LEFT,10)

        #platform
        #sbPlatformType = wx.StaticBox(self.pl, -1, 'Platform', size=(-1, -1))
        #sbsPlatformType = wx.StaticBoxSizer(sbPlatformType, wx.HORIZONTAL)
        #bsPlatformType = wx.BoxSizer ( wx.HORIZONTAL )
        self.rbplatform = wx.RadioBox(self.pl, -1, "Platform", choices=["Win XP", "WoA"], majorDimension=0, style=wx.RA_SPECIFY_COLS)

        #self.rbWinxp = wx.RadioButton(self.pl, -1, 'Win XP')
        #self.Bind(wx.EVT_RADIOBUTTON, self.PlatformType, id=self.rbWinxp.GetId())
        #self.rbWoA = wx.RadioButton(self.pl, -1, 'WoA')
        #self.Bind(wx.EVT_RADIOBUTTON, self.PlatformType, id=self.rbWoA.GetId())
        #bsPlatformType.Add(self.rbWinxp,0, wx.ALL,10)
        #bsPlatformType.Add(self.rbplatform,0, wx.ALL,10)
        #sbsPlatformType.Add(bsPlatformType, 0, wx.LEFT,10)
        # BranchSelection Part
        sbBranchSelection = wx.StaticBox(self.pl, -1, 'BranchSelection', size=(-1, -1))
        #sbBranchSelection.SetForegroundColour(wx.BLUE)
        sbsBranchSelection = wx.StaticBoxSizer(sbBranchSelection, wx.HORIZONTAL)
        #self.stBranchSelection = wx.StaticText(self.pl,-1,'BranchSelection')
       # sbsBranchSelection.Add(self.stBranchSelection, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsBranchSelection = wx.BoxSizer ( wx.HORIZONTAL )
        self.mainBr = wx.CheckBox(self.pl, -1 ,'main')
        self.cr3Br = wx.CheckBox(self.pl, -1 ,'cr3')
        self.FTBr = wx.CheckBox(self.pl, -1 ,'Sanity Test')
        bsBranchSelection.Add(self.mainBr,0, wx.ALL,10)
        bsBranchSelection.Add(self.cr3Br,0, wx.ALL,10)
        bsBranchSelection.Add(self.FTBr,0, wx.ALL,10)
        sbsBranchSelection.Add(bsBranchSelection, 0, wx.LEFT,10)


        # SelectScenario Part
        sbSelectScenario = wx.StaticBox(self.pl, -1, 'SelectScenario', size=(-1, -1))
        #sbSelectScenario.SetForegroundColour(wx.BLUE)
        sbsSelectScenario = wx.StaticBoxSizer(sbSelectScenario, wx.VERTICAL)
        #self.stSelectScenario = wx.StaticText(self.pl,-1,'SelectScenario')
       # sbsSelectScenario.Add(self.stSelectScenario, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsSelectScenario = wx.BoxSizer ( wx.VERTICAL )
        bsSelectScenariox = []
        bsSelectScenariox.append(wx.BoxSizer ( wx.HORIZONTAL ))
        bsSelectScenariox.append(wx.BoxSizer ( wx.HORIZONTAL ))
        self.scen = []



        for i in range(0,len(scenario_implemented)):
            if re.search('MIMO',scenario_implemented[i]):
                self.scen.append(wx.CheckBox(self.pl, -1 ,scenario_implemented[i]))
                #self.scen[i].SetFont(font)
                bsSelectScenariox[1].Add(self.scen[i],0, wx.ALL,1)
            else:
                self.scen.append(wx.CheckBox(self.pl, -1 ,scenario_implemented[i]))
                #self.scen[i].SetFont(font)
                bsSelectScenariox[0].Add(self.scen[i],0, wx.ALL,1)

        self.Alltest = wx.CheckBox(self.pl, -1 ,"All")
        bsSelectScenariox[1].Add(self.Alltest)

        for i in range(0,len(bsSelectScenariox)):
            bsSelectScenario.Add(bsSelectScenariox[i])

        sbsSelectScenario.Add(bsSelectScenario, 0, wx.LEFT,5)

        # BandSelection Part
        sbBandSelection = wx.StaticBox(self.pl, -1, 'BandSelection', size=(-1, -1))
        #sbBandSelection.SetForegroundColour(wx.BLUE)
        sbsBandSelection = wx.StaticBoxSizer(sbBandSelection, wx.VERTICAL)
        #self.stBandSelection = wx.StaticText(self.pl,-1,'BandSelection')
       # sbsBandSelection.Add(self.stBandSelection, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsBandSelection = wx.BoxSizer ( wx.HORIZONTAL )
        self.band4 = wx.CheckBox(self.pl, -1 ,'4')
        self.band17 = wx.CheckBox(self.pl, -1 ,'17')
        bsBandSelection.Add(self.band4,0, wx.ALL,10)
        bsBandSelection.Add(self.band17,0, wx.ALL,10)
        sbsBandSelection.Add(bsBandSelection, 0, wx.CENTER,10)

        # AdditionalOption Part
        sbAdditionOption = wx.StaticBox(self.pl, -1, 'AdditionOption', size=(-1, -1))
        #sbAdditionOption.SetForegroundColour(wx.BLUE)
        sbsAdditionOption = wx.StaticBoxSizer(sbAdditionOption, wx.VERTICAL)
        #self.stAdditionOption = wx.StaticText(self.pl,-1,'AdditionOption')
       # sbsAdditionOption.Add(self.stAdditionOption, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsAdditionOption = wx.BoxSizer ( wx.HORIZONTAL )
        self.stchangelist = wx.StaticText(self.pl,-1,' CL ')

        self.changelist = wx.TextCtrl(self.pl,-1,size=(60,20), value=u"")


        #self.build = wx.CheckBox(self.pl, -1 ,'Build')
        self.flash = wx.CheckBox(self.pl, -1 ,'Flash')
        #self.graph = wx.CheckBox(self.pl, -1 ,'Graph')
        self.force = wx.CheckBox(self.pl, -1 ,'Force')
        self.cReg = wx.CheckBox(self.pl, -1 ,'Reg')
        #self.cReg.SetValue(True)
        #self.stNbComp = wx.StaticText(self.pl,-1,'Nb.Comparisons')
        #self.tcNbComp = wx.TextCtrl(self.pl,-1,size=(20,20), value=u"")
        bsAdditionOption.Add(self.stchangelist,0, wx.ALL,10)
        bsAdditionOption.Add(self.changelist,0, wx.ALL,10)
        #self.stOk_cl = wx.StaticText(self.pl,-1,'Ref_CL')
        #self.tOk_cl = wx.TextCtrl(self.pl,-1,size=(60,20), value=u"")
        #bsAdditionOption.Add(self.stOk_cl,0, wx.ALL,10)
        #bsAdditionOption.Add(self.tOk_cl,0, wx.ALL,10)
        #self.stKo_cl = wx.StaticText(self.pl,-1,'Reg_CL')
        #self.tKo_cl = wx.TextCtrl(self.pl,-1,size=(60,20), value=u"")
        #bsAdditionOption.Add(self.stKo_cl,0, wx.ALL,10)
       # bsAdditionOption.Add(self.tKo_cl,0, wx.ALL,10)

        #bsAdditionOption.Add(self.build,0, wx.ALL,10)
        bsAdditionOption.Add(self.flash,0, wx.ALL,10)
        #bsAdditionOption.Add(self.graph,0, wx.ALL,10)
        bsAdditionOption.Add(self.force,0, wx.ALL,10)
        bsAdditionOption.Add(self.cReg,0, wx.ALL,10)
        #bsAdditionOption.Add(self.stNbComp,0, wx.ALL,10)
        #bsAdditionOption.Add(self.tcNbComp,0, wx.ALL,10)
        sbsAdditionOption.Add(bsAdditionOption, 0, wx.LEFT,10)

        # StartSection Part
        sbStartSection = wx.StaticBox(self.pl, -1, 'Execution', size=(-1, -1))
        #sbStartSection.SetForegroundColour(wx.BLUE)
        sbsStartSection = wx.StaticBoxSizer(sbStartSection, wx.VERTICAL)
        #self.stStartSection = wx.StaticText(self.pl,-1,'StartSection')
       # sbsStartSection.Add(self.stStartSection, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsStartSection = wx.BoxSizer ( wx.HORIZONTAL )
        bStart = wx.Button(self.pl,-1,label="Start")
        bStop = wx.Button(self.pl,-1,label="Stop")
        bResult = wx.Button(self.pl,-1,label="Result")
        bReg = wx.Button(self.pl,-1,label="REG")
        bSetting = wx.Button(self.pl,-1,label="Setting")
        self.Bind(wx.EVT_BUTTON, self.onRun, bStart)
        bsStartSection.Add(bStart,0, wx.ALL,10)
        self.Bind(wx.EVT_BUTTON, self.stopThread, bStop)
        bsStartSection.Add(bStop,0, wx.ALL,10)
        self.Bind(wx.EVT_BUTTON, self._onShowResult, bResult)
        bsStartSection.Add(bResult,0, wx.ALL,10)
        self.Bind(wx.EVT_BUTTON, self._onShowOptions, bSetting)
        self.Bind(wx.EVT_BUTTON, self.Regression_Start, bReg)
        bsStartSection.Add(bReg,0, wx.ALL,10)
        bsStartSection.Add(bSetting,0, wx.ALL,10)
        sbsStartSection.Add(bsStartSection, 0, wx.LEFT,10)


        # FlashSection Part
        sbFlashSection = wx.StaticBox(self.pl, -1, 'Flash', size=(-1, -1))
        #sbFlashSection.SetForegroundColour(wx.BLUE)
        sbsFlashSection = wx.StaticBoxSizer(sbFlashSection, wx.VERTICAL)
        #self.stFlashSection = wx.StaticText(self.pl,-1,'FlashSection')
       # sbsFlashSection.Add(self.stFlashSection, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsFlashSection = wx.BoxSizer ( wx.HORIZONTAL )
        #self.entry = wx.TextCtrl(self.pl,-1,size=(600,20))
        self.entry = wx.TextCtrl(self.pl,-1,r"",wx.Point(20,20), wx.Size(500,20), \
                wx.TE_MULTILINE | wx.TE_RICH2)

        self.Bind(wx.EVT_TEXT_ENTER, self.OnPressEnter, self.entry)
        bFile = wx.Button(self.pl,-1,label="File")
        self.Bind(wx.EVT_BUTTON, self.OnOpenScen, bFile)
        bFlash = wx.Button(self.pl,-1,label="Flash")
        self.Bind(wx.EVT_BUTTON,self.flash_only,bFlash)

        bsFlashSection.Add(self.entry,0, wx.ALL,10)
        bsFlashSection.Add(bFile,0, wx.ALL,10)
        bsFlashSection.Add(bFlash,0, wx.ALL,10)
        sbsFlashSection.Add(bsFlashSection, 0, wx.LEFT,10)

        # LogSection Part
        sbLogSection = wx.StaticBox(self.pl, -1, 'LogSection', size=(-1, -1))
        #sbLogSection.SetForegroundColour(wx.BLUE)
        sbsLogSection = wx.StaticBoxSizer(sbLogSection, wx.VERTICAL)
        #self.stLogSection = wx.StaticText(self.pl,-1,'LogSection')
       # sbsLogSection.Add(self.stLogSection, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsLogSection = wx.BoxSizer ( wx.HORIZONTAL )
        self.log = wx.TextCtrl(self.pl, wx.ID_ANY, size=(1150,300),style = wx.TE_MULTILINE|wx.TE_READONLY|wx.HSCROLL)
        #self.msg = wx.TextCtrl(self.pl, wx.ID_ANY, size=(800,100),style = wx.TE_MULTILINE|wx.TE_READONLY|wx.HSCROLL)

        bsLogSection.Add(self.log,1,wx.ALL,10)
        #bsLogSection.Add(self.msg,1,wx.ALL,10)

        sbsLogSection.Add(bsLogSection, 0, wx.LEFT,10)



        #TEST TYPE BRANCH BAND JOINING
        bsTTBS = wx.BoxSizer ( wx.HORIZONTAL)
        bsTTBS.Add(sbsTestType, 0, wx.EXPAND)
        bsTTBS.Add(sbsBranchSelection, 0, wx.EXPAND)
        bsTTBS.Add(sbsBandSelection, 0, wx.EXPAND)
        bsTTBS.Add(sbsAdditionOption, 0, wx.EXPAND)
        #bsTTBS.Add(sbsPlatformType, 0, wx.EXPAND)
        bsTTBS.Add(self.rbplatform, 0, wx.EXPAND)

        #StartSection and Flash Joining
        bsSSFS = wx.BoxSizer ( wx.HORIZONTAL)
        bsSSFS.Add(sbsStartSection, 0, wx.EXPAND)
        bsSSFS.Add(sbsFlashSection, 0, wx.EXPAND)


        # Fill the Frame
        self.sizer = wx.BoxSizer ( wx.VERTICAL)
        self.sizer.Add(bsTTBS, 0, wx.EXPAND)
        # sizer.Add(sbsTestType, 0, wx.EXPAND)
        # sizer.Add(sbsBranchSelection, 0, wx.EXPAND)
        self.sizer.Add(sbsSelectScenario, 0, wx.EXPAND)
        #sizer.Add(sbsBandSelection, 0, wx.EXPAND)
        #sizer.Add(sbsAdditionOption, 0, wx.EXPAND)
        self.sizer.Add(bsSSFS,0,wx.EXPAND)
        #self.sizer.Add(sbsStartSection, 0, wx.EXPAND)
        self.sizer.Add(sbsLogSection, 0, wx.EXPAND)

        self.addProgress()
        self.pl.SetSizer(self.sizer)
        self.Center()
        self.Show(True)

        self._resultPanel()
        self.Rsizer.Layout()
        self._Additional_options()
        self.Osizer.Layout()
        self.optionPanel.SetSize(self.GetClientSizeTuple())
        self.resultPanel.SetSize(self.GetClientSizeTuple())
        self.pl.SetSize(self.GetClientSizeTuple())

        #self._onShowMain(None)

        self.redir=RedirectText(self.log)
        sys.stdout=self.redir
        sys.stderr=self.redir


        #image_file = 'test.jpeg'
        #bmp1 = wx.Image(image_file, wx.BITMAP_TYPE_ANY).ConvertToBitmap()
        #self.bitmap1 = wx.StaticBitmap(self, -1, bmp1, (0, 0))

        #self.Refresh_Frame()

        self.flash_list = []

    def _resultPanel(self):

        #RESULT PANEL
        #self.resultPanel = wx.Panel(self.panel)
        self.resultPanel = ScrolledPanel(self.panel)
        self.resultPanel.SetupScrolling()

        # TestType Part
        sbTestType = wx.StaticBox(self.resultPanel, -1, 'Test Type', size=(-1, -1))
        #sbTestType.SetForegroundColour(wx.BLUE)
        sbsTestType = wx.StaticBoxSizer(sbTestType, wx.HORIZONTAL)
        #self.stTestType = wx.StaticText(self.resultPanel,-1,'Test Type')
        #sbsTestType.Add(self.stTestType, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsTestType = wx.BoxSizer ( wx.HORIZONTAL )
        self.rbMainR = wx.RadioButton(self.resultPanel, -1, 'Main')
        self.Bind(wx.EVT_RADIOBUTTON, self.TestType, id=self.rbMainR.GetId())
        self.rbCr3R   = wx.RadioButton(self.resultPanel, -1, 'Cr3')
        self.Bind(wx.EVT_RADIOBUTTON, self.TestType, id=self.rbCr3R.GetId())
        self.rbStR = wx.RadioButton(self.resultPanel, -1, 'ST')
        self.Bind(wx.EVT_RADIOBUTTON, self.TestType, id=self.rbStR.GetId())


       # bsTestType.Add(self.stTestType,0, wx.ALL,10)
        bsTestType.Add(self.rbCr3R,0, wx.ALL,10)
        bsTestType.Add(self.rbStR,0, wx.ALL,10)
        bsTestType.Add(self.rbMainR,0, wx.ALL,10)
        sbsTestType.Add(bsTestType, 0, wx.LEFT,10)

        # SelectScenario Part
        sbSelectScenario = wx.StaticBox(self.resultPanel, -1, 'SelectScenario', size=(-1, -1))
        #sbSelectScenario.SetForegroundColour(wx.BLUE)
        sbsSelectScenario = wx.StaticBoxSizer(sbSelectScenario, wx.VERTICAL)
        #self.stSelectScenario = wx.StaticText(self.resultPanel,-1,'SelectScenario')
       # sbsSelectScenario.Add(self.stSelectScenario, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)


        bsSelectScenario = wx.BoxSizer ( wx.VERTICAL )
        bsSelectScenariox = []
        bsSelectScenariox.append(wx.BoxSizer ( wx.HORIZONTAL ))
        bsSelectScenariox.append(wx.BoxSizer ( wx.HORIZONTAL ))
        self.scenR = []

        font = wx.Font(7,wx.FONTFAMILY_DEFAULT,wx.FONTSTYLE_NORMAL,wx.FONTWEIGHT_LIGHT)

        for i in range(0,len(scenario_implemented)):
            if re.search('MIMO',scenario_implemented[i]):
                self.scenR.append(wx.CheckBox(self.resultPanel, -1 ,scenario_implemented[i]))
                #self.scen[i].SetFont(font)
                bsSelectScenariox[1].Add(self.scenR[i],0, wx.ALL,1)
            else:
                self.scenR.append(wx.CheckBox(self.resultPanel, -1 ,scenario_implemented[i]))
                #self.scen[i].SetFont(font)
                bsSelectScenariox[0].Add(self.scenR[i],0, wx.ALL,1)

        self.AlltestR = wx.CheckBox(self.resultPanel, -1 ,"All")
        bsSelectScenariox[1].Add(self.AlltestR)

        for i in range(0,len(bsSelectScenariox)):
            bsSelectScenario.Add(bsSelectScenariox[i])

        sbsSelectScenario.Add(bsSelectScenario, 0, wx.LEFT,5)







##        bsSelectScenario = wx.BoxSizer ( wx.HORIZONTAL )
##        self.scenR = []
##        for i in range(0,len(scenario_implemented)):
##            self.scenR.append(wx.CheckBox(self.resultPanel, -1 ,scenario_implemented[i]))
##            bsSelectScenario.Add(self.scenR[i],0, wx.ALL,1)
##        sbsSelectScenario.Add(bsSelectScenario, 0, wx.LEFT,5)

        # BandSelection Part
        sbBandSelection = wx.StaticBox(self.resultPanel, -1, 'BandSelection', size=(-1, -1))
        #sbBandSelection.SetForegroundColour(wx.BLUE)
        sbsBandSelection = wx.StaticBoxSizer(sbBandSelection, wx.VERTICAL)
        #self.stBandSelection = wx.StaticText(self.resultPanel,-1,'BandSelection')
       # sbsBandSelection.Add(self.stBandSelection, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsBandSelection = wx.BoxSizer ( wx.HORIZONTAL )
        self.band4R = wx.CheckBox(self.resultPanel, -1 ,'4')
        self.band17R = wx.CheckBox(self.resultPanel, -1 ,'17')
        bsBandSelection.Add(self.band4R,0, wx.ALL,10)
        bsBandSelection.Add(self.band17R,0, wx.ALL,10)
        sbsBandSelection.Add(bsBandSelection, 0, wx.CENTER,10)

        # GraphOption Part
        sbAdditionOption = wx.StaticBox(self.resultPanel, -1, 'GraphOption', size=(-1, -1))
        #sbAdditionOption.SetForegroundColour(wx.BLUE)
        sbsAdditionOption = wx.StaticBoxSizer(sbAdditionOption, wx.VERTICAL)
        #self.stAdditionOption = wx.StaticText(self.resultPanel,-1,'AdditionOption')
       # sbsAdditionOption.Add(self.stAdditionOption, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsAdditionOption = wx.BoxSizer ( wx.HORIZONTAL )
        self.stchangelistR = wx.StaticText(self.resultPanel,-1,' CL ')
        self.forceGraph = wx.CheckBox(self.resultPanel, -1 ,'Force')

        self.changelistR = wx.TextCtrl(self.resultPanel,-1,size=(60,20), value=u"")
        #self.Bind(wx.EVT_TEXT_ENTER, self.OnPressEnter, self.changelistR)
        self.stNbCompR = wx.StaticText(self.resultPanel,-1,'Nb.Comparisons')
        self.tcNbCompR = wx.TextCtrl(self.resultPanel,-1,size=(20,20), value=u"")
        bsAdditionOption.Add(self.stchangelistR,0, wx.ALL,10)
        bsAdditionOption.Add(self.changelistR,0, wx.ALL,10)
        bsAdditionOption.Add(self.stNbCompR,0, wx.ALL,10)
        bsAdditionOption.Add(self.tcNbCompR,0, wx.ALL,10)
        bsAdditionOption.Add(self.forceGraph,0, wx.ALL,10)

        sbsAdditionOption.Add(bsAdditionOption, 0, wx.LEFT,10)

        # StartSection Part
        sbStartSection = wx.StaticBox(self.resultPanel, -1, '', size=(-1, -1))
        #sbStartSection.SetForegroundColour(wx.BLUE)
        sbsStartSection = wx.StaticBoxSizer(sbStartSection, wx.VERTICAL)
        #self.stStartSection = wx.StaticText(self.resultPanel,-1,'StartSection')
       # sbsStartSection.Add(self.stStartSection, 0, wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
        bsStartSection = wx.BoxSizer ( wx.HORIZONTAL )
        bStart = wx.Button(self.resultPanel,-1,label="Generate")
        bStop = wx.Button(self.resultPanel,-1,label="Clear")
        bResult = wx.Button(self.resultPanel,-1,label="Main Window")
        self.Bind(wx.EVT_BUTTON, self._updateResultPanel, bStart)
        bsStartSection.Add(bStart,0, wx.ALL,10)
        self.Bind(wx.EVT_BUTTON, self._cleanResultPanel, bStop)
        bsStartSection.Add(bStop,0, wx.ALL,10)
        self.Bind(wx.EVT_BUTTON, self._onShowMain, bResult)
        bsStartSection.Add(bResult,0, wx.ALL,10)
        sbsStartSection.Add(bsStartSection, 0, wx.CENTER,10)

        #TEST TYPE BRANCH BAND JOINING
        bsTTBS = wx.BoxSizer ( wx.HORIZONTAL)
        bsTTBS.Add(sbsTestType, 0, wx.EXPAND)
        bsTTBS.Add(sbsBandSelection, 0, wx.EXPAND)
        bsTTBS.Add(sbsAdditionOption, 0, wx.EXPAND)

        #IMAGE
##        self.img = []
##        self.caption = []
##        gridSizer = wx.GridSizer(rows=20, cols=1)
##
##        InsideBoxer =[]
##        #self.stcHelp = wx.StaticText(self.resultPanel, label="help help help\n"*8)
##        #btn = wx.Button(self.resultPanel, label="close[x]")
##        #btn.Bind(wx.EVT_BUTTON, self._onShowMain)
##        sizer2 = wx.BoxSizer(wx.VERTICAL)
##        sizer2.Add((20,20),proportion=1)
##       # sizer2.Add(self.stcHelp)
##        for i in range(0,len(BAND_TANGO_ALLOWED)):
##            for j in range(0,len(scenario_implemented)):
##                InsideBoxer.append(wx.BoxSizer(wx.VERTICAL))
##                self.caption.append(wx.StaticText(self.resultPanel,-1," "))
##                self.img.append(wx.StaticBitmap(self.resultPanel, wx.ID_ANY, wx.BitmapFromImage(wx.EmptyImage(240,240))))
##                InsideBoxer[i*len(scenario_implemented)+j].Add(self.caption[i*len(scenario_implemented)+j],-1,wx.ALL|wx.ALIGN_CENTER_VERTICAL,10)
##
##                InsideBoxer[i*len(scenario_implemented)+j].Add(self.img[i*len(scenario_implemented)+j])
##                gridSizer.Add(InsideBoxer[i*len(scenario_implemented)+j])

        #gridSizer.Add(btn)

        #gridSizer.Add(sizer2, 0, wx.ALIGN_RIGHT)

        # Fill the Frame
        self.Rsizer = wx.BoxSizer ( wx.VERTICAL)
        self.Rsizer.Add(bsTTBS, 0, wx.EXPAND)
        self.Rsizer.Add(sbsSelectScenario, 0, wx.EXPAND)
        self.Rsizer.Add(sbsStartSection, 0, wx.EXPAND)
        #self.Rsizer.Add((5,5), proportion=1)
       # self.Rsizer.Add(gridSizer, 0, wx.EXPAND)

        self.resultPanel.SetSizer(self.Rsizer)
        self.resultPanel.Hide()
        self.resultPanel.Raise()
        #self.resultPanel.SetBackgroundColour((240,250,240))
        self.Bind(wx.EVT_SIZE, self._onSize)

    def _Additional_options(self):
        self.optionPanel = ScrolledPanel(self.panel)
        self.optionPanel.SetupScrolling()

        # Options Part
        sboptions = wx.StaticBox(self.optionPanel, -1, 'Options', size=(-1, -1))
        sbsoptions = wx.StaticBoxSizer(sboptions, wx.HORIZONTAL)
        bsoptions = wx.BoxSizer ( wx.HORIZONTAL )

        #AT PORT
        satport = wx.StaticText(self.optionPanel,-1,'AT PORT')
        bsoptions.Add(satport,0, wx.ALL,10)
        self.atport = wx.TextCtrl(self.optionPanel,-1,size=(60,20), value=u"")
        bsoptions.Add(self.atport,0, wx.ALL,10)
        #self.Bind(wx.EVT_RADIOBUTTON, self.options, id=self.rbStR.GetId())

        #MODEM PORT
        smodemport = wx.StaticText(self.optionPanel,-1,'MODEM PORT')
        bsoptions.Add(smodemport,0, wx.ALL,10)
        self.modemport = wx.TextCtrl(self.optionPanel,-1,size=(60,20), value=u"")
        bsoptions.Add(self.modemport,0, wx.ALL,10)

        #VID
        svid = wx.StaticText(self.optionPanel,-1,'VID ')
        bsoptions.Add(svid,0, wx.ALL,10)
        self.vid_no = wx.TextCtrl(self.optionPanel,-1,size=(60,20), value=u"")
        bsoptions.Add(self.vid_no,0, wx.ALL,10)
        #NEXT WIDGET

        #END

        sbsoptions.Add(bsoptions, 0, wx.LEFT,10)
        #Return
        sbreturn = wx.StaticBox(self.optionPanel, -1, '', size=(-1, -1))
        sbsreturn = wx.StaticBoxSizer(sbreturn, wx.HORIZONTAL)
        bsreturn = wx.BoxSizer ( wx.HORIZONTAL )
        bReturn = wx.Button(self.optionPanel,-1,label="Return")
        self.Bind(wx.EVT_BUTTON, self._onShowMain, bReturn)
        bsreturn.Add(bReturn,0, wx.ALL,10)
        sbsreturn.Add(bsreturn,0,wx.CENTER,10)

        self.Osizer = wx.BoxSizer ( wx.VERTICAL)
        self.Osizer.Add(sbsoptions, 0, wx.EXPAND)
        self.Osizer.Add(sbsreturn, 0, wx.EXPAND)
        self.optionPanel.SetSizer(self.Osizer)
        self.optionPanel.Hide()
        self.optionPanel.Raise()
        self.Bind(wx.EVT_SIZE, self._onSize)

    def _onShowMain(self, event):
        self.pl.SetPosition((0,0))
        self.resultPanel.Hide()
        self.optionPanel.Hide()
        self.pl.Show()
        self.Refresh_Frame()

    def _onShowResult(self, event):
        self.resultPanel.SetPosition((0,0))
        self.resultPanel.Show()
        self.pl.Hide()
        self.optionPanel.Hide()
        self.resultPanel.Refresh()
        self.Refresh_Frame()

    def _onShowOptions(self, event):
        self.optionPanel.SetPosition((0,0))
        self.optionPanel.Show()
        self.pl.Hide()
        self.resultPanel.Hide()
        self.optionPanel.Refresh()
        self.Refresh_Frame()

    def GetResultPanelOption(self):
        if (self.rbMainR.GetValue()):
            self.branch_selected = "main"
            self.branch_idx = 0
        elif (self.rbCr3R.GetValue()):
            self.branch_selected = "cr3"
            self.branch_idx = 1
        else :
            self.branch_selected = "ST"
            self.branch_idx = 2

        self.band_4graph =[]
        if self.band4R.IsChecked():
           self.band_4graph.append(4)
        if self.band17R.IsChecked():
            self.band_4graph.append(17)

        self.scenario_4graph = []
        if self.AlltestR.IsChecked():
            self.scenario_4graph = scenario_implemented
        else :
            for i in range(0,len(scenario_implemented)):
                if self.scenR[i].IsChecked() :
                    self.scenario_4graph.append(scenario_implemented[i])

        try:
            self.clR = int(self.changelistR.GetValue())
        except:
            self.clR = 0

        try:
            self.nbcmpR = int(self.tcNbCompR.GetValue())
        except:
            self.nbcmpR = 5

        self.image_3d()

    def init_2d(self,foo,x,y):
        return [[foo for i in range(x)] for j in range(y)]

    def image_3d(self):

        #cdir ='\\\serv2.icerasemi.com\home\gcflab\workspace\callbox-test_cr3\swtools\main.br\\auto_regression\callbox\chart\\'
        #cdir = '\\\\'+CHART_LOC+'\\'
        cdir = 'chart\\'
        self.image_list =[]
        for i in range(0,len(BRANCH_ALLOWED)):
            self.image_list.append(" ")
            self.image_list[i] = self.init_2d("x",len(scenario_implemented),len(BAND_TANGO_ALLOWED))


        for i in range(0,len(BRANCH_ALLOWED)):
            for j in range(0,len(BAND_TANGO_ALLOWED)):
                for k in range(0,len(scenario_implemented)):
                    file = scenario_implemented[k]+"_"+"Band"+str(BAND_TANGO_ALLOWED[j])+"_"+BRANCH_ALLOWED[i]+".jpeg"
                    self.image_list[i][j][k] = cdir+file



    def _updateResultPanel(self,event):

        self.GetResultPanelOption()
        self._cleanResultPanel()
        self.img = []
        self.caption = []
        size = len(self.band_4graph) * len(self.scenario_4graph)
        self.gridSizer = wx.GridSizer(rows=size, cols=1)
        InsideBoxer =[]
        if self.forceGraph.IsChecked() or self.clR !=0 :
            for i in self.band_4graph:
                for j in self.scenario_4graph:
                    Chart().chart_scenario(int(i),self.clR,self.nbcmpR,j,self.branch_selected)


        for j in range(0,len(self.band_4graph)):
            for k in range(0,len(self.scenario_4graph)):
                self.caption.append(wx.StaticText(self.resultPanel,-1,"BAND "+str(self.band_4graph[j])+" "+self.scenario_4graph[k]))
                self.caption[j*len(self.scenario_4graph)+k].SetFont(self.Font_Result)
                self.img.append(wx.StaticBitmap(self.resultPanel, wx.ID_ANY, wx.BitmapFromImage(wx.Image(self.image_list[self.branch_idx][self.find_index(BAND_TANGO_ALLOWED,self.band_4graph[j])][self.find_index(scenario_implemented,self.scenario_4graph[k])],wx.BITMAP_TYPE_ANY))))
                #print "img loc",self.image_list[self.branch_idx][l][self.find_index(scenario_implemented,self.scenario_4graph[k])]
                InsideBoxer.append(wx.BoxSizer(wx.VERTICAL))
                InsideBoxer[j*len(self.scenario_4graph)+k].Add(self.caption[j*len(self.scenario_4graph)+k])
                InsideBoxer[j*len(self.scenario_4graph)+k].Add(wx.StaticText(self.resultPanel,-1,""))
                InsideBoxer[j*len(self.scenario_4graph)+k].Add(self.img[j*len(self.scenario_4graph)+k])

                self.gridSizer.Add(InsideBoxer[j*len(self.scenario_4graph)+k])

                #self.img[j*len(self.scenario_4graph)+k].SetBitmap(wx.BitmapFromImage( wx.Image(self.image_list[self.branch_idx][j][k],wx.BITMAP_TYPE_ANY)))

                #self.caption[j*len(self.scenario_4graph)+k].SetLabel("Band "+str(self.band_4graph[j])+"  "+self.scenario_4graph[k])
                #self.img[j*len(self.scenario_4graph)+k].SetBitmap(wx.BitmapFromImage( wx.Image(self.image_list[self.branch_idx][j][k],wx.BITMAP_TYPE_ANY)))


        self.Rsizer.Add(self.gridSizer, 0, wx.EXPAND)
        self.Refresh_Frame()

    def _cleanResultPanel(self,event=0):
        try:
            self.Rsizer.Hide(self.gridSizer)
            self.Rsizer.Remove(self.gridSizer)
        except:
            print "Already Empty"
        self.gridSizer = []
        self.Refresh_Frame()

    def _onSize(self, event):
        event.Skip()
        self.resultPanel.SetSize(self.GetClientSizeTuple())
        self.pl.SetSize(self.GetClientSizeTuple())

    def ParseArgs(self):
        self.branch_4test =[]
        if self.mainBr.IsChecked():
            self.branch_4test.append('main')
        if self.cr3Br.IsChecked():
            self.branch_4test.append('cr3')
        if self.FTBr.IsChecked():
            self.branch_4test.append('ST')

        self.band_4test =[]
        if self.band4.IsChecked():
           self.band_4test.append(4)
        if self.band17.IsChecked():
            self.band_4test.append(17)
        #BAND_TANGO_ALLOWED = band_4test

        self.scenario_4test = []
        if self.Alltest.IsChecked():
            self.scenario_4test = scenario_implemented
        else :
            for i in range(0,len(scenario_implemented)):
                if self.scen[i].IsChecked() :
                    self.scenario_4test.append(scenario_implemented[i])

        try:
            self.cl = int(self.changelist.GetValue())
        except:
            self.cl = 0

        try:
            self.nbcmp = int(self.tcNbComp.GetValue())
        except:
            self.nbcmp = 0

        #Option Panel
        if (str(self.atport.GetValue()) != "") :
            common.PORT_COM_TANGO = str(self.atport.GetValue())
            print "AT Port to use",common.PORT_COM_TANGO

        if(str(self.modemport.GetValue()) != ""):
            common.MODEM_PORT = str(self.modemport.GetValue())
            print "Modem Port to use",common.MODEM_PORT

        if(str(self.vid_no.GetValue()) != ""):
            common.VID = str(self.vid_no.GetValue())
            print "VID number to use",common.VID

        try:
            if self.rbplatform.GetSelection() == 0:
                common.CARDHU = False
            if self.rbplatform.GetSelection() == 1:
                common.CARDHU = True
                common.MODEM_PORT = common.CARDHU_MODEM_TCP
        except:
            common.CARDHU = False

    def Refresh_Progress(self):
        self.sizer.Layout()
        self.pl.Layout()

    def Refresh_Frame(self):
        #self.log.Clear()
        self.sizer.Layout()
        self.Rsizer.Layout()
        self.Osizer.Layout()
        self.panel.Refresh()
        self.pl.Refresh()
        self.resultPanel.Refresh()
        self.SendSizeEvent()
        self.resultPanel.SetupScrolling()
        self.resultPanel.Layout()
        self.optionPanel.Layout()
        self.pl.Layout()
        self.panel.Layout()

        self.log.Refresh()
        self.redir=RedirectText(self.log)
        sys.stdout=self.redir
        sys.stderr=self.redir

    def onRun(self, event):

        self.cleanProgress()
        self.ParseArgs()
        self.InitupdateProgress()

        self.t1 = KThread(target=self.Start_Test)
        self.t1.start()

        self.t2 = KThread(target=self.updateProgress)
        self.t2.start()

    def stopThread(self,event):
        print "Number of Active Threads",threading.activeCount()
        if threading.activeCount()>1:
            self.t1.kill() # Autocallbox Thread
            self.t2.kill() # Read_Status Thread
            print "Number of Active Threads",threading.activeCount()
        print "Number of Active Threads",threading.activeCount()
        #self.cleanProgress()

    def addProgress(self):

        #ProgressSection
        self.cl_run = "       "
        self.branch_run = "     "
        self.band_run = "  "
        self.scen_run = "               "
        self.scenario_4test = ""
        #current Run
        sbCurrentRun = wx.StaticBox(self.pl, -1, 'Current Test', size=(-1, -1))
        #sbCurrentRun.SetForegroundColour(wx.GREEN)
        sbsCurrentRun = wx.StaticBoxSizer(sbCurrentRun, wx.VERTICAL)
        bsCurrentRun = wx.BoxSizer ( wx.HORIZONTAL )
        self.stCL = wx.StaticText(self.pl,-1,self.cl_run)
        self.stBranch = wx.StaticText(self.pl,-1,self.branch_run)
        self.stBand = wx.StaticText(self.pl,-1,self.band_run)
        self.stScen = wx.StaticText(self.pl,-1,self.scen_run)
        bsCurrentRun.Add(wx.StaticText(self.pl,-1,"CL : "))
        bsCurrentRun.Add(self.stCL,1,wx.ALL|wx.EXPAND,10)
        bsCurrentRun.Add(wx.StaticText(self.pl,-1,"BRANCH : "))
        bsCurrentRun.Add(self.stBranch,1,wx.ALL|wx.EXPAND,10)
        bsCurrentRun.Add(wx.StaticText(self.pl,-1,"BAND : "))
        bsCurrentRun.Add(self.stBand,1,wx.ALL|wx.EXPAND,10)
        bsCurrentRun.Add(wx.StaticText(self.pl,-1,"SCENARIO : "))
        bsCurrentRun.Add(self.stScen,1,wx.ALL|wx.EXPAND,10)
        sbsCurrentRun.Add(bsCurrentRun, 0, wx.LEFT,10)

        #Band4
        sbBand4Prog = wx.StaticBox(self.pl, -1, 'Band 4 Progress', size=(-1, -1))
        #sbBand4Prog.SetForegroundColour(wx.BLUE)
        sbsBand4Prog = wx.StaticBoxSizer(sbBand4Prog, wx.VERTICAL)
        bsBand4Prog = wx.BoxSizer ( wx.HORIZONTAL )
        self.stscen_4 = []
        for i in range(0,len(scenario_implemented)):
            self.stscen_4.append(wx.StaticText(self.pl, -1 ,""))
            bsBand4Prog.Add(self.stscen_4[i],0, wx.ALL,10)
        sbsBand4Prog.Add(bsBand4Prog, 0, wx.LEFT,10)

        #Band17
        sbBand17Prog = wx.StaticBox(self.pl, -1, 'Band 17 Progress', size=(-1, -1))
        #sbBand17Prog.SetForegroundColour(wx.BLUE)
        sbsBand17Prog = wx.StaticBoxSizer(sbBand17Prog, wx.VERTICAL)
        bsBand17Prog = wx.BoxSizer ( wx.HORIZONTAL )
        self.stscen_17 = []
        for i in range(0,len(scenario_implemented)):
            self.stscen_17.append(wx.StaticText(self.pl, -1 ,""))
            bsBand17Prog.Add(self.stscen_17[i],0, wx.ALL,10)
        sbsBand17Prog.Add(bsBand17Prog, 0, wx.LEFT,10)

        #PROGRESS SECTION JOIN
        self.bsPRG = wx.BoxSizer ( wx.VERTICAL)
        self.bsPRG.Add(sbsCurrentRun, 0, wx.EXPAND)
        self.bsPRG.Add(sbsBand4Prog, 0, wx.EXPAND)
        self.bsPRG.Add(sbsBand17Prog, 0, wx.EXPAND)

        self.sizer.Add(self.bsPRG,0,wx.EXPAND)

        #self.pl.Refresh()

        #self.Refresh_Frame()
    def InitupdateProgress(self):
        self.stCL.SetLabel(self.cl_run)
        self.stBranch.SetLabel(self.branch_run)
        self.stBand.SetLabel(self.band_run)
        self.stScen.SetLabel(self.scen_run)

        if self.band4.IsChecked():
            for i in range(0,len(self.scenario_4test)):
                self.stscen_4[i].SetLabel(self.scenario_4test[i])


        if self.band17.IsChecked():
            for i in range(0,len(self.scenario_4test)):
                self.stscen_17[i].SetLabel(self.scenario_4test[i])

        self.Refresh_Frame()

    def updateProgress(self):

        self.scen_list_4x = []
        self.scen_list_17x = []
        for scen in self.scenario_4test:
            self.scen_list_4x.append(scen)
            self.scen_list_17x.append(scen)

        while threading.activeCount()>1:
        #while self.t1.isAlive() or self.t4.isAlive():
            time.sleep(5)
            Read = self.Read_Status()
            if Read == True :
                self.stCL.SetLabel(self.cl_run)
                self.stBranch.SetLabel(self.branch_run)
                self.stBand.SetLabel(self.band_run)
                self.stScen.SetLabel(self.scen_run)

                for i in range(0,len(self.scenario_4test)):
                    if self.band4.IsChecked():
                        if self.scen_list_4x[i] == STATUS_OK:
                            self.stscen_4[i].SetBackgroundColour("#1ad821") # GREEN
                        elif self.scen_list_4x[i] == STATUS_REGRESSION:
                            self.stscen_4[i].SetBackgroundColour("#ddff0f") # Orange
                        elif self.scen_list_4x[i] == STATUS_ASSERT:
                            self.stscen_4[i].SetBackgroundColour("#ff1515") # Red
                        elif self.scen_list_4x[i] != self.scenario_4test[i]:
                            self.stscen_4[i].SetBackgroundColour("#0000ff") # Blue

                    if self.band17.IsChecked():
                        if self.scen_list_17x[i] == STATUS_OK:
                            self.stscen_17[i].SetBackgroundColour("#1ad821") # GREEN
                        elif self.scen_list_17x[i] == STATUS_REGRESSION:
                            self.stscen_17[i].SetBackgroundColour("#ddff0f") # Orange
                        elif self.scen_list_17x[i] == STATUS_ASSERT:
                            self.stscen_17[i].SetBackgroundColour("#ff1515") # Red
                        elif self.scen_list_17x[i] != self.scenario_4test[i]:
                            self.stscen_17[i].SetBackgroundColour("#0000ff") # Blue

                self.Refresh_Progress()
                #self.pl.Refresh()
                # self.panel.Fit()
                # self.SendSizeEvent()
                # #self._onSize(wxFrame::SendSizeEvent)

    def cleanProgress(self):
        self.cl_run = ""
        self.branch_run = ""
        self.band_run = ""
        self.scen_run = ""
        self.scenario_4test = ""

        self.stCL.SetLabel(self.cl_run)
        self.stBranch.SetLabel(self.branch_run)
        self.stBand.SetLabel(self.band_run)
        self.stScen.SetLabel(self.scen_run)

        for i in range(0,len(scenario_implemented)):
            self.stscen_4[i].SetLabel("")
            self.stscen_4[i].SetBackgroundColour("#bfbfbf")

        for i in range(0,len(scenario_implemented)):
            self.stscen_17[i].SetLabel("")
            self.stscen_17[i].SetBackgroundColour("#bfbfbf")

        self.Refresh_Frame()

    def mainThread(self):
        self.kill_thread = False
        self.t1 = threading.Thread(target=self.Start_Test)
        self.t1.start()

        while True:
            if self.kill_thread == True :
                #self.t1.join()
                print "Killing Thread"
                sys.exit(1)


    def ResultWindow(self, event):
        print "Result Window"

    def TestType(self, event):
        if (self.rbAuto.GetValue()):
            self.testchoice = 0;
        elif (self.rbUnit.GetValue()):
            self.testchoice = 1
        else:
            self.testchoice = 2

    def PlatformType(self, event):
        if (self.rbWinxp.GetValue()):
            self.platformchoice = 0;
        elif (self.rbWoA.GetValue()):
            self.platformchoice = 1
            
    def Start_Test(self):
        iCT = CallboxTest()
        iCT.Init_Auto(self.branch_4test,self.band_4test,self.scenario_4test)
        if self.testchoice == 0:
            print "Call Auto scheduler"
            iCT.start()
        elif self.testchoice == 1:
            if self.flash.IsChecked() == True and self.entry.GetLabel() != "":
                print "Run Test with Flashing"
                Untar().main(self.entry.GetLabel())
                Flash().flash_modem(99999,'cr3')
                iCT.Init_Auto(self.branch_4test,self.band_4test,self.scenario_4test)
                iCT.Run_Branch_Test(Forced=self.force.IsChecked(),flash=False,Reg=self.cReg.IsChecked(),CL=self.cl)
                print "Run Test Finished"
            else:
                print "Run Test without Flashing"
                iCT.Run_Branch_Test(Forced=self.force.IsChecked(),flash=self.flash.IsChecked(),Reg=self.cReg.IsChecked(),CL=self.cl)
                print "Run Test Finished"

        elif self.testchoice == 2:
           print "No Test"


    def find_index(self,lst,item):
        for i in range(0,len(lst)):
            if(lst[i]==item):
                #print "find index %d"%i
                return i
        return 0

    def Read_Status(self):
        file = 'status.txt'
        self.scen_list_4 =[]
        self.status_list_4 = []
        self.scen_list_17 =[]
        self.status_list_17 = []

        latest_CL = 0
        latest_br = ""
        CURRENT_SCEN = ""
        if os.path.exists(file):
            FILE_OK = open(file,'r')
            for line in (FILE_OK.readlines()):#reversed
                try:
                    band_run = re.search(re.compile(r"Band:(\S+)"),line).group(1)
                    scen = re.search(re.compile(r"Test:(\S+)"), line).group(1)
                    Status = re.search(re.compile(r"Status:(\S+)(])"), line).group(1)
                    if int(band_run) == 4 :
                        self.scen_list_4x[self.find_index(self.scenario_4test,scen)] = Status
                    elif int(band_run) == 17 :
                        self.scen_list_17x[self.find_index(self.scenario_4test,scen)] = Status
                except:
                    pass

            self.cl_run = re.search(re.compile(r"CL([0-9]+)"),line).group(1)
            self.branch_run = re.search(re.compile(r"Branch:(\S+)"),line).group(1)
            self.scen_run  = re.search(re.compile(r"Test:(\S+)"), line).group(1)
            self.band_run = re.search(re.compile(r"Band:(\S+)"),line).group(1)

            FILE_OK.close()
            return True
        else:
            return False

    def OnPressEnter(self,event):
        x = 1
        #self.stInfo.SetLabel( self.entry.GetValue())

    def OnOpenScen(self,event):
        wildcard = " (*.tar,*.gz,*.wrapped,*.rar)|*.tar;*.gz;*.wrapped;*.rar"
        dialog = wx.FileDialog(None, "Choose a file",os.getcwd(), "", wildcard, wx.OPEN)
        if dialog.ShowModal() == wx.ID_OK:
            filename = dialog.GetPath()
            self.entry.AppendText(filename)
            self.flash_list.append(filename)
        dialog.Destroy()

    def flash_only(self,event):
        self.t3 = KThread(target=self.flash_thread)
        self.t3.start()

    def flash_thread(self):
        if self.entry.GetLabel() != "" :
            Untar().main(self.entry.GetLabel())
            Flash().flash_modem(99999,'cr3')
        else:
            print "Please select a binary to flash"


    def Regression_Start(self,event):
        self.cleanProgress()
        self.ParseArgs()
        self.InitupdateProgress()
        self.t4 = KThread(target=self.regression_thread)
        self.t4.start()
        self.t2 = KThread(target=self.updateProgress)
        self.t2.start()

    def regression_thread(self):
        cl = int(self.changelist.GetValue())
        if cl != 0 :
            Regression()._run(self.branch_4test,self.band_4test,self.scenario_4test,cl)
            return
Пример #19
0
    def appendEntries(self, request, context):  # receiving/server side
        # int32 term = 1;
        # int32 leaderID = 2;
        # int32 prevLogIndex = 3;
        # int32 prevLogTerm = 4;
        # repeated LogEntry entries = 5;
        # int32 leaderCommit = 6;
        self.leaderID = request.leaderID
        if random.uniform(0,
                          1) < self.cmserver.fail_mat[self.leaderID][self.id]:
            self.logger.warning(
                f'[ABORTED]: append entries from server <{self.leaderID}> '
                f'to <{self.id}>, because of ChaosMonkey')
        else:
            # Todo: if election timeout elapse without receiving AppendEntries RPC from current leader or granting vote
            #  to candidate: convert to candidate
            self.lastUpdate = time.time()
            success = False
            try:
                # Todo: If candidates receive AppendEntries RPC from a leader, convert to follower
                #  mcip: request term should be useless coz of last log term??????
                if self.role == KVServer.candidate:
                    self.save(current_term=max(self.lastLogTerm, request.term),
                              voted_for=-1)
                    self.role = KVServer.follower
                    with self.candidateCond:
                        self.candidateCond.notify_all()
                else:
                    # Todo: All servers: If RPC request or response contains term T> currentTerm,
                    #  set current term = T, convert to follower
                    self.convToFollowerIfHigherTerm(request.term, voted_for=-1)

                tmp_entries = []
                for row in request.entries:
                    r = [row.term, row.key, row.val]
                    tmp_entries.append(r)
                    # self.logger.info(f'row: <{r}>')
                # reply false if term < currentTerm,
                # or log doesn't log doesn't contain an entry at prevLogIndex whose term matches prevLogTerm
                # Todo: if it doesn't match the term, it will decrement and resend, thus following will remove entries
                if request.term < self.currentTerm or request.prevLogIndex > len(self.log) \
                        or (request.prevLogIndex < len(self.log) and
                            self.log[request.prevLogIndex][0] != request.prevLogTerm):

                    self.logger.warning(
                        f'[AP_En]: received on <{self.id}>, will return false to <{self.leaderID}>'
                    )
                    self.logger.warning(
                        f'[AP_En]: <{request.term < self.currentTerm}>, '
                        f'<{request.prevLogIndex > len(self.log)}>, '
                        f'<{(request.prevLogIndex < len(self.log) and self.log[request.prevLogIndex][0] != request.prevLogTerm)}>'
                    )
                    self.logger.info(
                        f'Parameters for false: req term: <{request.term}>, cur term: '
                        f'<{self.currentTerm}>, req prevLogIdx: <{request.prevLogIndex}>, '
                        f'length of server log <{len(self.log)}>')
                    if request.prevLogIndex < len(self.log):
                        self.logger.info(
                            f'term of log on prev log index: <{self.log[request.prevLogIndex][0]}>, '
                            f'request prev log term: <{request.prevLogTerm}>')
                        #     existing entry conflicts with a new one, same idx different terms,
                        #     delete the existing entry and all that follow it
                        # self.logger.info(f'RAFT: checking conflicting entries')
                        itr = 0
                        for a, b in zip(tmp_entries,
                                        self.log[request.prevLogIndex:]):
                            if a != b:
                                self.logger.warning(
                                    f'[Log]: Found conflict at index: '
                                    f'<{request.prevLogIndex + itr}>')
                                self.logModify(request.prevLogIndex + itr,
                                               LogMod.DELETION)
                            itr += 1
                else:
                    self.save(current_term=max(self.currentTerm, request.term),
                              voted_for=-1)
                    # self.logger.info("RAFT: AppendEntries should succeed unless there is conflict entries")
                    success = True
                    #     existing entry conflicts with a new one, same idx different terms,
                    #     delete the existing entry and all that follow it
                    # self.logger.info(f'RAFT: checking conflicting entries')
                    itr = 0
                    if len(self.log) > 0:
                        for a, b in zip(tmp_entries,
                                        self.log[request.prevLogIndex:]):
                            if a != b:
                                self.logger.warning(
                                    f'[Log]: Found conflict at index: '
                                    f'<{request.prevLogIndex + itr}>')
                                self.logModify(request.prevLogIndex + itr,
                                               LogMod.DELETION)
                            itr += 1
                    # Heartbeat
                    if len(tmp_entries) == 0:
                        self.logger.info("[Log]: received a heartbeat")
                    # Normal append entries
                    else:
                        self.logger.info(
                            f"[Log]: append entries, leader commit <{request.leaderCommit}>"
                        )
                        # Append any new entries not already in the log
                        to_append_length = request.prevLogIndex + len(
                            tmp_entries) - len(self.log)
                        # self.logger.debug(f'RAFT: length of log to append: <{to_append_length}>')
                        if to_append_length > 0:
                            self.logModify(tmp_entries[-to_append_length:],
                                           LogMod.ADDITION)
                        # If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry)
                        # print("Received log from appendEntries is: ", tmp_entries)
                        # self.logger.debug(f'RAFT: Checking if we need to write to disk: <{request.leaderCommit}>,'
                        #                   f'<{self.commitIndex}>, <{self.lastLogIndex}>')
                        if request.leaderCommit > self.lastApplied:
                            self.logger.info(
                                f"[Log]: apply to state machine, leader commit <{request.leaderCommit}> "
                                f"last applied <{self.lastApplied}>")
                            self.commitIndex = min(request.leaderCommit,
                                                   self.lastLogIndex)
                            app_state_mach_kth = KThread(
                                target=self.applyToStateMachine,
                                args=(self.lastApplied, ))
                            app_state_mach_kth.start()
                    # int32 term = 1;
                    # bool success = 2;
                self.lastUpdate = time.time()
                return kvstore_pb2.AppendResponse(term=self.currentTerm,
                                                  success=success)
            except Exception as e:
                self.logger.error("RAFT[Vote]: f(): appendEntries:")
                self.logger.error(e)
Пример #20
0
 def thread_append_entry(self, idx, addr, app_ent_term):
     try:
         append_request = kvstore_pb2.AppendRequest()
         append_request.term = app_ent_term  # int32 term = 1;
         append_request.leaderID = self.id  # int32 leaderID = 2;
         append_request.prevLogIndex = self.nextIndex[
             idx]  # int32 prevLogIndex = 3;
         # int32 prevLogTerm = 4;
         if 0 <= self.nextIndex[idx] < len(self.log):
             append_request.prevLogTerm = self.log[self.nextIndex[idx]][0]
         else:
             append_request.prevLogTerm = 0
         append_request.leaderCommit = self.commitIndex  # int32 leaderCommit = 6;
         last_req_log_idx = self.lastLogIndex
         self.logger.info(
             f"[AP_En]: thread_append_entry to <{idx}> prevLogInd <{append_request.prevLogIndex}> "
             f"prevLogTerm <{append_request.prevLogTerm}>")
         if self.nextIndex[idx] < len(self.log):
             for row in self.log[
                     self.
                     nextIndex[idx]:]:  # repeated LogEntry entries = 5;
                 entry = append_request.entries.add()
                 entry.term = row[0]
                 entry.key = row[1]
                 entry.val = row[2]
         self.nextIndex[
             idx] = self.lastLogIndex + 1  # Todo: should inc to +1 here?
         # with grpc.insecure_channel(addr) as channel:
         channel = grpc.insecure_channel(addr)
         grpc.channel_ready_future(channel).result()
         # int32 term = 1;
         # bool success = 2;
         stub = kvstore_pb2_grpc.KeyValueStoreStub(channel)
         if random.uniform(
                 0, 1) < self.cmserver.fail_mat[self.leaderID][self.id]:
             self.logger.warning(
                 f'[ABORTED]: we will not receive from <{self.leaderID}> '
                 f'because of ChaosMonkey')
         else:
             # self.logger.info(f'[AP_En]: thread_append_entry to <{idx}>, '
             #                  f'req last log <{last_req_log_idx}>')
             # f'req entries \n<{append_request.entries}>')
             append_entry_response = stub.appendEntries(
                 append_request, timeout=self.requestTimeout)
             if not append_entry_response.success:
                 self.logger.info(
                     f"[AP_En]: thread_append_entry to <{idx}> failed, "
                     f"its term <{append_entry_response.term}>, leader's <{self.currentTerm}>"
                 )
                 # Failed because of log inconsistency, decrement nextIndex and retry
                 if append_entry_response.term <= self.currentTerm:
                     self.logger.info(
                         f"[AP_En]: log inconsistency, nextIndex for <{idx}> dec from "
                         f"<{self.nextIndex[idx]}> to <{max(append_request.prevLogIndex - 1, 0) }>"
                     )
                     # Todo: how to decrement correctly
                     self.nextIndex[idx] = max(
                         append_request.prevLogIndex - 1, 0)
                 else:
                     # Todo: All servers: If RPC request or response contains term T> currentTerm,
                     #  set current term = T, convert to follower
                     self.convToFollowerIfHigherTerm(
                         append_entry_response.term, voted_for=-1)
             # Success
             else:
                 self.logger.info(
                     f"[AP_En]: thread_append_entry to <{idx}> success")
                 self.matchIndex[idx] = last_req_log_idx
                 self.logger.debug(
                     f'[KVStore]: matchIndex: <{self.matchIndex}>')
                 n_list = sorted(self.matchIndex)
                 # TODO: write to disk upon majority
                 # if there exists such N that N> commitIndex and majority of matchIndex[i] >= N
                 # and log[N].term ==currentTerm, set commitIndex = N
                 N = n_list[int(len(n_list) / 2)]
                 if N >= 0 and N > self.commitIndex and self.log[N][
                         0] == self.currentTerm:
                     self.commitIndex = N
                     self.logger.info(
                         f"RAFT: Commit index on leader updates to: {N}")
                     disk_write_kth = KThread(
                         target=self.applyToStateMachine,
                         args=(self.lastApplied, ))
                     disk_write_kth.start()
     except Exception as e:
         self.logger.error(
             "[Vote]: f() thread_append_entry, most likely name resolution error"
         )
         self.logger.error(e)  # Todo: Name resolution error
Пример #21
0
      elif message.find(':'+addrchar+'chgnick ')!=-1:
            acco=message.rsplit(":"+addrchar+"chgnick ") [1]
            if acco.find(' ')!=-1:
                return send("PRIVMSG %s :%s: Not a valid nick\r" % (channel,nick))
            else:
                botnick = acco
                return send("NICK "+ botnick )

      if channel == nick :
         return send("PRIVMSG %s :Not in a channel\r" % (nick))
   elif not cloak in ownercloak and message.find(':'+addrchar)!=-1:
      return send("PRIVMSG %s :%s: You are not authorized to run this, sorry\r" % (channel,nick))
          

if options.control == True:
   sthr.start()

def joinchan(chan):
   send("JOIN "+chan)
def ping():
   send("PONG :pingis")  

def sendmsg(chan , msg): 
  send("PRIVMSG "+ chan +" :"+ msg +"!") 
def hello(): 
  send("PRIVMSG "+ channel +" :Hello "+ nick )
def func():
   count = 1
   while 1:
      time.sleep(10)
      if count == 1:
Пример #22
0
class KVServer(kvstore_pb2_grpc.KeyValueStoreServicer):
    follower = 0
    candidate = 1
    leader = 2

    def __init__(self, addresses: list, id: int, server_config: dict):
        ### Persistent state on all servers, update on stable storage before responding to RPCs
        self.currentTerm = 0
        self.votedFor = -1
        self.log = []  # first index is 1

        # load persistent state from json file
        self.id = id
        self.persistent_file = 'log/config-%d' % self.id
        self.diskLog = "log/log-%d.pkl" % self.id
        # self.loggerFile = "log/log-%d.txt" % self.id
        self.diskStateMachine = "log/state_machine-%d.pkl" % self.id
        # Todo: will re-enable load later
        # self.load()
        self.stateMachine = {}  # used to be storage

        # Config
        self.requestTimeout = server_config["request_timeout"]  # in ms
        self.maxElectionTimeout = server_config["election_timeout"]
        self.keySizeLimit = server_config["key_size"]
        self.valSizeLimit = server_config["value_size"]
        self.appendEntriesTimeout = float(
            server_config["app_entries_timeout"]) / 1000

        ### Volatile state on all servers
        self.commitIndex = -1  # known to be commited, init to 0
        # if larger than lastApplied, apply log to state machine
        self.lastApplied = -1  # index of highest log entry applied to state machine, init to 0

        self.role = KVServer.candidate
        self.leaderID = -1
        self.peers = []
        self.lastUpdate = time.time()

        # Condition variables
        self.appendEntriesCond = Condition()
        self.appliedStateMachCond = Condition()
        self.lastCommittedTermCond = Condition()
        self.leaderCond = Condition()
        self.candidateCond = Condition()
        self.followerCond = Condition()

        # Client related
        self.registeredClients = []
        self.clientReqResults = {
        }  # clientID: [stateMachineOutput, sequenceNum]

        # current state
        self.currElectionTimeout = random.uniform(
            self.maxElectionTimeout / 2,
            self.maxElectionTimeout) / 1000  # in sec
        for idx, addr in enumerate(addresses):
            if idx != self.id:
                self.peers.append(idx)
        self.majority = int(len(addresses) / 2) + 1
        self.lastLogIndex = -1
        self.lastLogTerm = 0
        self.addresses = addresses  # number of nodes implied here
        self.cmserver = CMServer(num_server=len(addresses))

        # create logger with 'raft'
        self.logger = logging.getLogger('raft')
        self.logger.setLevel(logging.DEBUG)
        # create formatter and add it to the handlers
        formatter = logging.Formatter(
            '[%(asctime)s,%(msecs)d %(levelname)s]: %(message)s',
            datefmt='%M:%S')
        # create file handler which logs even debug messages
        os.makedirs(os.path.dirname('log/logger-%d.txt' % self.id),
                    exist_ok=True)
        fh = logging.FileHandler('log/logger-%d.txt' % self.id)
        fh.setLevel(logging.INFO)
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)
        # create console handler with a higher log level
        ch = logging.StreamHandler()
        ch.setLevel(logging.INFO)
        ch.setFormatter(formatter)
        self.logger.addHandler(ch)

        # # Last version logging setting
        # logging.basicConfig(filename='log/logger-%d.txt' % self.id,
        #                     filemode='a',
        #                     format='%(asctime)s,%(msecs)d %(levelname)s %(message)s',
        #                     datefmt='%H:%M:%S')
        # self.logger = logging.getLogger('raft')
        # self.logger.setLevel(logging.NOTSET)
        self.logger.debug(
            f'[Chaos]: Initial ChaosMonkey matrix: \n<{self.cmserver}>')

        ### Volatile state on leaders
        self.nextIndex = [0] * len(
            addresses)  # index of next log entry to send to that server
        self.matchIndex = [-1] * len(
            addresses)  # highest log entry known to be replicated
        # if there exists such N that N> commitIndex and majority of matchIndex[i] >= N
        # and log[N].term ==currentTerm, set commitIndex = N
        self.numVotes = 0

        # Todo: for debugging only
        self.debug1 = 0

    def load(self):
        if os.path.isfile(self.persistent_file):
            with open(self.persistent_file, 'r') as f:
                data_store = json.load(f)
                self.currentTerm = data_store["currentTerm"]
                self.votedFor = data_store["votedFor"]

    # Todo: check if all currentTerm and votedFor has .save() save persistent state to json file
    def save(self, current_term, voted_for):
        self.currentTerm = current_term
        self.votedFor = voted_for
        persistent = {
            "currentTerm": self.currentTerm,
            "votedFor": self.votedFor
        }
        with open(self.persistent_file, 'w') as f:
            json.dump(persistent, f)

    def follower(self):
        while True:
            # self.role = KVServer.follower  # Todo: is this correct
            self.logger.critical(
                f'[Role]: Running as a follower, elec timeout <%.4f>' %
                self.currElectionTimeout)
            self.save(current_term=self.currentTerm, voted_for=-1)
            self.lastUpdate = time.time()
            while time.time() - self.lastUpdate <= self.currElectionTimeout:
                with self.followerCond:
                    self.followerCond.wait(
                        self.currElectionTimeout -
                        (time.time() - self.lastUpdate))  # -elapsed time
            # self.logger.debug(f'Current time <{time.time()}>, last update <{self.lastUpdate}>, deduct to '
            #                   f'<{time.time() - self.lastUpdate}>election timeout <{self.currElectionTimeout}>')
            self.role = KVServer.candidate  # Todo: change to candidate here?
            with self.candidateCond:
                self.candidateCond.notify_all()
            with self.followerCond:
                self.followerCond.wait()

    def candidate(self):
        with self.candidateCond:
            self.candidateCond.wait()
        while True:
            self.logger.critical(
                f'[Role]: Running as a candidate, elec timeout <%.4f>' %
                self.currElectionTimeout)
            # Upon conversion to candidate, start election
            # Increment current term, vote for self, reset election timer, send requestVote RPCs to all other servers

            # self.logger.critical(f'RAFT[Vote]: Server <{self.id}> initiated voting for term <{self.currentTerm}> '
            #                      f'took <%.4f> seconds' % (time.time()-start_time))
            self.save(current_term=self.currentTerm + 1, voted_for=self.id)
            self.numVotes = 1
            self.currElectionTimeout = random.uniform(
                self.maxElectionTimeout / 2, self.maxElectionTimeout) / 1000
            self.election = KThread(target=self.initiateVote, args=())
            self.election.start()
            self.logger.info(
                f'[Vote]: Start, voted for self <{self.id}> term <{self.currentTerm}> '
                f'election timeout: <%.4f>' % self.currElectionTimeout)
            self.lastUpdate = time.time()
            while time.time(
            ) - self.lastUpdate <= self.currElectionTimeout and self.role == KVServer.candidate:
                with self.candidateCond:
                    self.candidateCond.wait(
                        self.currElectionTimeout -
                        (time.time() - self.lastUpdate))  # - elapse time
                if self.numVotes >= self.majority or self.role == KVServer.follower:
                    break
            self.save(current_term=self.currentTerm, voted_for=-1)
            if self.role == KVServer.leader:
                with self.leaderCond:
                    self.leaderCond.notify_all()
                with self.candidateCond:
                    # self.logger.critical(f"in candidate, larger than majority")
                    self.candidateCond.wait()
            elif self.role == KVServer.follower:
                with self.followerCond:
                    self.followerCond.notify_all()
                with self.candidateCond:
                    self.candidateCond.wait()
            # Todo: is this needed?
            # self.lastUpdate = time.time()
            # if time.time() - self.lastUpdate <= self.currElectionTimeout:
            #     with self.candidateCond:
            #         self.candidateCond.wait(self.currElectionTimeout-(time.time() - self.lastUpdate))

    def leader(self):
        while True:
            # mcip: Use condition to control instead
            with self.leaderCond:
                # self.logger.critical(f"reached leader111, larger than majority")
                self.leaderCond.wait()
            if self.role == KVServer.follower:
                with self.followerCond:
                    self.followerCond.notify_all()
                with self.leaderCond:
                    self.leaderCond.wait()
            elif self.role == KVServer.candidate:
                with self.candidateCond:
                    self.candidateCond.notify_all()
                with self.leaderCond:
                    self.leaderCond.wait()

            # self.role = KVServer.leader  # Todo: is this correct?
            self.logger.critical(f'[Role]: Running as a leader')
            self.save(current_term=self.currentTerm, voted_for=-1)
            self.leaderID = self.id
            # for each server it's the index of next log entry to send to that server
            # init to leader last log index + 1
            self.nextIndex = [self.lastLogIndex + 1] * len(self.addresses)
            # for each server, index of highest log entry known to be replicated on server
            # init to 0, increase monotonically
            self.matchIndex = [0] * len(self.addresses)
            # Todo: might need debugging?
            # Upon becoming leader, append no-op entry to log (6.4)
            self.logModify(
                [self.currentTerm, f"no-op: leader-{self.id}", "no-op"],
                LogMod.APPEND)
            self.append_entries()

    def initiateVote(self):
        # Todo: mcip, make sure the term is the same while request vote????
        req_term = self.currentTerm
        for idx, addr in enumerate(self.addresses):
            if idx == self.id:
                continue
            # Create a thread for each request vote
            # Todo: mcip: req_term should be the same
            election_thread = KThread(target=self.thread_election,
                                      args=(
                                          idx,
                                          addr,
                                          req_term,
                                      ))
            election_thread.start()

    # Todo: All servers: If RPC request or response contains term T> currentTerm, set current term = T,
    #  convert to follower
    def convToFollowerIfHigherTerm(self, term, voted_for):
        if term > self.currentTerm:
            if self.role == KVServer.candidate:
                self.save(current_term=term, voted_for=voted_for)
                self.role = KVServer.follower
                with self.candidateCond:
                    self.candidateCond.notify_all()
            elif self.role == KVServer.leader:  # leader
                self.save(current_term=term, voted_for=voted_for)
                self.role = KVServer.follower
                with self.leaderCond:
                    self.leaderCond.notify_all()

    # Todo: Add chaos monkey?
    def thread_election(self, idx, addr, req_term):
        try:
            # Todo: shouldn't always increment term here ???
            vote_request = kvstore_pb2.VoteRequest(
                term=self.currentTerm,
                candidateID=self.id,
                lastLogIndex=self.lastLogIndex,
                lastLogTerm=req_term)
            # with grpc.insecure_channel(addr) as channel:
            channel = grpc.insecure_channel(addr)
            grpc.channel_ready_future(channel).result()
            stub = kvstore_pb2_grpc.KeyValueStoreStub(channel)
            # self.logger.debug(f'Send vote request to server: <{idx}>')
            req_vote_resp = stub.requestVote(
                vote_request,
                timeout=self.requestTimeout)  # timeout keyword ok?
            # Todo: mcip, does this improve?
            # Todo: Add lock here to consider concurrency
            if req_vote_resp.voteGranted:
                self.logger.info(
                    f'[Vote]: received from <{idx}>, vote count: <{self.numVotes}>'
                )
                if self.role == KVServer.candidate:
                    self.numVotes += 1
                    if self.numVotes >= self.majority:
                        self.role = KVServer.leader
                        with self.candidateCond:
                            # self.logger.critical(f"thread_election, larger than majority")
                            self.candidateCond.notify_all()
            else:
                self.logger.info(
                    f'[Vote]: rejected from <{idx}> its term: {req_vote_resp.term}'
                )
                # Todo: added by mcip, does this actually improve?
                if self.role == KVServer.follower and req_vote_resp.term > self.currentTerm:
                    self.save(current_term=req_vote_resp.term, voted_for=-1)
                # Todo: All servers: If RPC request or response contains term T> currentTerm, set current term = T,
                #  convert to follower
                self.convToFollowerIfHigherTerm(req_vote_resp.term,
                                                voted_for=-1)
                # self.role = KVServer.follower
                # with self.candidateCond:
                #     self.candidateCond.notify_all()
                # elif num_rej_votes > self.majority:
                #     self.save(current_term=self.currentTerm, votedFor=-1)
        except Exception as e:
            self.logger.error("[Vote]: f() thread_election:")
            self.logger.error(e)

    def requestVote(self, request,
                    context):  # Receiving vote request and process
        # Todo: not needed?
        # self.lastUpdate = time.time()
        try:
            req_term = request.term
            req_candidate_id = request.candidateID
            req_last_log_ind = request.lastLogIndex
            req_last_log_term = request.lastLogTerm
            # self.logger.debug(f'RAFT[Vote]: Receive request vote from <{req_candidate_id}>')
            vote_granted = True
            # Todo: not sure if req_last_log_term < self.lastLogTerm is needed
            # Reply false if term < currentTerm
            # If votedFor is null/candidateID, and candidate's log is at least as updated as receiver's log, grant vote
            if req_term < self.currentTerm or req_last_log_ind < self.lastLogIndex or \
                    req_last_log_term < self.lastLogTerm or \
                    (self.votedFor != -1 and self.votedFor != req_candidate_id):
                vote_granted = False
                self.logger.info(
                    f'[Vote]: reject vote request from <{req_candidate_id}>, '
                    f'currentTerm <{self.currentTerm}>'
                    f'\n reason: <{req_term < self.currentTerm}>, <{req_last_log_ind < self.lastLogIndex}>'
                    f', <{req_last_log_term < self.lastLogTerm}> or voted for another'
                )
                if self.role == KVServer.follower and req_term > self.currentTerm:
                    self.save(current_term=req_term, voted_for=-1)
                # Todo: All servers: If RPC request or response contains term T> currentTerm, set current term = T,
                #  convert to follower
                self.convToFollowerIfHigherTerm(req_term,
                                                voted_for=req_candidate_id)
            elif req_term == self.currentTerm:
                self.lastUpdate = time.time()
                self.save(current_term=self.currentTerm,
                          voted_for=req_candidate_id)  # TODO: Add lock here?
                self.logger.info(
                    f'[Vote]: vote granted for <{req_candidate_id}> w term <{self.currentTerm}>'
                )
            # Find higher term in RequestVote message
            elif req_term > self.currentTerm:
                self.lastUpdate = time.time()
                if self.role == KVServer.follower:
                    self.save(current_term=req_term,
                              voted_for=req_candidate_id)
                # Todo: All servers: If RPC request or response contains term T> currentTerm, set current term = T,
                #  convert to follower
                self.convToFollowerIfHigherTerm(req_term,
                                                voted_for=req_candidate_id)
                self.logger.critical(
                    f'[Vote]: vote granted for <{req_candidate_id}> '
                    f'due to higher term <{req_term}>')
            # Todo: mcip: if granting the vote to someone, should set back the lastUpdate time?
            return kvstore_pb2.VoteResponse(term=self.currentTerm,
                                            voteGranted=vote_granted)
        except Exception as e:
            self.logger.error("[Vote]: f() requestVote:")
            self.logger.error(e)

    # Leader sends append_entry message as log replication and heart beat
    def append_entries(self):
        while self.role == KVServer.leader:
            # Todo: for debugging only
            # # self.debug1 += 1
            # self.logModify([self.debug1, "aa", "bb"], LogMod.APPEND)
            # self.logModify([self.debug1, "bb", "cc"], LogMod.APPEND)
            app_ent_term = self.currentTerm
            for idx, addr in enumerate(self.addresses):
                if idx == self.id:
                    continue
                # Create a thread for each append_entry message
                # Todo: mcip: append entries term is the same
                append_thread = KThread(target=self.thread_append_entry,
                                        args=(
                                            idx,
                                            addr,
                                            app_ent_term,
                                        ))
                append_thread.start()
            # Send append entry every following seconds, or be notified and wake up
            # Todo: will release during wait
            with self.appendEntriesCond:
                self.appendEntriesCond.wait(timeout=self.appendEntriesTimeout)

    def thread_append_entry(self, idx, addr, app_ent_term):
        try:
            append_request = kvstore_pb2.AppendRequest()
            append_request.term = app_ent_term  # int32 term = 1;
            append_request.leaderID = self.id  # int32 leaderID = 2;
            append_request.prevLogIndex = self.nextIndex[
                idx]  # int32 prevLogIndex = 3;
            # int32 prevLogTerm = 4;
            if 0 <= self.nextIndex[idx] < len(self.log):
                append_request.prevLogTerm = self.log[self.nextIndex[idx]][0]
            else:
                append_request.prevLogTerm = 0
            append_request.leaderCommit = self.commitIndex  # int32 leaderCommit = 6;
            last_req_log_idx = self.lastLogIndex
            self.logger.info(
                f"[AP_En]: thread_append_entry to <{idx}> prevLogInd <{append_request.prevLogIndex}> "
                f"prevLogTerm <{append_request.prevLogTerm}>")
            if self.nextIndex[idx] < len(self.log):
                for row in self.log[
                        self.
                        nextIndex[idx]:]:  # repeated LogEntry entries = 5;
                    entry = append_request.entries.add()
                    entry.term = row[0]
                    entry.key = row[1]
                    entry.val = row[2]
            self.nextIndex[
                idx] = self.lastLogIndex + 1  # Todo: should inc to +1 here?
            # with grpc.insecure_channel(addr) as channel:
            channel = grpc.insecure_channel(addr)
            grpc.channel_ready_future(channel).result()
            # int32 term = 1;
            # bool success = 2;
            stub = kvstore_pb2_grpc.KeyValueStoreStub(channel)
            if random.uniform(
                    0, 1) < self.cmserver.fail_mat[self.leaderID][self.id]:
                self.logger.warning(
                    f'[ABORTED]: we will not receive from <{self.leaderID}> '
                    f'because of ChaosMonkey')
            else:
                # self.logger.info(f'[AP_En]: thread_append_entry to <{idx}>, '
                #                  f'req last log <{last_req_log_idx}>')
                # f'req entries \n<{append_request.entries}>')
                append_entry_response = stub.appendEntries(
                    append_request, timeout=self.requestTimeout)
                if not append_entry_response.success:
                    self.logger.info(
                        f"[AP_En]: thread_append_entry to <{idx}> failed, "
                        f"its term <{append_entry_response.term}>, leader's <{self.currentTerm}>"
                    )
                    # Failed because of log inconsistency, decrement nextIndex and retry
                    if append_entry_response.term <= self.currentTerm:
                        self.logger.info(
                            f"[AP_En]: log inconsistency, nextIndex for <{idx}> dec from "
                            f"<{self.nextIndex[idx]}> to <{max(append_request.prevLogIndex - 1, 0) }>"
                        )
                        # Todo: how to decrement correctly
                        self.nextIndex[idx] = max(
                            append_request.prevLogIndex - 1, 0)
                    else:
                        # Todo: All servers: If RPC request or response contains term T> currentTerm,
                        #  set current term = T, convert to follower
                        self.convToFollowerIfHigherTerm(
                            append_entry_response.term, voted_for=-1)
                # Success
                else:
                    self.logger.info(
                        f"[AP_En]: thread_append_entry to <{idx}> success")
                    self.matchIndex[idx] = last_req_log_idx
                    self.logger.debug(
                        f'[KVStore]: matchIndex: <{self.matchIndex}>')
                    n_list = sorted(self.matchIndex)
                    # TODO: write to disk upon majority
                    # if there exists such N that N> commitIndex and majority of matchIndex[i] >= N
                    # and log[N].term ==currentTerm, set commitIndex = N
                    N = n_list[int(len(n_list) / 2)]
                    if N >= 0 and N > self.commitIndex and self.log[N][
                            0] == self.currentTerm:
                        self.commitIndex = N
                        self.logger.info(
                            f"RAFT: Commit index on leader updates to: {N}")
                        disk_write_kth = KThread(
                            target=self.applyToStateMachine,
                            args=(self.lastApplied, ))
                        disk_write_kth.start()
        except Exception as e:
            self.logger.error(
                "[Vote]: f() thread_append_entry, most likely name resolution error"
            )
            self.logger.error(e)  # Todo: Name resolution error

    def appendEntries(self, request, context):  # receiving/server side
        # int32 term = 1;
        # int32 leaderID = 2;
        # int32 prevLogIndex = 3;
        # int32 prevLogTerm = 4;
        # repeated LogEntry entries = 5;
        # int32 leaderCommit = 6;
        self.leaderID = request.leaderID
        if random.uniform(0,
                          1) < self.cmserver.fail_mat[self.leaderID][self.id]:
            self.logger.warning(
                f'[ABORTED]: append entries from server <{self.leaderID}> '
                f'to <{self.id}>, because of ChaosMonkey')
        else:
            # Todo: if election timeout elapse without receiving AppendEntries RPC from current leader or granting vote
            #  to candidate: convert to candidate
            self.lastUpdate = time.time()
            success = False
            try:
                # Todo: If candidates receive AppendEntries RPC from a leader, convert to follower
                #  mcip: request term should be useless coz of last log term??????
                if self.role == KVServer.candidate:
                    self.save(current_term=max(self.lastLogTerm, request.term),
                              voted_for=-1)
                    self.role = KVServer.follower
                    with self.candidateCond:
                        self.candidateCond.notify_all()
                else:
                    # Todo: All servers: If RPC request or response contains term T> currentTerm,
                    #  set current term = T, convert to follower
                    self.convToFollowerIfHigherTerm(request.term, voted_for=-1)

                tmp_entries = []
                for row in request.entries:
                    r = [row.term, row.key, row.val]
                    tmp_entries.append(r)
                    # self.logger.info(f'row: <{r}>')
                # reply false if term < currentTerm,
                # or log doesn't log doesn't contain an entry at prevLogIndex whose term matches prevLogTerm
                # Todo: if it doesn't match the term, it will decrement and resend, thus following will remove entries
                if request.term < self.currentTerm or request.prevLogIndex > len(self.log) \
                        or (request.prevLogIndex < len(self.log) and
                            self.log[request.prevLogIndex][0] != request.prevLogTerm):

                    self.logger.warning(
                        f'[AP_En]: received on <{self.id}>, will return false to <{self.leaderID}>'
                    )
                    self.logger.warning(
                        f'[AP_En]: <{request.term < self.currentTerm}>, '
                        f'<{request.prevLogIndex > len(self.log)}>, '
                        f'<{(request.prevLogIndex < len(self.log) and self.log[request.prevLogIndex][0] != request.prevLogTerm)}>'
                    )
                    self.logger.info(
                        f'Parameters for false: req term: <{request.term}>, cur term: '
                        f'<{self.currentTerm}>, req prevLogIdx: <{request.prevLogIndex}>, '
                        f'length of server log <{len(self.log)}>')
                    if request.prevLogIndex < len(self.log):
                        self.logger.info(
                            f'term of log on prev log index: <{self.log[request.prevLogIndex][0]}>, '
                            f'request prev log term: <{request.prevLogTerm}>')
                        #     existing entry conflicts with a new one, same idx different terms,
                        #     delete the existing entry and all that follow it
                        # self.logger.info(f'RAFT: checking conflicting entries')
                        itr = 0
                        for a, b in zip(tmp_entries,
                                        self.log[request.prevLogIndex:]):
                            if a != b:
                                self.logger.warning(
                                    f'[Log]: Found conflict at index: '
                                    f'<{request.prevLogIndex + itr}>')
                                self.logModify(request.prevLogIndex + itr,
                                               LogMod.DELETION)
                            itr += 1
                else:
                    self.save(current_term=max(self.currentTerm, request.term),
                              voted_for=-1)
                    # self.logger.info("RAFT: AppendEntries should succeed unless there is conflict entries")
                    success = True
                    #     existing entry conflicts with a new one, same idx different terms,
                    #     delete the existing entry and all that follow it
                    # self.logger.info(f'RAFT: checking conflicting entries')
                    itr = 0
                    if len(self.log) > 0:
                        for a, b in zip(tmp_entries,
                                        self.log[request.prevLogIndex:]):
                            if a != b:
                                self.logger.warning(
                                    f'[Log]: Found conflict at index: '
                                    f'<{request.prevLogIndex + itr}>')
                                self.logModify(request.prevLogIndex + itr,
                                               LogMod.DELETION)
                            itr += 1
                    # Heartbeat
                    if len(tmp_entries) == 0:
                        self.logger.info("[Log]: received a heartbeat")
                    # Normal append entries
                    else:
                        self.logger.info(
                            f"[Log]: append entries, leader commit <{request.leaderCommit}>"
                        )
                        # Append any new entries not already in the log
                        to_append_length = request.prevLogIndex + len(
                            tmp_entries) - len(self.log)
                        # self.logger.debug(f'RAFT: length of log to append: <{to_append_length}>')
                        if to_append_length > 0:
                            self.logModify(tmp_entries[-to_append_length:],
                                           LogMod.ADDITION)
                        # If leaderCommit > commitIndex, set commitIndex = min(leaderCommit, index of last new entry)
                        # print("Received log from appendEntries is: ", tmp_entries)
                        # self.logger.debug(f'RAFT: Checking if we need to write to disk: <{request.leaderCommit}>,'
                        #                   f'<{self.commitIndex}>, <{self.lastLogIndex}>')
                        if request.leaderCommit > self.lastApplied:
                            self.logger.info(
                                f"[Log]: apply to state machine, leader commit <{request.leaderCommit}> "
                                f"last applied <{self.lastApplied}>")
                            self.commitIndex = min(request.leaderCommit,
                                                   self.lastLogIndex)
                            app_state_mach_kth = KThread(
                                target=self.applyToStateMachine,
                                args=(self.lastApplied, ))
                            app_state_mach_kth.start()
                    # int32 term = 1;
                    # bool success = 2;
                self.lastUpdate = time.time()
                return kvstore_pb2.AppendResponse(term=self.currentTerm,
                                                  success=success)
            except Exception as e:
                self.logger.error("RAFT[Vote]: f(): appendEntries:")
                self.logger.error(e)

    # Todo: always use this to update log
    def logModify(self, para, operation: LogMod):
        # self.logger.debug(f'RAFT[Log]: Log modify: <{para}>, '
        #                   f'operation: <{operation}>')
        if operation == LogMod.ADDITION:
            self.log += para
        elif operation == LogMod.APPEND:
            self.log.append(para)
        elif operation == LogMod.REPLACEMENT:
            self.log = para
        elif operation == LogMod.DELETION:
            self.log = self.log[:para]
        self.lastLogIndex = len(self.log) - 1
        self.lastLogTerm = self.log[self.lastLogIndex][0]
        with open(self.diskLog, 'wb') as f:
            pkl.dump(self.log, f)
        #     Todo: not needed when it's the leader, what about follower?
        if self.id == self.leaderID:
            self.matchIndex[self.id] = self.lastLogIndex
            # Wait until last committed entry is from leader's term, notify all upon leader's log change
            with self.lastCommittedTermCond:
                self.lastCommittedTermCond.notify_all()
        if self.lastLogTerm > self.currentTerm:
            self.save(current_term=self.lastLogTerm, voted_for=self.votedFor)
        self.logger.info(f'[Log]: Log updated on disk of server <{self.id}> ,'
                         f'last log index now: <{self.lastLogIndex}>, '
                         f'log is: LOG!!! ')  # <{self.log}>')

    def applyToStateMachine(self, last_applied):
        # TODO: maybe we can append only? maybe we need synchronization
        to_update = self.log[last_applied + 1:self.commitIndex + 1]
        for row in to_update:
            self.stateMachine[row[1]] = row[2]
        with open(self.diskStateMachine, 'wb') as f:
            pkl.dump(self.stateMachine, f)
        self.lastApplied = self.commitIndex
        # Apply command in log order, notify all upon completion
        with self.appliedStateMachCond:
            self.appliedStateMachCond.notify_all()
        self.logger.info(
            f'[StateMach]: Last applied index: <{self.lastApplied}>, ')
        # f'state machine updated to: <{self.stateMachine}>')

    # def readWithKey(self, key):
    #     n = len(self.log)
    #     for i in range(n - 1, -1, -1):
    #         if self.log[i][1] == key: return self.log[i][2]
    #     return ""

    def run(self):
        # Create a thread to run as follower
        leader_state = KThread(target=self.leader, args=())
        leader_state.start()
        candidate_state = KThread(target=self.candidate, args=())
        candidate_state.start()
        follower_state = KThread(target=self.follower, args=())
        follower_state.start()

    # Checkpoint 1 Get Put Methods
    # Todo: no longer needed?
    # def localGet(self, key):
    #     '''
    #     val = self.readWithKey(key)
    #     if val == "": return kvstore_pb2.GetResponse(ret = kvstore_pb2.FAILURE, value = val)
    #     else: return kvstore_pb2.GetResponse(ret = kvstore_pb2.SUCCESS, value = val)
    #     '''
    #     resp = kvstore_pb2.GetResponse()
    #     try:
    #         resp.value = self.stateMachine[key]
    #         resp.ret = kvstore_pb2.SUCCESS
    #         self.logger.info(f'RAFT[KVStore]: localGet <{key}, {resp.value}>')
    #     except KeyError:
    #         resp.ret = kvstore_pb2.FAILURE
    #         self.logger.warning(f'RAFT[KVStore]: localGet failed, no such key: [{key}]')
    #     return resp

    # Todo: no longer needed?
    # def localPut(self, key, val):
    #     resp = kvstore_pb2.PutResponse()
    #     self.stateMachine[key] = val  # dictionary
    #     resp.ret = kvstore_pb2.SUCCESS
    #     self.logger.info(f'RAFT[KVStore]: localPut <{key}, {val}>')
    #     return resp

    # Todo: add client ID and sequence number
    def Get(self, request, context):
        try:
            # string key = 1;
            # Reply NOT_LEADER if not leader, providing hint when available
            if self.role != KVServer.leader:
                # string value = 1;
                # ClientRPCStatus status = 2;
                # int32 leaderHint = 3;
                self.logger.info(
                    f'[KVStore]: Get redirect to leader <{self.leaderID}>')
                return kvstore_pb2.GetResponse(value="",
                                               status=kvstore_pb2.NOT_LEADER,
                                               leaderHint=self.leaderID)
            try:
                # Wait until last committed entry is from leader's term
                with self.lastCommittedTermCond:
                    self.lastCommittedTermCond.wait_for(lambda: self.log[
                        self.commitIndex][0] == self.currentTerm)
                    # Save commitIndex as local variable readIndex
                    read_index = self.commitIndex
                # Todo: Is this done?
                # Send new round of heartbeats, and wait for reply from majority of servers
                with self.appendEntriesCond:
                    self.appendEntriesCond.notify_all()
                # Wait for state machine to advance at least the readIndex log entry
                with self.appliedStateMachCond:
                    self.appliedStateMachCond.wait_for(
                        lambda: self.lastApplied >= read_index)
                # Process query
                # Reply OK with state machine output
                self.logger.info(
                    f'[KVStore]: Get success: <{request.key}, {self.stateMachine[request.key]}>'
                )
                context.set_code(grpc.StatusCode.OK)
                return kvstore_pb2.GetResponse(
                    value=self.stateMachine[request.key],
                    status=kvstore_pb2.OK2CLIENT,
                    leaderHint=self.id)
            except KeyError:
                self.logger.warning(
                    f'[KVStore]: Get failed, no such key: [{request.key}]')
                context.set_code(grpc.StatusCode.CANCELLED)
                return kvstore_pb2.GetResponse(value="",
                                               status=kvstore_pb2.ERROR2CLIENT,
                                               leaderHint=self.id)
        except Exception as e:
            self.logger.error("RAFT[KVStore]: f(): Get")
            self.logger.error(e)

    def Put(self, request, context):
        try:
            # string key = 1;
            # string value = 2;
            # int32 clientID = 3;
            # int32 sequenceNum = 4;
            # NOT_LEADER = 0;
            # SESSION_EXPIRED = 1;
            # OK2CLIENT = 2;
            # ERROR2CLIENT = 3;
            # if command received from client: append entry to local log, respond after entry applied to state machine
            # Reply NOT_LEADER if not leader, providing hint when available
            if self.role != KVServer.leader:
                return kvstore_pb2.PutResponse(status=kvstore_pb2.NOT_LEADER,
                                               response="",
                                               leaderHint=self.leaderID)
            # Reply SESSION_EXPIRED if not record of clientID or if the response for client's sequenceNum
            # already discarded
            if request.clientID not in self.registeredClients or \
                    self.clientReqResults[request.clientID][1] > request.sequenceNum:
                return kvstore_pb2.PutResponse(
                    status=kvstore_pb2.SESSION_EXPIRED,
                    response="",
                    leaderHint=self.leaderID)
            # If sequenceNum already processed from client, reply OK with stored response
            if self.clientReqResults[
                    request.clientID][1] == request.sequenceNum:
                return kvstore_pb2.PutResponse(
                    status=kvstore_pb2.OK2CLIENT,
                    response=self.clientReqResults[request.clientID][0],
                    leaderHint=self.leaderID)
            # Todo: Following line has correct order?
            # Append command to log, replicate and commit it
            self.logModify([[self.currentTerm, request.key, request.value]],
                           LogMod.ADDITION)
            put_log_ind = self.lastLogIndex
            # wake up threads to append entries
            with self.appendEntriesCond:
                self.appendEntriesCond.notify_all()
            # Apply command in log order
            with self.appliedStateMachCond:
                self.appliedStateMachCond.wait_for(
                    lambda: (self.lastApplied >= put_log_ind),
                    timeout=self.requestTimeout)
            # Save state machine output with sequenceNum for client, discard any prior sequenceNum for client
            self.clientReqResults[request.clientID] = [
                self.stateMachine[request.key], request.sequenceNum
            ]
            # ClientRPCStatus status = 1;
            # string response = 2;
            # int32 leaderHint = 3;
            # Todo: no need for state machine output for put?
            # Reply OK with state machine output
            if self.lastApplied >= put_log_ind:
                self.logger.info(
                    f'[KVStore]: Server put success on leader <{self.id}>')
                context.set_code(
                    grpc.StatusCode.OK)  # Todo: why is this needed?
                return kvstore_pb2.PutResponse(
                    status=kvstore_pb2.OK2CLIENT,
                    response=self.stateMachine[request.key],
                    leaderHint=self.id)
            else:
                self.logger.warning(
                    f'[KVStore]: Server put error (timeout?) on leader <{self.id}>'
                )
                context.set_code(grpc.StatusCode.CANCELLED)
                return kvstore_pb2.PutResponse(status=kvstore_pb2.ERROR2CLIENT,
                                               response="",
                                               leaderHint=self.id)
        except Exception as e:
            self.logger.error("RAFT[KVStore]: f(): Put")
            self.logger.error(e)

    def registerClient(self, request, context):
        try:
            # ClientRPCStatus status = 1;
            # int32 clientID = 2;
            # int32 leaderHint = 3;
            # Reply NOT_LEADER if not leader, provide hint when available
            if self.role != KVServer.leader:
                return kvstore_pb2.RegisterResponse(
                    status=kvstore_pb2.NOT_LEADER,
                    clientID=-1,
                    leaderHint=self.leaderID)
            else:
                # Append register command to log, replicate and commit it
                cur_last_log_ind = len(self.log)
                self.logModify([[
                    self.currentTerm, "client-" + str(cur_last_log_ind),
                    str(cur_last_log_ind)
                ]], LogMod.ADDITION)
                # wake up threads to register clients
                with self.appendEntriesCond:
                    self.appendEntriesCond.notify_all()
                    self.registeredClients.append(
                        cur_last_log_ind
                    )  # Todo: faster if we put following 2 here?
                    self.clientReqResults[cur_last_log_ind] = [
                        "", -1
                    ]  # init client result dictionary
                # Apply command in log order, allocating session for new client
                with self.appliedStateMachCond:
                    self.logger.info(
                        f'[Client]: Register client: lastApplied, <{self.lastApplied}>, '
                        f'cur_last_log_ind, <{cur_last_log_ind}>, matchIndex, <{self.matchIndex}>'
                    )
                    self.appliedStateMachCond.wait_for(
                        lambda: self.lastApplied >= cur_last_log_ind)
                # Todo: allocating new session?
                # Reply OK with unique client identifier (the log index of the register command could be used)
                return kvstore_pb2.RegisterResponse(
                    status=kvstore_pb2.OK2CLIENT,
                    clientID=cur_last_log_ind,
                    leaderHint=self.leaderID)
        except Exception as e:
            self.logger.error("RAFT[Register]: f(): registerClient")
            self.logger.error(e)

    def clientRequest(self, request, context):
        pass

    def clientQuery(self, request, context):
        pass

    def updateConfigs(self, request, context):
        # int32 requestTimeout = 1;
        # int32 maxElectionTimeout = 2;
        # int32 keySizeLimit = 3;
        # int32 valSizeLimit = 4;
        try:
            self.requestTimeout = request.requestTimeout
            self.maxElectionTimeout = request.maxElectionTimeout
            self.keySizeLimit = request.keySizeLimit
            self.valSizeLimit = request.valSizeLimit
            # ReturnCode ret = 1;
            return kvstore_pb2.UpdateConfigsResponse(ret=kvstore_pb2.SUCCESS)
        except Exception as e:
            self.logger.error("RAFT[ConfigChn]: f(): updateConfigs\n" + e)
            return kvstore_pb2.UpdateConfigsResponse(ret=kvstore_pb2.FAILURE)
class Server(object):
    def __init__(self, id_):
        self.id = id_
        self.friendlyName = self._formFriendlyName(id_)
        self.configFile = 'config-%d' % self.id
        logger.debug('Server Init. Config File = {}'.format(self.configFile))
        self.role = 'follower'
        self.commitIndex = 0
        self.lastApplied = 0
        self.leaderID = 0
        address = json.load(file('config.json'))
        port_list = address['AddressBook']
        running = address['running']
        self.groupInfo = {}
        # self.initialState = 100
        self.addressbook = {}
        for id_ in running:
            self.addressbook[id_] = port_list[id_ - 1]
            
        # need to put it into file later on
        self.load()


        self.port = self.addressbook[self.id]
        self.request_votes = self.peers[:]

        self.numVotes = 0
        self.oldVotes = 0
        self.newVotes = 0

        self.lastLogIndex = 0
        self.lastLogTerm = 0

        self.listener = KThread(target = self.listen, args= (acceptor,))
        self.listener.start()
        logger.info('Started listening on port {}'.format(self.port))

        self.during_change = 0
        self.newPeers = []
        self.new = None
        self.old = None

    def listen(self, on_accept):
        logger.debug('Server Listen Method')
        srv = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        srv.bind(("", self.port))
        print 'start listenning ', self.id, " ", self.currentTerm
        while True:
            data,addr = srv.recvfrom(1024)
            #print 'listening ', self.id
            thr = KThread(target=on_accept, args=(self,data,addr))
            thr.start()
        srv.close()

    def follower(self):
        logger.debug('Server Follower Method')
        print ' '
        print '*************************'
        print 'Running as a follower ', self.id, " ", self.currentTerm
        print 'My ID is ', self.id
        print "The terms is ", self.currentTerm
        print "My Peers are ", self.peers
        print '*************************'
        logger.info('Running as a follower ')
        self.role = 'follower'
        self.last_update = time.time()
        election_timeout = 5 * random.random() + 5
        while time.time() - self.last_update <= election_timeout:
            pass
        self.startElection()
        while True:
            self.last_update = time.time()
            election_timeout = 5 * random.random() + 5
            while time.time() - self.last_update <= election_timeout:
                pass

            if self.election.is_alive():
                self.election.kill()
            self.startElection()

    def startElection(self):
        logger.info('Server startElection Method server ID - {} Current Term = {}'.format(self.id, self.currentTerm))
        self.role = 'candidate'
        self.election = KThread(target =self.threadElection, args = ())
        if len(self.peers) != 0:
            self.currentTerm += 1
            self.votedFor = self.id
            self.save()
            self.numVotes = 1

            self.election.start()

    def threadElection(self):
        logger.info('Server threadElection Method')
        print 'timeouts, start a new election with term %d' % self.currentTerm
        self.role = 'candidate'
        self.request_votes = self.peers[:]
        sender = self.id

        while 1:
            for peer in self.peers:
                if peer in self.request_votes:
                    Msg = str(self.lastLogTerm) + ' ' + str(self.lastLogIndex)
                    msg = RequestForVoteMessage(sender, peer, self.currentTerm, Msg)
                    data = pickle.dumps(msg)
                    sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
                    sock.sendto(data, ("", self.addressbook[peer]))
                    sock.close()
            time.sleep(1) # wait for servers to receive

    def leader(self):
        logger.info('Server leader Method')
        print ' '
        print '*************************'
        print 'Running as a leader'
        print 'My ID is ', self.id
        print "The terms is ", self.currentTerm
        print "My Peers are ", self.peers
        print '*************************'
        logger.info('Running as a leader')
        self.role = 'leader'
        self.nextIndex = {}
        self.matchIndex = {}
        for peer in self.peers:
            self.nextIndex[peer] = len(self.log) + 1
            self.matchIndex[peer] = 0
        nodes = {}
        leaderNodeInfo = NodeInformation(self.id, self._formFriendlyName(self.id), 1)
        nodes[self.id] = leaderNodeInfo
        for peer in self.peers:
            peerNodeInfo = NodeInformation(peer, self._formFriendlyName(peer), 1)
            nodes[peer] = peerNodeInfo
        # self.groupInfo[SERVER_NODE_GROUP_NAME] = nodes
        _uuid = uuid.uuid1()
        newAppendLogEntry = LogEntry(self.currentTerm, self.groupInfo, BaseMessage.LocalMessageAddress, _uuid)
        self.log.append(newAppendLogEntry)
        self.appendEntries()

    def _formFriendlyName(self, id):
        return 'S '+ str(id)


    def appendEntries(self):
        logger.debug('Server appendEntries Method ', self.id, " ", self.currentTerm)
        receipts = self.peers[:]
        while 1:
            receipts = self.peers[:]
            if self.during_change != 0:
                for peer in receipts:
                    if peer not in self.nextIndex:
                        self.nextIndex[peer] = len(self.log) + 1
                        self.matchIndex[peer] = 0
            for peer in receipts:
                if len(self.log) >= self.nextIndex[peer]:
                    prevLogIndex = self.nextIndex[peer] - 1
                    if prevLogIndex != 0:
                        prevLogTerm = self.log[prevLogIndex-1].term
                    else:
                        prevLogTerm = 0
                    entries = [self.log[self.nextIndex[peer]-1]]
                else:
                    entries = []
                    prevLogIndex = len(self.log)
                    if prevLogIndex != 0:
                        prevLogTerm = self.log[prevLogIndex-1].term
                    else:
                        prevLogTerm = 0

                msg = AppendEntriesMessage(self.id, peer, self.currentTerm, entries, self.commitIndex, prevLogIndex, prevLogTerm)
                data = pickle.dumps(msg)
                sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
                sock.sendto(data, ("", self.addressbook[peer]))
                sock.close()
            time.sleep(5)

    def stepDown(self):
        logger.debug('Server stepDown Method')
        if self.role == 'candidate':
            print 'candidate step down when higher term and becomes follower', self.id, " ", self.currentTerm
            self.election.kill()
            self.last_update = time.time()
            self.role = 'follower'
        elif self.role == 'leader':
            self.leader_state.kill()
            self.follower_state = KThread(target = self.follower, args = ())
            self.follower_state.start()

    def load(self):
        print 'Server load config Method ', self.id
        initial_running = [1,2,3,4]
        try:
            with open(self.configFile) as f:
                serverConfig = pickle.load(f)
        except Exception as e:
            logger.error('Exception in loading the config file {}'.format(e))
            initialState = {}
            initialState[SERVER_NODE_GROUP_NAME] = {}
            if self.id not in initial_running:
                logger.info('Not a part of initial running. Starting with empty configuration')
                # TODO: THis is the tricky point
                serverConfig = ServerConfig(initialState, 0, -1, [], [])
            else:
                logger.info('Part of initial running list. So trying to connect with others as peers')
                initial_running.remove(self.id)
                serverConfig = ServerConfig(initialState, 0, -1, [], initial_running)

        self.groupInfo = serverConfig.groupInfo
        self.currentTerm = serverConfig.currentTerm
        self.votedFor = serverConfig.votedFor
        self.log = serverConfig.log
        self.peers = serverConfig.peers
        self.majority = (len(self.peers) + 1)/2 + 1

    def save(self):
        logger.debug('Server save config Method ', self.id, " ", self.currentTerm)
        serverConfig = ServerConfig(self.groupInfo, self.currentTerm, self.votedFor, self.log, self.peers)
        with open(self.configFile, 'w') as f:
            pickle.dump(serverConfig, f)
            f.flush()

    def run(self):
        logger.debug('Server thread run Method ', self.id, " ", self.currentTerm)
        time.sleep(1)
        self.follower_state = KThread(target = self.follower, args = ())
        self.follower_state.start()
Пример #24
0
class CommObject():
  def __init__(self, host='localhost', userid='guest', password='******'): 
    self.error = False
    self.debug = False
    self.host = host
    self.userid = userid
    self.password = password        
    self.configList = []            # scalable config for rabbitmq
    self.chanList = []              # [id,obj] ordered pairs
    self.lockObj = Lock()           # used for thread safety
    self.thread_params = list()     # used for passing params into thread
       
    self.exchange = 'myexchange'
    self.uuid = str(uuid.uuid1())  # make each queue unique
    self.myqueue = 'myqueue' + self.uuid
    self.mykey = 'myq.myx'
    self.running = False
    self.subscribe = None # self.subscribe.start() to start thread for listening on exchange
 
    try:     
      self.connection = amqp.Connection(userid=self.userid, 
                           password=self.password, host=self.host,
                           virtualhost='/', ssl=False)
    except Exception as eobj:
      print eobj
      print "Is the server running? (sh rabbitmq-server-2.1.1/scripts/rabbitmq-server)"
      self.error = True
      #
      # an object is returned, not None
      #
      
  # end __init__ 
  #-----------------------------------------------------------   
  def SimpleConfigure(self):  
    try:
      self.channel = self.connection.channel()
    except Exception as eobj:
      print eobj
      self.connection.close()
      return -1
      
    print 'channel = ', self.channel.channel_id
    
    try:
      self.channel.access_request('/data', active=True, write=True, read=True)
    except Exception as eobj:
      print eobj
      self.channel.close()
      return -1
      
    #
    # Note: When configuring/reconfiguring the server may not like changes
    #       to existing exchanges. Closing channel on exception clears server.
    
    #
    # with mulitiple listeners 'direct' ping-pongs 1:1 across n listeners
    #
    
    #
    # defaults on exchange_declare:are same as queue_declare:
    # passive=False, durable=False,
    # auto_delete=True, internal=False, nowait=False
    #
    # if you mix defaults between the two, then you will raise an exception
    # 
    # auto_delete -> queue is deleted when last consumer closes
    # durable -> survives server reboot
    # passive -> when set tests presence of queue, if not present then "access refused" returned
    # internal -> special purpose, keep false
    # nowait -> server doesn't reply, keep false
    #
    queue_type = 'fanout' # 'direct' # 
    try:
      self.channel.exchange_declare(self.exchange,type=queue_type,durable=False, auto_delete=True)
    except Exception as eobj:
      print eobj
      print "The server is unhappy."
      print "You re-declared an exchange and changed a parameter."
      print "Kill and restart the server."
      self.channel.close()
      return -1
      
    try:  
      self.channel.queue_declare(self.myqueue,durable=False, auto_delete=True)
    except Exception as eobj:
      print eobj
      self.channel.close()
      return -1
    
    try:    
      self.channel.queue_bind(self.myqueue,self.exchange,routing_key=self.mykey)  
    except Exception as eobj:
      print eobj
      self.channel.close()
      return -1
      
  # end SimpleConfigure
  #-----------------------------------------------------------
  #
  # private
  #
  def _getChannelObject(self,channelList,chid):
    # channelList format:
    #   [ [<channel-id>, <channelObject>] ]
    ii = 0
    while (ii<len(channelList)):
      chpair = channelList[ii]
      if (chid == chpair[0]):
        return chpair[1]
      
      ii = ii+1
      
      #end while 
      
    return None
  #end _getChannelObject
  #-----------------------------------------------------------
 
  def Configure(self, config=[1, 'myexchange', 'direct', 'myqueue', 'myq.myx']):
    #
    # in: config format:
    #   [<channel>, <exchange>, <type>, <optional_queue>, <optional_routing_key>] 
    #        0          1         2            3                4
    #
    # queue is not necessary for writes, only for reads
    # routing_key is used for filtering reads
    #
    # out: self.configList, self.chanList
    #   chanlList format:
    #   [ [<channel-id>, <channelObject>] ]
    #          0            1
    #
    #
    # entity-relationship for channels:
    #
    # 1 configuration : multiple channels
    # 1 channel : multiple exchanges
    # 1 exchange : 0,1 queue
    # 1 queue : 0,1 message tag
    #
    err_f = False
    channelList = self.chanList
   
    #
    # if channel does not exist then create channel
    # else skip create, use pre-existing channel in channelList
    #     
    chobj = self._getChannelObject(channelList,config[0])
    if (chobj == None):
      try:
         chobj = self.connection.channel()
      except Exception as eobj:
         print eobj
	 err_f = True
	 
    if (not err_f):  
      try:
        chobj.access_request('/data',active=True,write=True,read=True)
      except Exception as eobj:
         print eobj
         err_f = True
         channel.close()
       
    if (not err_f):	 
      try:
        chobj.exchange_declare(exchange=config[1],type=config[2],durable=False, auto_delete=True)
      except Exception as eobj:
        print eobj
        print "The server is unhappy."
        print "You re-used an exchange: %s and changed the type parameter: %s." % (config[1], config[2])
        print "Closing the channel. Rename the exchange or kill and restart the server (server has memory)."
        chobj.close()
	err_f = True
    #
    # (type, queue, publish) = ('direct', routing key, match routing key)             -> yes delivery
    # (type, queue, publish) = ('direct', routing key, no or mismatched routing key) -> no 
    # (type, queue, publish) = ('fanout', routing key, no or mismatched routing key) -> yes
    #	
    if (not err_f):
      if (config[3] != None): 	   # queues are for reading from exchange
        try:  
          chobj.queue_declare(queue=config[3],durable=False,auto_delete=True)
        except Exception as eobj:
          print eobj
          err_f = True
        
        if (not err_f):	   
          try:	
            chobj.queue_bind(queue=config[3],exchange=config[1],routing_key=config[4])  
          except Exception as eobj:
            print eobj
            err_f = True            
      
    newElement = []
    
    #
    # build object's channel list out of channel id from configuration list plus channel object
    # build objects's configuration list out of input list
    #
    if (not err_f):
      newElement = [config[0]] + [chobj]
      self.chanList = self.chanList + [ newElement ] # building list of lists
      
      self.configList = self.configList + [ config ] # building list of lists
      
    return err_f
    
  #end Configure
 
  #-----------------------------------------------------------
     
  def ConfigureList(self, configSet):
    #
    # config format:
    #   [[<channel>, <exchange>, <type>, <optional_queue>, <optional_routing_key>], ...]
    # 
    # channelList format:
    #   [ <channel-id>, <channelObject> ]
    #
    
    err_f = False
    
    ii=0                      # channelList is poplulated as channels are created
    while (ii<len(configSet) and (not err_f)):
      err_f = self.Configure(configSet[ii])
      if (err_f):
        break
	
      ii = ii +1
    #end while
    
    return err_f
    
  #end ConfigureList
  #-----------------------------------------------------------     
  def SimplePut(self,msg_in):
    
    #
    # routing key is used for 'direct' exchanges
    #
    base_time = time.time();
    self.channel.basic_publish(amqp.Message(msg_in),self.exchange,self.mykey)
    #
    # (type, queue, publish) = ('direct', routing key, routing key)                  -> yes delivery
    # (type, queue, publish) = ('direct', routing key, no or mismatched routing key) -> no 
    # (type, queue, publish) = ('fanout', routing key, no or mismatched routing key) -> yes
    #
    
    print "sec:", time.time() - base_time
  # end SimplePut
  #-----------------------------------------------------------
  def _searchListOfLists(self,listOfLists,  # form: [ [...], [...],...]
                              index,        # index into unwrapped list
			      key):         # what to match
    match_f = False
    ii =0
    
    while(ii<len(listOfLists)):
      theList = listOfLists[ii]
      if (theList[index] == key):
        match_f = True
	break
      ii = ii + 1	
    #end while
    
    return match_f, ii
  #end _searchListOfLists
  
  #-----------------------------------------------------------     
  def Put(self,msg_in,exchange=None, routing_key=None):
    #
    # the purpose with the defaults is to allow the creation of simple communication links
    # with the minimal amount of programmatic effort (for testing), in parallel with
    # complex links that scale for large networks
    #
    err_f = False
    if (exchange == None):
      exchange = self.exchange
      routing_key = self.mykey    # routing key is used for 'direct' exchanges
      ch_obj = self.channel
    else:
       # search the configure list for exchange, match index yields channel id and routing_key
       # search the channel list for channel id, match yields channel object
       
       # there are many ways for this function to fail by
       # passing in bad parameters
       try:
         match_f, e_index = self._searchListOfLists(self.configList,1,exchange)
       except Exception as eobj:
          print eobj
	  err_f = True
	  return err_f
	  	 
       if (not match_f):
         print "Put(), exchange: %s not found in %s" % (exchange,self.configList)
	 err_f = True
	 return err_f
	 
       chan_id = self.configList[e_index][0]
       routing_key = self.configList[e_index][4]
	 
       # there are many ways for this function to fail by
       # passing in bad parameters
       try:
         match_f, c_index = self._searchListOfLists(self.chanList,0,chan_id)
       except Exception as eobj:
          print eobj
	  err_f = True
	  return err_f
	  
       if (not match_f):
         print "Put(), channel %d not found in: %s" % (chan_id,self.chanList)
	 err_f = True
	 return err_f
       
       ch_obj = self.chanList[c_index][1]
   
    ch_obj.basic_publish(amqp.Message(msg_in),exchange,routing_key)
      
    return err_f
  # end Put
  
  #-----------------------------------------------------------  
  def SimpleGet(self):
    try:
      response = self.channel.basic_get(self.myqueue,no_ack=False)
    except Exception as eobj:
      print self.myqueue, eobj
      response = None
      
    if response is not None:
      print "ch: %d, body = \"%s\"" %(self.channel.channel_id, response.body) 
      self.channel.basic_ack(response.delivery_tag)
    else:
      print "no message" 
       
  # end SimpleGet
  #-----------------------------------------------------------
  def _queue2chobj(self,queue):
    ch_obj = None
    err_f = False
    #
    # given queue, search for channel id
    #
    try:
         match_f, q_index = self._searchListOfLists(self.configList,3,queue)
    except Exception as eobj:
          print eobj
	  err_f = True
      	  
    if (not err_f):  	 
        if (not match_f):
          print "_queue2chobj(), queue %s not found in: %s" % (queue,self.configList)
	  err_f = True
	  
    if (not err_f): 
        chan_id = self.configList[q_index][0]
	 
        # there are many ways for this function to fail by
        # passing in bad parameters
        try:
          match_f, c_index = self._searchListOfLists(self.chanList,0,chan_id)
        except Exception as eobj:
          print eobj
	  err_f = True
	  
    if (not err_f):  
        if (not match_f):
          print "_queue2chobj(), channel %d not found in: %s" % (chan_id,self.chanList)
	  err_f = True
	  
    if (not err_f):         
        ch_obj = self.chanList[c_index][1]
	
    return err_f, ch_obj
  # end _queue2chobj
  #-----------------------------------------------------------  
  def Get(self, queue=None):
    err_f = False
    msg_f = False
    msg = None
    if (queue == None):   # for simple configuration case
      queue = self.myqueue
      ch_obj = self.channel
    else:
      #
      # given queue, search for channel id
      #
      err_f, ch_obj = self._queue2chobj(queue)
	  
      if (not err_f):         
	response = None  
        try:
          response = ch_obj.basic_get(queue,no_ack=False)
        except amqp.Timeout:
	  pass                 # not an error
	except: 
         print "queue=", queue
	 print eobj
         err_f = True
     
      if (not err_f): 
        if response is not None:
	  msg_f = True
          msg = response.body
          ch_obj.basic_ack(response.delivery_tag)
        else:
          print "no data" 
      
    return err_f, msg_f, msg
  # end Get  
  #-----------------------------------------------------------     
  #
  # private
  #  
  def _mycallback(self,msg):
    print
    for key,val in msg.properties.items():
      print '%s: %s' % (key,str(val))
    for key,val in msg.delivery_info.items():
      print '> %s: %s' % (key, str(val))
    print 'received <', msg.body, '> from channel #', msg.channel.channel_id
    msg.channel.basic_ack(msg.delivery_tag)
  # end __mycallback
  #----------------------------------------------------------- 
  #
  # private
  #    
  def _SimpleSubscribeThread(self):
     self.channel.basic_consume(queue=self.myqueue, callback=self._mycallback, no_ack=False)
     #
     # init loop
     #
     self.running = True
     #
     # loop - need a shared object (event) to determine external stop
     # 
     while self.running:
        try:
           self.channel.wait(timeout = 5)   
        except amqp.Timeout:
           print "timeout"
      
  # end _SimpleSubscribeThread
  #----------------------------------------------------------- 
  #
  # private
  #    
  def _SubscribeThread(self):
    #
    # do once: get loop invarients 
    #      
    self.lockObj.acquire()
    if len(self.thread_params):
       #
       # note: it is very easy to mess up order when
       #       adding/changing parameters
       #
       evobj = self.thread_params.pop()     # 3
       callback = self.thread_params.pop()  # 2
       queue = self.thread_params.pop()     # 1
       
    self.lockObj.release()
    
    #
    # given queue, return channel 
    #
    err_f, ch_obj = self._queue2chobj(queue)
    if (err_f):
      print "Subscribe failed"
    else:
      ch_obj.basic_consume(queue=queue, callback=callback, no_ack=False)
      #
      # init loop
      #
      self.running = True
      #
      # loop - need a shared object (event) to determine external stop
      # 
      while not self.ev_obj.is_set():    # while not signaled halt
        try:
           ch_obj.wait(timeout = 5)   
        except amqp.Timeout:
	   if (self.debug):
             print "timeout"
      
  # end _SubscribeThread
  #----------------------------------------------------------- 
  #    
  def SimpleSubscribe(self):
  
     self.subscribe = KThread(target = self._SimpleSubscribeThread)
     self.subscribe.start() 
     
  # end SimpleSubscribe
  
  #----------------------------------------------------------- 
  #    
  def Subscribe(self,queue=None,callback=None):
     #
     # the queue and callback being subscribed to is not passed in here
     # but passed in through via "self"
     # the question is how to make this scale?
     #
     #
     #
     
     self.lockObj.acquire()     
     if (queue == None):    # done for compatibility with "simple" version
       queue = self.myqueue
     if (callback == None):
       callback = self._mycallback
       
     #
     # communication parameters within thread
     #
     self.thread_params.append(queue)       # 1
     self.thread_params.append(callback)    # 2
     #
     # control param for thread
     #
     self.ev_obj = Event()
     self.thread_params.append(self.ev_obj)      # 3 - top of stack
      
     subscribe = KThread(target = self._SubscribeThread)
     subscribe.start() 
     self.lockObj.release()    
     #
     # return thread and its control object
     #
     return subscribe, self.ev_obj
     
  # end Subscribe
  #----------------------------------------------------------- 
  #
  def Unsubscribe(self,thread_object, event_object):
     #
     # object may no longer exists, so bound by "try"
     # also, belts and suspenders, i.e. signal before kill
     #
     try: 
       event_object.set()    # signals to stop running
     except:
       print "attempting to signal a non-existent event"
     try:
       thread_object.kill()    
     except Exception as eobj:
       print "attempting to signal a non-existent thread"   
  # end SimpleUnsubscribe
  #----------------------------------------------------------- 
  #
  def SimpleUnsubscribe(self): 
     self.running = False
     try:
       self.subscribe.kill()    # might not have ever subscribed
     except Exception as eobj:
       pass   
  # end SimpleUnsubscribe
  #----------------------------------------------------------- 
  #   
  def SimpleClose(self):
    self.SimpleUnsubscribe()           
    #self.channel.close()     # sometimes hangs at sock.recv
    self.connection.close()   # connection close kills socket
  # end SimpleClose
  #----------------------------------------------------------- 
  #   
  def CloseChannel(self, chobj):
    #self.Unsubscribe(chobj)           
    self.chobj.close()   # connection close kills socket
  # end Close
  #----------------------------------------------------------- 
  #   
  def CloseAll(self):
    ii =0
    while (ii < len(self.chanList)):          
      self.Close(self.chanList[ii][1])   # connection close kills socket
      ii = ii + 1
    # end while
    self.connection.close()