def send_and_receive(self):
        in_buffer = self.rfile
        out_buffer = self.wfile
        s = self.server.addtional_obj
        command = CommandProcessor.receive_command(in_buffer)
        #print("process cmd: ", command.cmd)
        if command is not None:
            reply = CommandStruct(cmd=ServerCommand.Com_ReplyOK)
            if command.cmd == ServerCommand.Com_Start:
                #print("start conversation:")
                CommandProcessor.send_command(out_buffer, reply)
            elif command.cmd == ServerCommand.Com_Stop_Mining:
                if s is not None and isinstance(s, SiteCheckProcessManager):
                    s.set_stop()
                CommandProcessor.send_command(out_buffer, reply)
            elif command.cmd == ServerCommand.Com_Setup:  # test this
                data = command.data
                if isinstance(data, SetupData):
                    cap2 = data.cap2
                    if cap2 == 0:
                        cap2 = SlaveAutoScaler(data.max_page_level, data.max_page_limit).get_optimal_capacity()
                    if isinstance(s, SiteCheckProcessManager):  # need to fix this
                        if s.is_alive():
                            s.set_stop()
                            s.join()
                    # elif isinstance(s, threading.Thread):
                    #     s.join(0)
                    print("init new process manager with para: ", data.get_serializable(False))
                    total_memory =MachineInfo.get_memory()[0]
                    mem_limit_per_crawler = int(total_memory * 0.8 / cap2)
                    if mem_limit_per_crawler >=  SiteCheckProcessManager.MEM_MINIMUM_REQ:
                        self.server.addtional_obj = SiteCheckProcessManager(data.ref, max_procss=cap2,
                                                                            concurrent_page=data.cap3,
                                                                            page_max_level=data.max_page_level,
                                                                            max_page_per_site=data.max_page_limit,
                                                                            memory_limit_per_process=mem_limit_per_crawler)
                        self.server.addtional_obj.start()
                    else:
                        reply.cmd = ServerCommand.Com_ReplyError
                        reply.data = "Not enough memory allocation for each crawler, must at least 100 Mb."
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Clear_Cache:
                if isinstance(s, SiteCheckProcessManager):  # need to fix this
                    if s.is_alive():
                        s.set_stop()
                        s.join()
                    s.clear_cache()
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Data:
                print("incoming data....")
                data = command.data
                if s is not None and isinstance(s, SiteCheckProcessManager):
                    if data.data is not None and len(data.data) > 0:
                        s.put_to_input_queue(data.data)
                else:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "something is wrong with data"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Get_Data:  # test this
                if s is not None and isinstance(s, SiteCheckProcessManager):
                    rawdata = []
                    if s.get_temp_result_count() > 0:
                        rawdata += [ScrapeDomainData(x.link, x.response_code) for x in s.get_temp_result_and_clear()
                                    if isinstance(x, OnSiteLink)]
                    #print("sending back:")
                    #print(rawdata)
                    if s.get_site_info_list_count() > 0:
                        rawdata += [info for info in s.get_site_info_list_and_clear()
                                    if isinstance(info, SeedSiteFeedback)]
                    data = MiningList(s.name, rawdata)
                    reply.data = data
                else:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "something is wrong with return data"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Status:
                #print("send back status!")
                CPU = MachineInfo.get_cpu(1)
                MEM = MachineInfo.get_memory()
                NET = MachineInfo.get_network(1)
                if s is not None and isinstance(s, SiteCheckProcessManager):
                    manager_state = s.get_state()
                    filter_progress = s.get_filter_progress()
                    status = ServerStatus(wait_job=manager_state.job_wait, done_job=manager_state.job_done,
                                          all_job=manager_state.job_all, total_page_done=manager_state.total_page_done,
                                          page_per_site=manager_state.average_page,
                                          result=manager_state.result_count, cpu_cores=CPU[0], cpu_percent=CPU[1],
                                          toal_memory=MEM[0], memory_percent=MEM[1], net_recieved=NET[0], net_send=NET[1],
                                          cap_slave=1, cap_process=s.max_prcess, cap_concurrent_page=s.concurrent_page,
                                          filter_done=filter_progress[0], filter_total=filter_progress[1])
                    reply.data = status
                else:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "something is wrong with the crawler."
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Stop:
                #print("end conversation:")
                return
            else:
                reply.cmd = ServerCommand.Com_ReplyError
                reply.data = "command is not valid, please try again"
                CommandProcessor.send_command(out_buffer, reply)
            #print("finished cmd", command.cmd)
            self.send_and_receive()
 def get_optimal_capacity(self) -> int:
     mem_per_process = self.maxpagelimit * 50/1000  # normally it cost 0.005 MB per page
     total_mem = MachineInfo.get_memory()[0]
     return int(total_mem*0.7/mem_per_process)
    def send_and_receive(self):
        in_buffer = self.rfile
        out_buffer = self.wfile
        s = self.server.addtional_obj
        command = CommandProcessor.receive_command(in_buffer)
        #print("process cmd: ", command.cmd)
        if command is not None:
            reply = CommandStruct(cmd=ServerCommand.Com_ReplyOK)
            if command.cmd == ServerCommand.Com_Start:
                #print("start conversation:")
                CommandProcessor.send_command(out_buffer, reply)
            elif command.cmd == ServerCommand.Com_Stop_Mining:
                if s is not None and isinstance(s, SiteCheckProcessManager):
                    s.set_stop()
                CommandProcessor.send_command(out_buffer, reply)
            elif command.cmd == ServerCommand.Com_Setup:  # test this
                data = command.data
                if isinstance(data, SetupData):
                    cap2 = data.cap2
                    if cap2 == 0:
                        cap2 = SlaveAutoScaler(
                            data.max_page_level,
                            data.max_page_limit).get_optimal_capacity()
                    if isinstance(s,
                                  SiteCheckProcessManager):  # need to fix this
                        if s.is_alive():
                            s.set_stop()
                            s.join()
                    # elif isinstance(s, threading.Thread):
                    #     s.join(0)
                    print("init new process manager with para: ",
                          data.get_serializable(False))
                    total_memory = MachineInfo.get_memory()[0]
                    mem_limit_per_crawler = int(total_memory * 0.8 / cap2)
                    if mem_limit_per_crawler >= SiteCheckProcessManager.MEM_MINIMUM_REQ:
                        self.server.addtional_obj = SiteCheckProcessManager(
                            data.ref,
                            max_procss=cap2,
                            concurrent_page=data.cap3,
                            page_max_level=data.max_page_level,
                            max_page_per_site=data.max_page_limit,
                            memory_limit_per_process=mem_limit_per_crawler)
                        self.server.addtional_obj.start()
                    else:
                        reply.cmd = ServerCommand.Com_ReplyError
                        reply.data = "Not enough memory allocation for each crawler, must at least 100 Mb."
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Clear_Cache:
                if isinstance(s, SiteCheckProcessManager):  # need to fix this
                    if s.is_alive():
                        s.set_stop()
                        s.join()
                    s.clear_cache()
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Data:
                print("incoming data....")
                data = command.data
                if s is not None and isinstance(s, SiteCheckProcessManager):
                    if data.data is not None and len(data.data) > 0:
                        s.put_to_input_queue(data.data)
                else:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "something is wrong with data"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Get_Data:  # test this
                if s is not None and isinstance(s, SiteCheckProcessManager):
                    rawdata = []
                    if s.get_temp_result_count() > 0:
                        rawdata += [
                            ScrapeDomainData(x.link, x.response_code)
                            for x in s.get_temp_result_and_clear()
                            if isinstance(x, OnSiteLink)
                        ]
                    #print("sending back:")
                    #print(rawdata)
                    if s.get_site_info_list_count() > 0:
                        rawdata += [
                            info for info in s.get_site_info_list_and_clear()
                            if isinstance(info, SeedSiteFeedback)
                        ]
                    data = MiningList(s.name, rawdata)
                    reply.data = data
                else:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "something is wrong with return data"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Status:
                #print("send back status!")
                CPU = MachineInfo.get_cpu(1)
                MEM = MachineInfo.get_memory()
                NET = MachineInfo.get_network(1)
                if s is not None and isinstance(s, SiteCheckProcessManager):
                    manager_state = s.get_state()
                    filter_progress = s.get_filter_progress()
                    status = ServerStatus(
                        wait_job=manager_state.job_wait,
                        done_job=manager_state.job_done,
                        all_job=manager_state.job_all,
                        total_page_done=manager_state.total_page_done,
                        page_per_site=manager_state.average_page,
                        result=manager_state.result_count,
                        cpu_cores=CPU[0],
                        cpu_percent=CPU[1],
                        toal_memory=MEM[0],
                        memory_percent=MEM[1],
                        net_recieved=NET[0],
                        net_send=NET[1],
                        cap_slave=1,
                        cap_process=s.max_prcess,
                        cap_concurrent_page=s.concurrent_page,
                        filter_done=filter_progress[0],
                        filter_total=filter_progress[1])
                    reply.data = status
                else:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "something is wrong with the crawler."
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Stop:
                #print("end conversation:")
                return
            else:
                reply.cmd = ServerCommand.Com_ReplyError
                reply.data = "command is not valid, please try again"
                CommandProcessor.send_command(out_buffer, reply)
            #print("finished cmd", command.cmd)
            self.send_and_receive()
    def send_and_receive(self):
        in_buffer = self.rfile
        out_buffer = self.wfile
        s = self.server.addtional_obj
        command = CommandProcessor.receive_command(in_buffer)
        #print("process cmd: ", command.cmd)
        if command is not None and isinstance(s, MiningMasterController):
            reply = CommandStruct(cmd=ServerCommand.Com_ReplyOK)
            if command.cmd == ServerCommand.Com_Start:
                #print("start conversation:")
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Stop:
                #print("end conversation:")
                return  # exit point

            elif command.cmd == ServerCommand.Com_Get_DB_DATA:
                data = command.data
                if isinstance(data, DBRequestFields):
                    try:
                        reply.data = s.get_db_results(db_type=data.db_type, db_name=data.db_name, index=data.index, length=data.length)
                    except Exception as ex:
                        ErrorLogger.log_error("MasterRequestHandler.send_and_receive()", ex,
                                              "cmd = ServerCommand.Com-Get-DB-DATA")
                        reply.cmd = ServerCommand.Com_ReplyError
                        reply.data = "Get DB data failed"
                else:
                    ErrorLogger.log_error("MasterRequestHandler.send_and_receive()",
                                          Exception("wrong data type recieved."), "cmd = ServerCommand.Com-Get-DB-DATA")
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Get DB data failed"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Remove_DB:
                data = command.data
                if isinstance(data, DBRequestFields):
                    try:
                        s.remove_db(db_type=data.db_type, db_name=data.db_name)
                    except Exception as ex:
                        ErrorLogger.log_error("MasterRequestHandler.send_and_receive()", ex,
                                              "cmd = ServerCommand.Com_DB-RM-DB")
                        reply.cmd = ServerCommand.Com_ReplyError
                        reply.data = "Remove DB failed"
                else:
                    ErrorLogger.log_error("MasterRequestHandler.send_and_receive()",
                                          Exception("wrong data type recieved."), "cmd = ServerCommand.Com_DB-RM-DB")
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Remove DB failed"
                CommandProcessor.send_command(out_buffer, reply)
            elif command.cmd == ServerCommand.Com_Start_Filter:
                data = command.data
                try:
                    if s.is_alive():
                        s.stop()
                        s.join()
                    if isinstance(data,  FilteringSetupData):  #MatrixFilterControl.FilteringSetupData
                        self.server.addtional_obj = MiningMasterController(ref=data.ref, accounts=data.accounts,
                                                                           crawl_matrix=data.crawl_matrix,
                                                                           filtering_only_mode=True,
                                                                           filtering_offset=data.offset,
                                                                           filtering_total=data.total
                                                                           )
                        self.server.addtional_obj.start()
                except Exception as ex:
                    print(ex)
                    ErrorLogger.log_error("MasterRequestHandler.send_and_receive()", ex, "cmd = ServerCommand.Com_Start_Filter()")
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Com_Start_Filter failed"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Setup:  # test this
                data = command.data
                try:
                    if s.is_alive():
                        s.stop()
                        s.join()
                    if isinstance(data, SetupData):
                        self.server.addtional_obj = MiningMasterController(ref=data.ref, accounts=data.accounts,
                                                                           cap_slave=data.cap,
                                                                           cap_slave_process=data.cap2,
                                                                           cap_concurrent_page=data.cap3,
                                                                           all_job=data.total,
                                                                           offset=data.offset,
                                                                           max_page_level=data.max_page_level,
                                                                           max_page_limit=data.max_page_limit,
                                                                           loopback_database=data.loopback,
                                                                           refresh_rate=data.refresh_rate,
                                                                           filters=data.db_filter,
                                                                           crawl_matrix=data.crawl_matrix,
                                                                           )
                        if data.addtional_data is not None and isinstance(data.addtional_data, SlaveOperationData):
                            self.add_slaves(self.server.addtional_obj, data.addtional_data)
                            self.server.addtional_obj.setup_minging_slaves()
                        self.server.addtional_obj.start()
                    else:
                        raise NotImplementedError("other data type is not implemented.")
                except Exception as ex:
                    print(ex)
                    ErrorLogger.log_error("MasterRequestHandler.send_and_receive()", ex, "cmd = ServerCommand.Com_Setup()")
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Setup failed"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Clear_Cache:
                try:
                    if s.is_alive():
                        s.stop()
                        s.join()
                    s.clear_host_cache()
                    s.clear_slave_cache()
                except Exception as ex:
                    print(ex)
                    ErrorLogger.log_error("MasterRequestHandler.send_and_receive()", ex, "cmd = ServerCommand.Com_Clear_Cache()")
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Setup failed"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Add_Seed:
                data = command.data
                if isinstance(data, MiningList):
                    s.add_seeds(data)
                else:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Add Seed Failed, format is wrong in server handler."
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Add_Slave: # test this
                try:
                    data = command.data
                    if isinstance(data, SlaveOperationData):
                        self.add_slaves(s, data)
                    else:
                        raise NotImplementedError("other data type is not implemented.")
                except Exception as ex:
                    print(ex)
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Add slave failed"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Del_Slave: # test this
                try:
                    data = command.data
                    if isinstance(data, SlaveOperationData):
                        self.remove_slaves(s, data)
                    else:
                        raise NotImplementedError("other data type is not implemented.")
                except Exception as ex:
                    print(ex)
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Add slave failed"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Begin_Mining:  # not implemented, use setup to begin mining
                reply.cmd = ServerCommand.Com_ReplyError
                reply.data = "Add slave failed"

            elif command.cmd == ServerCommand.Com_Stop_Mining:  # test this
                try:
                    EC2 = EC2Controller("")
                    addrs = [slave.address.address for slave in s.slaves if isinstance(slave, Server)]
                    s.pause()
                    #s.slaves.clear()
                    #if s.isAlive:
                    #    s.join(0)
                    #self.server.addtional_obj = MiningMasterController()

                    EC2.shut_down_machines_list(addrs)
                except:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "Stop site failed"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Status:  # test this
                try:
                    CPU = MachineInfo.get_cpu(1)
                    MEM = MachineInfo.get_memory()
                    NET = MachineInfo.get_network(1)
                    slaveStatus = [slave.status for slave in s.slaves]
                    totalPage = sum([slave.total_page_done for slave in slaveStatus])
                    ave_page = 0
                    filter_progress = s.get_filter_progress()
                    if len(s.slaves) > 0:
                        ave_page = int(sum([slave.page_per_site for slave in slaveStatus])/len(s.slaves))
                    total_result = sum([slave.result for slave in slaveStatus])
                    total_cap_slave = sum([slave.cap_slave for slave in slaveStatus])
                    total_cap_process = sum([slave.cap_slave * slave.cap_process for slave in slaveStatus])
                    total_cap_page = sum([slave.cap_slave * slave.cap_process * slave.cap_concurrent_page for slave in slaveStatus])
                    status = ServerStatus(wait_job=s.job_all - s.job_done, done_job=s.job_done, all_job=s.job_all,
                                          total_page_done=totalPage, page_per_site=ave_page,
                                          result=total_result, cpu_cores=CPU[0], cpu_percent=CPU[1],
                                          toal_memory=MEM[0], memory_percent=MEM[1], net_recieved=NET[0], net_send=NET[1],
                                          cap_slave=total_cap_slave, cap_process= total_cap_process, cap_concurrent_page= total_cap_page,
                                          filter_done=filter_progress[0], filter_total=filter_progress[1])
                    server = Server(server_type=ServerType.ty_Host, status=status, address=ServerAddress("localhost", MiningTCPServer.DefaultListenPort))
                    servers = []
                    servers.append(server)
                    servers += s.slaves
                    reply.data = MiningList(s.ref, servers)
                except:
                    reply.cmd = ServerCommand.Com_ReplyError
                    reply.data = "getting status failed"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_DataBase_Status:  # test this
                reply.data = s.get_db_stats()  # send back a copy
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Set_DB_Filter:
                data = command.data
                if isinstance(data, DBFilterCollection):
                    if data != s.filter_shadow:
                        s.filter_shadow = data
                        s.update_db_stats(True)
                else:
                    reply.cmd =ServerCommand.Com_ReplyError
                    reply.data = "wrong data type for filters, should be DBFilterCollection"
                CommandProcessor.send_command(out_buffer, reply)

            elif command.cmd == ServerCommand.Com_Progress: # this this
                reply.data = PrograssData(ref=s.ref, done=s.job_done, all_job=s.job_all,offset=s.offset,
                                          duration=s.end_time - s.start_time, in_progress=s.in_progress)
                CommandProcessor.send_command(out_buffer, reply)

            else:
                reply.cmd = ServerCommand.Com_ReplyError
                reply.data = "command is not valid, please try again"
                CommandProcessor.send_command(out_buffer, reply)

            #CommandProcessor.send_command(out_buffer, reply)
            #print("finished cmd ", command.cmd)
            self.send_and_receive()  # recursive to make a conversation
 def get_optimal_capacity(self) -> int:
     mem_per_process = self.maxpagelimit * 50 / 1000  # normally it cost 0.005 MB per page
     total_mem = MachineInfo.get_memory()[0]
     return int(total_mem * 0.7 / mem_per_process)