def read_controller_socket(data, conn_type="TCP", frequency=20, port=8080): if conn_type=="UDP": raise ValueError("do not use UDP sockets") elif conn_type=="TCP": s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("192.168.22.207", port)) while True: start = time.time() pkt = s.recv(128) parse_packet(pkt, data) data["timestamp"] = time.time() while time.time()<(start+1.0/frequency): pass
def start(self): while True: # receive arp requests packet, addr = self.sock.recvfrom(65565) eth_frame, arp_packet = parse_packet(packet) arp_type = struct.unpack("!h", arp_packet["oper"])[0] logger.debug("Received ARP-" + ("REQUEST" if (arp_type == 1) else "REPLY") + " SRC: " + eth_frame["src_mac"] + " / " + arp_packet["src_ip"] + " " + "DST: " + eth_frame["dst_mac"] + " / " + arp_packet["dst_ip"]) if arp_type == 1: # check if the arp request stems from one of the participants requester_srcmac = eth_frame["src_mac"] requested_ip = arp_packet["dst_ip"] # Send the ARP request message to respective controller and forget about it if IPAddress(requested_ip) in config.vnhs: self.send_arp_request(requester_srcmac, requested_ip) # TODO: If the requested IP address belongs to a non-SDN participant # then refer the structure `self.nonSDN_nhip_2_nhmac` and # send an immediate ARP response. """
def start(self): while True: # receive arp requests packet, addr = self.sock.recvfrom(65565) eth_frame, arp_packet = parse_packet(packet) arp_type = struct.unpack("!h", arp_packet["oper"])[0] logger.debug( "Received ARP-" + ("REQUEST" if (arp_type == 1) else "REPLY") + " SRC: " + eth_frame["src_mac"] + " / " + arp_packet["src_ip"] + " " + "DST: " + eth_frame["dst_mac"] + " / " + arp_packet["dst_ip"] ) if arp_type == 1: # check if the arp request stems from one of the participants requester_srcmac = eth_frame["src_mac"] requested_ip = arp_packet["dst_ip"] # Send the ARP request message to respective controller and forget about it if IPAddress(requested_ip) in config.vnhs: self.send_arp_request(requester_srcmac, requested_ip) # TODO: If the requested IP address belongs to a non-SDN participant # then refer the structure `self.nonSDN_nhip_2_nhmac` and # send an immediate ARP response. """
def _handle_connection(self, client, address): """ Handle the incoming connection :param client: the worker or client wanting to talk to us. :type client: socket.socket :type address: List[str] :param address: possibly the INET address? It's returned from sock.accept() :return: True if we close OK, False if we don't. I don't think this matters. """ size = 1024 while True: try: packet = utils.parse_packet(client.recv(size)) if packet is not None: if "execmd" in packet: self._handle_cmd(client, address, packet["raw_packet"]) elif "worker" in packet: self._handle_worker_req(client, address, packet["raw_packet"]) elif "client" in packet: if "worker_state" in packet: # the client thinks the worker is bad, let's see if they are a valid client. if packet['worker_state'] == "error": req = self._find_req_by_uuid(packet["uuid"]) # valid request if req is not None: worker = self._find_worker_by_uuid( req.get_worker_uuid()) if worker is not None: worker.set_status("error") else: utils.print_err( "Error: client reported error on unknown worker." ) else: utils.print_err( "Error: unknown request attempted to report worker error" ) self._handle_client_req(client, address, packet["raw_packet"]) elif "verify" in packet: req_uuid = packet["uuid"] client_req = self._find_req_by_uuid( req_uuid) # type: MasterServerClientRequest if client_req is None: client.send("err: unable to find request") else: client.send(client_req.serialize() + "uuid=" + req_uuid) else: client.send("Error: Unrecognized command.\n") else: client.close() return False except socket.timeout: # print e client.close() return False
def _register_with_master(self): """ register ourself with the master server. :return: """ # set this function up as a re-occuring timer based on the -b/--heartbeat option. # utils.print_err("HB: interval " + str(self.__hb_timer_interval) + " at " + str(time())) if self.__master_connection is None: print "Setting up new master connection." self.__master_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) master_address = (self.__config.get_conf_val("ms"), 4017) # master should reply in 30 seconds or so... self.__master_connection.settimeout(60) try: self.__master_connection.connect(master_address) except Exception as e: self.__master_connection.close() self.__master_connection = None utils.print_err( "Error: Problem communicating with master server. Will Retry" ) if self.__master_connection is not None: try: my_hostname = socket.gethostname() self.__master_connection.sendall( "worker name=" + my_hostname + " worker_uuid=" + self.__my_uuid + " slots=" + str(self.__config.get_conf_val("slots")) + " status=" + self.__state) packet = utils.parse_packet( self.__master_connection.recv(1024)) if "ok" not in packet: utils.print_err( "Error: Master Server responded poorly to our register request. Will retry." ) self.__master_connection.close() self.__master_connection = None except Exception as e: if self.__master_connection is not None: self.__master_connection.close() self.__master_connection = None utils.print_err( "Error: Problem communicating with master server. Will Retry" ) # set this function up as a re-occuring timer based on the -b/--heartbeat option. # utils.print_err("HB: interval " + str(self.__hb_timer_interval) + " at " + str(time())) self.__hb_timer = threading.Timer(self.__hb_timer_interval, self._register_with_master) self.__hb_timer.start()
def _handle_cmd(self, connection, address, data): """ sub function to handle execmd commands from a direct connection to the socket. :param connection: :param address: :param data: incoming command string. :type connection: socket.socket :return: """ # security check, only localhost connections. if address[0] != "127.0.0.1": connection.send("Permission Denied.\n\n") connection.close() return packet = utils.parse_packet(data) if "list" in packet: if "workers" in packet: connection.send("Currently registered workers:\n") for worker in self.workers: # type: MasterServerWorkerEntry connection.send( "\t" + worker.get_name() + " " + worker.get_uuid() + " " + worker.get_ip() + " " + datetime.fromtimestamp(worker.get_last_sync( )).strftime('%Y-%m-%d %H:%M:%S') + " " + str(worker.get_slots_total()) + " " + str(worker.get_slots_in_use()) + " " + worker.get_status() + "\n") elif "clients" in packet: connection.send("Current client requests:\n") for m_client in self.client_requests: # type: MasterServerClientRequest connection.send("\t" + m_client.get_ip() + " " + m_client.get_client_uuid() + " " + m_client.get_req_mod() + " " + str(m_client.get_start_time()) + " " + m_client.get_uuid() + "\n") else: connection.send("Error: Unrecognized command.\n") elif "purge" in packet: # issue a request to purge any stale worker nodes. # worker_purge calls the function to also purge stale clients. self._do_stale_purge() connection.send("ok\n") elif "sync" in packet: # issue a request to sync all worker nodes. print "Full sync requested. Invalidating all workers." self._do_worker_syncs() connection.send("ok\n") else: connection.send("Error: Unrecognized command.\n")
def start_arp_listener(self): while self.run: # receive arp requests try: packet, addr = self.raw_socket.recvfrom(65565) eth_frame, arp_packet = parse_packet(packet) arp_type = struct.unpack("!h", arp_packet["oper"])[0] logger.debug( "Received ARP-" + ("REQUEST" if (arp_type == 1) else "REPLY") + " SRC: " + eth_frame["src_mac"] + " / " + arp_packet["src_ip"] + " " + "DST: " + eth_frame["dst_mac"] + " / " + arp_packet["dst_ip"] ) if arp_type == 1: # check if the arp request stems from one of the participants requester_srcmac = eth_frame["src_mac"] requested_ip = arp_packet["dst_ip"] # Send the ARP request message to respective controller and forget about it if IPAddress(requested_ip) in self.vnhs: self.send_arp_request(requester_srcmac, requested_ip) # TODO: If the requested IP address belongs to a non-SDN participant # then refer the structure `self.nonSDN_nhip_2_nhmac` and # send an immediate ARP response. """ response_vmac = self.get_vmac_default(requester_srcmac, requested_ip) if response_vmac != "": logger.debug("ARP-PROXY: reply with VMAC "+response_vmac) data = self.craft_arp_packet(arp_packet, response_vmac) eth_packet = self.craft_eth_frame(eth_frame, response_vmac, data) self.raw_socket.send(''.join(eth_packet)) """ except socket.timeout: # prints about once per second # logger.debug('Socket Timeout Occured') pass
def __init__(self, info, master_lock): self.__master_lock = master_lock packet = utils.parse_packet(info) if packet is None: return None self.__master_lock.acquire() self.__name = packet["name"] self.__UUID = packet["worker_uuid"] self.__slots_total = packet["slots"] self.__ip = packet["ip"] if "status" in packet: self.__status = packet["status"] else: self.__status = "outdated" self.__master_lock.release()
def __init__(self, data): """ :param data: :type data: str """ packet = utils.parse_packet(data) self.__client_UUID = packet["client_uuid"] self.__req_mod = packet["image"] self.__ip = packet["ip"] self.__done = packet["state"] if "uuid" in packet: self.__UUID = packet["uuid"] else: #print "New Client Request from: " + self.__ip self.__UUID = str(uuid.uuid4()) self.__start_time = time()
def start_arp_listener(self): while self.run: # receive arp requests try: packet, addr = self.raw_socket.recvfrom(65565) eth_frame, arp_packet = parse_packet(packet) arp_type = struct.unpack("!h", arp_packet["oper"])[0] logger.debug("Received ARP-" + ("REQUEST" if (arp_type == 1) else "REPLY") + " SRC: " + eth_frame["src_mac"] + " / " + arp_packet["src_ip"] + " " + "DST: " + eth_frame["dst_mac"] + " / " + arp_packet["dst_ip"]) if arp_type == 1: # check if the arp request stems from one of the participants requester_srcmac = eth_frame["src_mac"] requested_ip = arp_packet["dst_ip"] # Send the ARP request message to respective controller and forget about it if IPAddress(requested_ip) in self.vnhs: self.send_arp_request(requester_srcmac, requested_ip) # TODO: If the requested IP address belongs to a non-SDN participant # then refer the structure `self.nonSDN_nhip_2_nhmac` and # send an immediate ARP response. """ response_vmac = self.get_vmac_default(requester_srcmac, requested_ip) if response_vmac != "": logger.debug("ARP-PROXY: reply with VMAC "+response_vmac) data = self.craft_arp_packet(arp_packet, response_vmac) eth_packet = self.craft_eth_frame(eth_frame, response_vmac, data) self.raw_socket.send(''.join(eth_packet)) """ except socket.timeout: # prints about once per second #logger.debug('Socket Timeout Occured') pass
def _handle_client_control(self, connection, req_uuid): # let's see if we have a client request for this uuid cli_req = self._find_request_by_uuid( req_uuid) # type: WorkerServerSync if cli_req is None: connection.sendall("err") return False # start the control link by sending "ok" connection.sendall("ok") # we have a valid request UUID, so let's check it's status while cli_req.is_sync_active(): try: packet = utils.parse_packet(connection.recv(10)) if "ok" not in packet: # something went wrong on teh client side. # Clean up the request and remove it. connection.sendall("err") self._cleanup_client_req(req_uuid) return False connection.sendall("ok") except Exception as e: connection.sendall("err") self._cleanup_client_req(req_uuid) return False # a small sleep here. sleep(1) # if we exit the loop, it's time to turn things down. connection.sendall("close") connection.close() # Clean up the request and remove it. self._cleanup_client_req(req_uuid) return False
def _sync_worker(self, worker): """ :param worker: worker to sync :type worker: MasterServerWorkerEntry :return: True | False :rtype: int """ force_master_sync = False if worker.get_status() == "syncing": return # attempt to sync to a worker. As a last result, try to sync from master. while not force_master_sync: self.__master_data_lock.acquire() worker.set_status("waiting") self.__master_data_lock.release() # only try to send us to a worker if we aren't being forced to a master sync # if we are not forcing a master sync, # attempt to hand off the client to a worker worker_sync = self._find_least_updated_worker() if worker_sync is not None: # don't allow a worker to sync to itself. if worker_sync.get_uuid() == worker.get_uuid(): worker_sync = None self.__master_data_lock.acquire() worker_sync.set_slots_in_use( worker_sync.get_slots_in_use() - 1) # if we are alone, no one is syncing to the master, let us do it. if self.__sync_slots_used < self.__sync_slots: self.__sync_slots_used = self.__sync_slots_used + 1 force_master_sync = True self.__master_data_lock.release() else: # we have a valid, updated worker, try a sync self.__master_data_lock.acquire() worker.set_sync_src_uuid(worker_sync.get_uuid()) self.__master_data_lock.release() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: # get the ip of the worker we are going to sync TO worker_address = (worker.get_ip(), 4018) # connect to the recipient worker and tell it to sync FROM worker_sync sock.connect(worker_address) # increment the worker's in use counter. self.__master_data_lock.acquire() worker.set_status("syncing") self.__master_data_lock.release() sock.sendall("sync master worker=" + worker_sync.get_ip()) packet = utils.parse_packet( sock.recv(1024)) # type: dict # then decrement it after it's over. self.__master_data_lock.acquire() worker_sync.set_slots_in_use( worker_sync.get_slots_in_use() - 1) self.__master_data_lock.release() if packet is None: #utils.print_err("Error: empty packet from " + worker.get_ip()) self.__master_data_lock.acquire() worker.set_status("waiting") self.__master_data_lock.release() else: if "ok" not in packet: self.__master_data_lock.acquire() worker.set_status("waiting") self.__master_data_lock.release() else: # status is updated from an updated worker, set to upadted self.__master_data_lock.acquire() worker.set_status("updated") worker.set_last_sync(time()) self.__master_data_lock.release() return True except Exception as e: self.__master_data_lock.acquire() worker.set_status("error") self.__master_data_lock.release() utils.print_err("Error: worker to worker sync failed") utils.print_err(e) # then decrement it after it's over. self.__master_data_lock.acquire() worker_sync.set_slots_in_use( worker_sync.get_slots_in_use() - 1) self.__master_data_lock.release() sock.close() else: # no updated nodes, if the master is available, let's try that. self.__master_data_lock.acquire() if self.__sync_slots_used < self.__sync_slots: self.__sync_slots_used = self.__sync_slots_used + 1 force_master_sync = True self.__master_data_lock.release() # otherwise, wait a bit and try again. sleep(5) # no worker found or master sync forced, so proceed with # normal master sync. # connect to the worker and wait for it to reply that it's ready. # create a new socket for the worker connection. worker.set_sync_src_uuid(self.__UUID) self.__master_data_lock.acquire() worker.set_status("syncing") self.__master_data_lock.release() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) worker_address = (worker.get_ip(), 4018) sock.settimeout(30) try: sock.connect(worker_address) sock.sendall("sync master repo_uuid=" + self.__UUID) packet = utils.parse_packet(sock.recv(1024)) # type: dict if packet is None: self.__sync_slots_used = self.__sync_slots_used - 1 utils.print_err("Error: empty packet from " + worker.get_ip()) self.__master_data_lock.acquire() worker.set_status("outdated") self.__master_data_lock.release() sock.close() return False if "ok" not in packet: self.__sync_slots_used = self.__sync_slots_used - 1 utils.print_err("Error: Received bad packet from " + worker.get_ip()) utils.print_err("Error: Packet: " + packet["raw_packet"]) self.__master_data_lock.acquire() worker.set_status("outdated") self.__master_data_lock.release() sock.close() return False except Exception as e: self.__sync_slots_used = self.__sync_slots_used - 1 utils.print_err("Error: Unable to talk to worker at " + worker_address[0] + ": " + str(e)) sock.close() self.__master_data_lock.acquire() worker.set_status("error") self.__master_data_lock.release() return False # get the worker's parameters worker_uuid = packet["worker_uuid"] port = packet["port"] rsync_mod = packet["module"] # worker_uuid, port, rsync_mod = data[3:].split() # double check the worker's supplied params. if worker_uuid == "": self.__sync_slots_used = self.__sync_slots_used - 1 utils.print_err("Error: Received bad packet from " + worker_sync.get_ip()) utils.print_err("Error: Packet: " + packet["raw_packet"]) self.__master_data_lock.acquire() worker.set_status("error") self.__master_data_lock.release() sock.close() return False if port == "": self.__sync_slots_used = self.__sync_slots_used - 1 utils.print_err("Error: Received bad packet from " + worker_sync.get_ip()) utils.print_err("Error: Packet: " + packet["raw_packet"]) self.__master_data_lock.acquire() worker.set_status("error") self.__master_data_lock.release() sock.close() return False if rsync_mod == "": self.__master_data_lock.acquire() utils.print_err("Error: Received bad packet from " + worker_sync.get_ip()) utils.print_err("Error: Packet: " + packet["raw_packet"]) worker.set_status("error") self.__master_data_lock.release() sock.close() return False sock.close() # if the worker is ready, start the sync # create a temporary file with the password fd, file_path = mkstemp() pass_file = open(file_path, "w") pass_file.write(worker_uuid) pass_file.close() os.close(fd) if not os.path.isdir("/tmp/mprov/"): # mprov tmp dir doesn't exist. os.mkdir("/tmp/mprov", 0700) # open an rsync log for logging. rsync_log = open("/tmp/mprov/master_sync_" + rsync_mod + ".log", "w+") rsync_args = [ "/usr/bin/rsync", "-avx", "--delete", "--delete-after", "--progress", "--exclude=/tmp", "--exclude=/proc", "--exclude=/sys", "--exclude=/run", "--exclude=/dev", "--password-file=" + file_path, "--port=" + port, self.__path + "/", "root@" + worker.get_ip() + "::" + rsync_mod ] #print "Run: " + " ".join(rsync_args) rsync_log.write("Run: " + " ".join(rsync_args) + "\n") # wait a couple of seconds for the worker to set up the sync sleep(5) #print("Run Rsync") rsync_proc = subprocess.Popen(rsync_args, stdout=rsync_log, stderr=rsync_log) #print("Call Communicate") # wait for the rsync to finish. rsync_proc.communicate() #print("Grab Return Code") return_code = rsync_proc.returncode rsync_log.close() result_pass = False # now, let's check the return code of the rsync process and see if it was an error. if return_code != 0 and return_code != 24: utils.print_err("Error: rsync returned '" + str(return_code) + "'") utils.print_err( "Error: marking worker as status 'error' for worker: " + worker.get_ip()) self.__master_data_lock.acquire() worker.set_status("error") self.__master_data_lock.release() self.__sync_slots_used = self.__sync_slots_used - 1 else: self.__master_data_lock.acquire() worker.set_status("updated") self.__sync_slots_used = self.__sync_slots_used - 1 if self.__sync_slots_used < 0: self.__sync_slots_used = 0 worker.set_last_sync(time()) self.__master_data_lock.release() os.remove(file_path) result_pass = True # connect to the worker and tell it to close up the rsync channel. # create a new socket for the worker connection. sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) worker_address = (worker.get_ip(), 4018) # worker should reply in 30 seconds or so... sock2.settimeout(30) try: sock2.connect(worker_address) # send it the signal to shut-down. sock2.sendall("stop master repo_uuid=" + self.__UUID) except Exception as e: utils.print_err(str(e)) self.__master_data_lock.acquire() self.__sync_slots_used = self.__sync_slots_used - 1 worker.set_status("error") self.__master_data_lock.release() sock2.close() return False sock2.close() # once the sync is done, exit our self. return result_pass
def _handle_worker_sync(self, connection, address): """ handle a new worker request for a sync. :param connection: :param address: :return: """ print "Worker to worker sync requested from: " + address[0] # tell the other end to proceed. connection.sendall("ok repo_uuid=" + self.__my_uuid) # now we wait for the worker to tell us it's ready. connection.settimeout(60) packet_worker = None # type: dict try: packet_worker = utils.parse_packet(connection.recv( 1024)) # this will block until the worker is ready. except Exception as e: utils.print_err( "Error: Client didn't reply back. Stopping request.") utils.print_err("Error: Exception: " + e.message) connection.close() return if packet_worker is None: utils.print_err( "Error: Client didn't reply back. Stopping request.") connection.close() return if "ok" not in packet_worker: utils.print_err("Error: Client wasn't ready.") utils.log("Error: Client wasn't ready.", self.__config.get_conf_val("ms"), 4017) connection.close() return # handle incoming worker sync request. # check for available slots. if self.__slots_in_use > 0: if self.__slots_in_use >= self.__worker_slots: utils.print_err( "Error: Worker Full, but client asked for sync. Shouldn't happen." ) # no slots available. Disconnect connection.close() return # parse the "ok" packet rsync_uuid = packet_worker["worker_uuid"] sync_port = packet_worker["port"] mod_name = packet_worker["module"] # and set up the rsync. # if the client is ready, start the sync # create a temporary file with the password fd, file_path = mkstemp() pass_file = open(file_path, "w") pass_file.write(rsync_uuid) pass_file.close() os.close(fd) rsync_args = [ "/usr/bin/rsync", "-avx", "--delete", "--delete-after", "--progress", "--exclude=/tmp", "--exclude=/proc", "--exclude=/sys", "--exclude=/run", "--exclude=/dev", "--port=" + sync_port, "--password-file=" + file_path, self.__path + "/", "root@" + address[0] + "::" + mod_name ] print " ".join(rsync_args) # logging of the rsync. if not os.path.isdir("/tmp/mprov/"): # mprov tmp dir doesn't exist. os.mkdir("/tmp/mprov", 0700) # open an rsync log for logging. rsync_log = open("/tmp/mprov/worker_sync_" + mod_name + ".log", "w+") # we are about to run but give the worker a couple of seconds to make sure the # rsync is set up on their end sleep(10) rsync_proc = subprocess.Popen(rsync_args, stdout=rsync_log, stderr=rsync_log) # wait for the rsync to finish. rsync_proc.communicate() rsync_log.close() # examine return code and log.... something... return_code = rsync_proc.returncode if return_code != 0 and return_code != 24: utils.print_err("Error: Worker rsync to " + address[0] + " died prematurely! RC=" + str(return_code)) else: os.remove(file_path) # connect to the worker and tell it to close up the rsync channel. # create a new socket for the worker connection. sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM) worker_address = (address[0], 4018) # worker should reply in 30 seconds or so... sock2.settimeout(30) try: sock2.connect(worker_address) # send it the signal to shut-down. sock2.sendall("stop master repo_uuid=" + self.__my_uuid) except Exception as e: print e sock2.close() return False sock2.close() connection.close() return
def _handle_client_sync(self, connection, address, req_uuid): """ handle a new client request for a sync. :param connection: :param address: :param req_uuid: :return: """ # first check with the master for the information about the client. # create a new socket for the master connection. sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) master_address = (self.__config.get_conf_val("ms"), 4017) sock.settimeout(30) packet = None # type: dict try: sock.connect(master_address) sock.sendall("verify uuid=" + req_uuid) packet = utils.parse_packet(sock.recv(1024)) except Exception as e: utils.print_err("Error: Network issue communicating to master.") utils.print_err("Error: Exception: " + str(e)) sock.close() if packet is None: utils.print_err("Error: Master didn't verify sync request.") self._cleanup_client_req(req_uuid) return if packet["raw_packet"] == "" or "err" in packet: utils.print_err("Error: Master didn't verify sync request.") self._cleanup_client_req(req_uuid) return print packet["raw_packet"] cli_uuid = packet["client_uuid"] req_mod = packet["image"] cli_ip = packet["ip"] req_uuid = packet["uuid"] print( "Master verified the request. Telling the client and waiting until it's ready." ) # make an WorkerServerSync object and stick it in the list. cli_req = WorkerServerSync(req_uuid, cli_uuid, address[0], req_mod, True) self.__sync_requests.append(cli_req) # now we have a valid client object, so tell the client to proceed. connection.sendall("ok") # now we wait for the client to tell us it's ready. connection.settimeout(60) packet_client = None # type: dict try: packet_client = utils.parse_packet(connection.recv( 1024)) # this will block until the client is ready. except Exception as e: utils.print_err( "Error: Client didn't reply back. Stopping request.") utils.print_err("Error: Exception: " + e.message) connection.close() cli_req.set_sync_active(False) return if packet_client is None: utils.print_err( "Error: Client didn't reply back. Stopping request.") connection.close() cli_req.set_sync_active(False) return if "ok" not in packet_client: #utils.print_err("Error: Client wasn't ready.") utils.log("Error: Client wasn't ready.", self.__config.get_conf_val("ms"), 4017) connection.close() cli_req.set_sync_active(False) return # handle incoming client sync request. # check for available slots. if self.__slots_in_use > 0: if self.__slots_in_use >= self.__worker_slots: utils.print_err( "Error: Worker Full, but client asked for sync. Shouldn't happen." ) # no slots available. Disconnect cli_req.set_sync_active(False) connection.close() return # parse the "ok" packet rsync_uuid = packet_client["client_uuid"] sync_port = packet_client["port"] mod_name = packet_client["module"] if rsync_uuid != cli_req.get_client_uuid(): utils.print_err("Error: request ID mismatch!") cli_req.set_sync_active(False) connection.sendall("err") connection.close() return # and set up the rsync. # if the client is ready, start the sync # create a temporary file with the password fd, file_path = mkstemp() pass_file = open(file_path, "w") pass_file.write(rsync_uuid) pass_file.close() os.close(fd) rsync_args = [ "/usr/bin/rsync", "-avx", "--progress", "--port=" + sync_port, "--exclude=/tmp", "--exclude=/proc", "--exclude=/sys", "--exclude=/run", "--exclude=/dev", "--password-file=" + file_path, self.__path + "/" + cli_req.get_req_mod() + "/", "root@" + address[0] + "::" + mod_name ] print " ".join(rsync_args) # logging of the rsync. if not os.path.isdir("/tmp/mprov/"): # mprov tmp dir doesn't exist. os.mkdir("/tmp/mprov", 0700) # open an rsync log for logging. rsync_log = open("/tmp/mprov/client_sync_" + mod_name + ".log", "w+") # we are about to run but give the client a couple of seconds to make sure the # rsync is set up on their end sleep(5) rsync_proc = subprocess.Popen(rsync_args, stdout=rsync_log, stderr=rsync_log) # wait for the rsync to finish. rsync_proc.communicate() rsync_log.close() # examine return code and log.... something... return_code = rsync_proc.returncode if return_code != 0 and return_code != 24: utils.print_err("Error: Client rsync died prematurely! RC=" + str(return_code)) connection.sendall("ok result=error") else: connection.sendall("ok result=pass") os.remove(file_path) # mark the sync as done. cli_req.set_sync_active(False) connection.close() return
def _handle_master_sync(self, connection, packet): """ handle a sync request from the master server. :param connection: the socket connection to communicate on :param packet: the incoming packet :type connection: socket.socket :type packet: dict :return: """ # let the master server redirect us to another worker that has an updated # copy of the image repo print "Repository Sync" self.__state = "syncing" sync_connection = None if "repo_uuid" in packet: self.__repo_sync_uuid = packet["repo_uuid"] # if the master server is sending us a worker to sync from, then connect to the worker. if "worker" in packet: worker_address = packet["worker"] #connect to the worker sync_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) worker_address = (worker_address, 4018) try: sync_connection.connect(worker_address) sync_connection.sendall("sync worker") packet_reply = utils.parse_packet(sync_connection.recv(1024)) if "retry" in packet_reply: print("Worker not redy, will retry") sync_connection.close() connection.close() return if "ok" not in packet_reply: utils.print_err( "Error in worker to worker sync. bad reply from: " + worker_address[0]) utils.print_err("Error Packet: " + packet_reply["raw_packet"]) sync_connection.close() connection.close() return if "repo_uuid" in packet_reply: self.__repo_sync_uuid = packet_reply["repo_uuid"] except Exception as e: utils.print_err("Error in worker to worker sync from: " + worker_address[0]) sync_connection.close() connection.close() return else: sync_connection = connection self.__last_master_sync_ip = sync_connection.getpeername()[0] # generate the module_name module_name = str(uuid.uuid4()) sync_port = "8970" # TODO: make this dynamic, and random. # output the rsyncd config and a secret file rsyncd_fd, rsyncd_path = mkstemp() secrets_fd, secrets_path = mkstemp() rsyncd_file = open(rsyncd_path, "w") file_contents = "[" + module_name + "]\n" + \ "\tpath = " + self.__path + "\n" + \ "\tread only = no\n" + \ "\tauth users = root\n" + \ "\tsecrets file = " + secrets_path + "\n" + \ "\tuid = 0\n" + \ "\tgid = 0\n" + \ "\tpost-xfer exec = /usr/bin/pkill rsync\n" rsyncd_file.write(file_contents) rsyncd_file.close() os.close(rsyncd_fd) # output the rsyncd secrets file. rsyncd_file = open(secrets_path, "w") rsyncd_file.write("root:" + self.__my_uuid) rsyncd_file.close() os.close(secrets_fd) # before we run, kill off any other rsync daemons running rsyncd_running = subprocess.Popen( ["/usr/bin/pgrep", "-f", "rsync --daemon"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) (pgrep_stdout, pgrep_stderr) = rsyncd_running.communicate() utils.print_err("") if pgrep_stdout != "" and pgrep_stdout is not None: for pid in pgrep_stdout.split("\n"): if pid != "": os.kill(int(pid), 9) # setup the rsyncd command. rsyncd_proc = subprocess.Popen([ "/usr/bin/rsync", "--daemon", "--port=" + sync_port, "--no-detach", "-v", "-4", "--config=" + rsyncd_path ], shell=False) self.__rsyncd_pid = rsyncd_proc.pid sync_connection.sendall("ok worker_uuid=" + self.__my_uuid + " port=" + sync_port + " module=" + module_name) rsyncd_proc.communicate() reply = "ok" self.__state = "updated" # check the return code here! if rsyncd_proc.returncode != 0 and rsyncd_proc.returncode != 24 and rsyncd_proc.returncode != 20: utils.print_err("Error: rsync from " + sync_connection.getpeername()[0] + " died unexpectedly with RC=" + str(rsyncd_proc.returncode) + "!!!") reply = "err" self.__state = "err" # if we used a different connection for the sync, then tell the master we are done. if sync_connection != connection: connection.sendall(reply) connection.close() # clean up the tmp files. os.remove(rsyncd_path) os.remove(secrets_path) sync_connection.close() print "Sync Complete."
def _handle_connection(self, connection, address): """ Handle the incoming connection :param connection: socket to communicate on. :type connection: socket.socket :type address: List[str] :param address: the address that is connected to us. :return: True if we close OK, False if we don't. I don't think this matters. """ size = 1024 while True: try: data = connection.recv(size) packet = utils.parse_packet(data) if packet is not None: if "sync" in packet: if "master" in packet: self.__master_sync_active = True self._handle_master_sync(connection, packet) self.__master_sync_active = False elif "client" in packet: if self.__master_sync_active: connection.sendall("retry") connection.close() return False self._handle_client_sync(connection, address, packet["uuid"]) elif "worker" in packet: if self.__master_sync_active: connection.sendall("retry") connection.close() return False self._handle_worker_sync(connection, address) elif "stop" in packet: if "master" in packet: if "repo_uuid" in packet: if packet[ "repo_uuid"] == self.__repo_sync_uuid: # we are getting a stop from where we expect. self._handle_stop_master_sync( connection, address) return True else: utils.print_err( "Error: stop command unknown repo_uuid: " + packet["repo_uuid"]) else: utils.print_err( "Error: stop command from source with no UUID." ) utils.print_err("Error: should be: " + self.__repo_sync_uuid) utils.print_err("Error: this is probably bad.") connection.close() return False elif "client" in packet: # purge a stale client request, kill any rsyncs running, and open the slot. if address[0] == self.__config.get_conf_val("ms"): self._handle_stop_client_sync( connection, address, packet["uuid"]) connection.close() return False elif "control" in packet: # a control connection request if "client" in packet: # this is a client connection request. Run the handler. self._handle_client_control( connection, packet["uuid"]) else: utils.print_err("Unrecognized Packet: " + data) connection.close() return False # done parsing, close the connection connection.close() return True except Exception as e: utils.print_err("Error: Unhandled exception thrown: " + str(e)) utils.print_err("Error: Packet generating error: " + data) traceback.print_exc(file=sys.stderr) connection.close() return False
def __init__(self, info): packet = utils.parse_packet(info) self.__name = packet["name"] self.__UUID = packet["worker_uuid"] self.__slots_total = packet["slots"] self.__ip = packet["ip"]