def read_controller_socket(data, conn_type="TCP", frequency=20, port=8080):
if conn_type=="UDP":
raise ValueError("do not use UDP sockets")
elif conn_type=="TCP":
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("192.168.22.207", port))
while True:
start = time.time()
pkt = s.recv(128)
parse_packet(pkt, data)
data["timestamp"] = time.time()
while time.time()<(start+1.0/frequency):
pass
def start(self):
while True:
# receive arp requests
packet, addr = self.sock.recvfrom(65565)
eth_frame, arp_packet = parse_packet(packet)
arp_type = struct.unpack("!h", arp_packet["oper"])[0]
logger.debug("Received ARP-" +
("REQUEST" if (arp_type == 1) else "REPLY") +
" SRC: " + eth_frame["src_mac"] + " / " +
arp_packet["src_ip"] + " " + "DST: " +
eth_frame["dst_mac"] + " / " + arp_packet["dst_ip"])
if arp_type == 1:
# check if the arp request stems from one of the participants
requester_srcmac = eth_frame["src_mac"]
requested_ip = arp_packet["dst_ip"]
# Send the ARP request message to respective controller and forget about it
if IPAddress(requested_ip) in config.vnhs:
self.send_arp_request(requester_srcmac, requested_ip)
# TODO: If the requested IP address belongs to a non-SDN participant
# then refer the structure `self.nonSDN_nhip_2_nhmac` and
# send an immediate ARP response.
"""
def start(self):
while True:
# receive arp requests
packet, addr = self.sock.recvfrom(65565)
eth_frame, arp_packet = parse_packet(packet)
arp_type = struct.unpack("!h", arp_packet["oper"])[0]
logger.debug(
"Received ARP-"
+ ("REQUEST" if (arp_type == 1) else "REPLY")
+ " SRC: "
+ eth_frame["src_mac"]
+ " / "
+ arp_packet["src_ip"]
+ " "
+ "DST: "
+ eth_frame["dst_mac"]
+ " / "
+ arp_packet["dst_ip"]
)
if arp_type == 1:
# check if the arp request stems from one of the participants
requester_srcmac = eth_frame["src_mac"]
requested_ip = arp_packet["dst_ip"]
# Send the ARP request message to respective controller and forget about it
if IPAddress(requested_ip) in config.vnhs:
self.send_arp_request(requester_srcmac, requested_ip)
# TODO: If the requested IP address belongs to a non-SDN participant
# then refer the structure `self.nonSDN_nhip_2_nhmac` and
# send an immediate ARP response.
"""
def _handle_connection(self, client, address):
"""
Handle the incoming connection
:param client: the worker or client wanting to talk to us.
:type client: socket.socket
:type address: List[str]
:param address: possibly the INET address? It's returned from sock.accept()
:return: True if we close OK, False if we don't. I don't think this matters.
"""
size = 1024
while True:
try:
packet = utils.parse_packet(client.recv(size))
if packet is not None:
if "execmd" in packet:
self._handle_cmd(client, address, packet["raw_packet"])
elif "worker" in packet:
self._handle_worker_req(client, address,
packet["raw_packet"])
elif "client" in packet:
if "worker_state" in packet:
# the client thinks the worker is bad, let's see if they are a valid client.
if packet['worker_state'] == "error":
req = self._find_req_by_uuid(packet["uuid"])
# valid request
if req is not None:
worker = self._find_worker_by_uuid(
req.get_worker_uuid())
if worker is not None:
worker.set_status("error")
else:
utils.print_err(
"Error: client reported error on unknown worker."
)
else:
utils.print_err(
"Error: unknown request attempted to report worker error"
)
self._handle_client_req(client, address,
packet["raw_packet"])
elif "verify" in packet:
req_uuid = packet["uuid"]
client_req = self._find_req_by_uuid(
req_uuid) # type: MasterServerClientRequest
if client_req is None:
client.send("err: unable to find request")
else:
client.send(client_req.serialize() + "uuid=" +
req_uuid)
else:
client.send("Error: Unrecognized command.\n")
else:
client.close()
return False
except socket.timeout:
# print e
client.close()
return False
def _register_with_master(self):
"""
register ourself with the master server.
:return:
"""
# set this function up as a re-occuring timer based on the -b/--heartbeat option.
# utils.print_err("HB: interval " + str(self.__hb_timer_interval) + " at " + str(time()))
if self.__master_connection is None:
print "Setting up new master connection."
self.__master_connection = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
master_address = (self.__config.get_conf_val("ms"), 4017)
# master should reply in 30 seconds or so...
self.__master_connection.settimeout(60)
try:
self.__master_connection.connect(master_address)
except Exception as e:
self.__master_connection.close()
self.__master_connection = None
utils.print_err(
"Error: Problem communicating with master server. Will Retry"
)
if self.__master_connection is not None:
try:
my_hostname = socket.gethostname()
self.__master_connection.sendall(
"worker name=" + my_hostname + " worker_uuid=" +
self.__my_uuid + " slots=" +
str(self.__config.get_conf_val("slots")) + " status=" +
self.__state)
packet = utils.parse_packet(
self.__master_connection.recv(1024))
if "ok" not in packet:
utils.print_err(
"Error: Master Server responded poorly to our register request. Will retry."
)
self.__master_connection.close()
self.__master_connection = None
except Exception as e:
if self.__master_connection is not None:
self.__master_connection.close()
self.__master_connection = None
utils.print_err(
"Error: Problem communicating with master server. Will Retry"
)
# set this function up as a re-occuring timer based on the -b/--heartbeat option.
# utils.print_err("HB: interval " + str(self.__hb_timer_interval) + " at " + str(time()))
self.__hb_timer = threading.Timer(self.__hb_timer_interval,
self._register_with_master)
self.__hb_timer.start()
def _handle_cmd(self, connection, address, data):
"""
sub function to handle execmd commands from a direct connection to the socket.
:param connection:
:param address:
:param data: incoming command string.
:type connection: socket.socket
:return:
"""
# security check, only localhost connections.
if address[0] != "127.0.0.1":
connection.send("Permission Denied.\n\n")
connection.close()
return
packet = utils.parse_packet(data)
if "list" in packet:
if "workers" in packet:
connection.send("Currently registered workers:\n")
for worker in self.workers: # type: MasterServerWorkerEntry
connection.send(
"\t" + worker.get_name() + " " + worker.get_uuid() +
" " + worker.get_ip() + " " +
datetime.fromtimestamp(worker.get_last_sync(
)).strftime('%Y-%m-%d %H:%M:%S') + " " +
str(worker.get_slots_total()) + " " +
str(worker.get_slots_in_use()) + " " +
worker.get_status() + "\n")
elif "clients" in packet:
connection.send("Current client requests:\n")
for m_client in self.client_requests: # type: MasterServerClientRequest
connection.send("\t" + m_client.get_ip() + " " +
m_client.get_client_uuid() + " " +
m_client.get_req_mod() + " " +
str(m_client.get_start_time()) + " " +
m_client.get_uuid() + "\n")
else:
connection.send("Error: Unrecognized command.\n")
elif "purge" in packet:
# issue a request to purge any stale worker nodes.
# worker_purge calls the function to also purge stale clients.
self._do_stale_purge()
connection.send("ok\n")
elif "sync" in packet:
# issue a request to sync all worker nodes.
print "Full sync requested. Invalidating all workers."
self._do_worker_syncs()
connection.send("ok\n")
else:
connection.send("Error: Unrecognized command.\n")
def start_arp_listener(self):
while self.run:
# receive arp requests
try:
packet, addr = self.raw_socket.recvfrom(65565)
eth_frame, arp_packet = parse_packet(packet)
arp_type = struct.unpack("!h", arp_packet["oper"])[0]
logger.debug(
"Received ARP-"
+ ("REQUEST" if (arp_type == 1) else "REPLY")
+ " SRC: "
+ eth_frame["src_mac"]
+ " / "
+ arp_packet["src_ip"]
+ " "
+ "DST: "
+ eth_frame["dst_mac"]
+ " / "
+ arp_packet["dst_ip"]
)
if arp_type == 1:
# check if the arp request stems from one of the participants
requester_srcmac = eth_frame["src_mac"]
requested_ip = arp_packet["dst_ip"]
# Send the ARP request message to respective controller and forget about it
if IPAddress(requested_ip) in self.vnhs:
self.send_arp_request(requester_srcmac, requested_ip)
# TODO: If the requested IP address belongs to a non-SDN participant
# then refer the structure `self.nonSDN_nhip_2_nhmac` and
# send an immediate ARP response.
"""
response_vmac = self.get_vmac_default(requester_srcmac, requested_ip)
if response_vmac != "":
logger.debug("ARP-PROXY: reply with VMAC "+response_vmac)
data = self.craft_arp_packet(arp_packet, response_vmac)
eth_packet = self.craft_eth_frame(eth_frame, response_vmac, data)
self.raw_socket.send(''.join(eth_packet))
"""
except socket.timeout:
# prints about once per second
# logger.debug('Socket Timeout Occured')
pass
def __init__(self, info, master_lock):
self.__master_lock = master_lock
packet = utils.parse_packet(info)
if packet is None:
return None
self.__master_lock.acquire()
self.__name = packet["name"]
self.__UUID = packet["worker_uuid"]
self.__slots_total = packet["slots"]
self.__ip = packet["ip"]
if "status" in packet:
self.__status = packet["status"]
else:
self.__status = "outdated"
self.__master_lock.release()
def __init__(self, data):
"""
:param data:
:type data: str
"""
packet = utils.parse_packet(data)
self.__client_UUID = packet["client_uuid"]
self.__req_mod = packet["image"]
self.__ip = packet["ip"]
self.__done = packet["state"]
if "uuid" in packet:
self.__UUID = packet["uuid"]
else:
#print "New Client Request from: " + self.__ip
self.__UUID = str(uuid.uuid4())
self.__start_time = time()
def start_arp_listener(self):
while self.run:
# receive arp requests
try:
packet, addr = self.raw_socket.recvfrom(65565)
eth_frame, arp_packet = parse_packet(packet)
arp_type = struct.unpack("!h", arp_packet["oper"])[0]
logger.debug("Received ARP-" +
("REQUEST" if (arp_type == 1) else "REPLY") +
" SRC: " + eth_frame["src_mac"] + " / " +
arp_packet["src_ip"] + " " + "DST: " +
eth_frame["dst_mac"] + " / " +
arp_packet["dst_ip"])
if arp_type == 1:
# check if the arp request stems from one of the participants
requester_srcmac = eth_frame["src_mac"]
requested_ip = arp_packet["dst_ip"]
# Send the ARP request message to respective controller and forget about it
if IPAddress(requested_ip) in self.vnhs:
self.send_arp_request(requester_srcmac, requested_ip)
# TODO: If the requested IP address belongs to a non-SDN participant
# then refer the structure `self.nonSDN_nhip_2_nhmac` and
# send an immediate ARP response.
"""
response_vmac = self.get_vmac_default(requester_srcmac, requested_ip)
if response_vmac != "":
logger.debug("ARP-PROXY: reply with VMAC "+response_vmac)
data = self.craft_arp_packet(arp_packet, response_vmac)
eth_packet = self.craft_eth_frame(eth_frame, response_vmac, data)
self.raw_socket.send(''.join(eth_packet))
"""
except socket.timeout:
# prints about once per second
#logger.debug('Socket Timeout Occured')
pass
def _handle_client_control(self, connection, req_uuid):
# let's see if we have a client request for this uuid
cli_req = self._find_request_by_uuid(
req_uuid) # type: WorkerServerSync
if cli_req is None:
connection.sendall("err")
return False
# start the control link by sending "ok"
connection.sendall("ok")
# we have a valid request UUID, so let's check it's status
while cli_req.is_sync_active():
try:
packet = utils.parse_packet(connection.recv(10))
if "ok" not in packet:
# something went wrong on teh client side.
# Clean up the request and remove it.
connection.sendall("err")
self._cleanup_client_req(req_uuid)
return False
connection.sendall("ok")
except Exception as e:
connection.sendall("err")
self._cleanup_client_req(req_uuid)
return False
# a small sleep here.
sleep(1)
# if we exit the loop, it's time to turn things down.
connection.sendall("close")
connection.close()
# Clean up the request and remove it.
self._cleanup_client_req(req_uuid)
return False
def _sync_worker(self, worker):
"""
:param worker: worker to sync
:type worker: MasterServerWorkerEntry
:return: True | False
:rtype: int
"""
force_master_sync = False
if worker.get_status() == "syncing":
return
# attempt to sync to a worker. As a last result, try to sync from master.
while not force_master_sync:
self.__master_data_lock.acquire()
worker.set_status("waiting")
self.__master_data_lock.release()
# only try to send us to a worker if we aren't being forced to a master sync
# if we are not forcing a master sync,
# attempt to hand off the client to a worker
worker_sync = self._find_least_updated_worker()
if worker_sync is not None:
# don't allow a worker to sync to itself.
if worker_sync.get_uuid() == worker.get_uuid():
worker_sync = None
self.__master_data_lock.acquire()
worker_sync.set_slots_in_use(
worker_sync.get_slots_in_use() - 1)
# if we are alone, no one is syncing to the master, let us do it.
if self.__sync_slots_used < self.__sync_slots:
self.__sync_slots_used = self.__sync_slots_used + 1
force_master_sync = True
self.__master_data_lock.release()
else:
# we have a valid, updated worker, try a sync
self.__master_data_lock.acquire()
worker.set_sync_src_uuid(worker_sync.get_uuid())
self.__master_data_lock.release()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
# get the ip of the worker we are going to sync TO
worker_address = (worker.get_ip(), 4018)
# connect to the recipient worker and tell it to sync FROM worker_sync
sock.connect(worker_address)
# increment the worker's in use counter.
self.__master_data_lock.acquire()
worker.set_status("syncing")
self.__master_data_lock.release()
sock.sendall("sync master worker=" +
worker_sync.get_ip())
packet = utils.parse_packet(
sock.recv(1024)) # type: dict
# then decrement it after it's over.
self.__master_data_lock.acquire()
worker_sync.set_slots_in_use(
worker_sync.get_slots_in_use() - 1)
self.__master_data_lock.release()
if packet is None:
#utils.print_err("Error: empty packet from " + worker.get_ip())
self.__master_data_lock.acquire()
worker.set_status("waiting")
self.__master_data_lock.release()
else:
if "ok" not in packet:
self.__master_data_lock.acquire()
worker.set_status("waiting")
self.__master_data_lock.release()
else:
# status is updated from an updated worker, set to upadted
self.__master_data_lock.acquire()
worker.set_status("updated")
worker.set_last_sync(time())
self.__master_data_lock.release()
return True
except Exception as e:
self.__master_data_lock.acquire()
worker.set_status("error")
self.__master_data_lock.release()
utils.print_err("Error: worker to worker sync failed")
utils.print_err(e)
# then decrement it after it's over.
self.__master_data_lock.acquire()
worker_sync.set_slots_in_use(
worker_sync.get_slots_in_use() - 1)
self.__master_data_lock.release()
sock.close()
else:
# no updated nodes, if the master is available, let's try that.
self.__master_data_lock.acquire()
if self.__sync_slots_used < self.__sync_slots:
self.__sync_slots_used = self.__sync_slots_used + 1
force_master_sync = True
self.__master_data_lock.release()
# otherwise, wait a bit and try again.
sleep(5)
# no worker found or master sync forced, so proceed with
# normal master sync.
# connect to the worker and wait for it to reply that it's ready.
# create a new socket for the worker connection.
worker.set_sync_src_uuid(self.__UUID)
self.__master_data_lock.acquire()
worker.set_status("syncing")
self.__master_data_lock.release()
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
worker_address = (worker.get_ip(), 4018)
sock.settimeout(30)
try:
sock.connect(worker_address)
sock.sendall("sync master repo_uuid=" + self.__UUID)
packet = utils.parse_packet(sock.recv(1024)) # type: dict
if packet is None:
self.__sync_slots_used = self.__sync_slots_used - 1
utils.print_err("Error: empty packet from " + worker.get_ip())
self.__master_data_lock.acquire()
worker.set_status("outdated")
self.__master_data_lock.release()
sock.close()
return False
if "ok" not in packet:
self.__sync_slots_used = self.__sync_slots_used - 1
utils.print_err("Error: Received bad packet from " +
worker.get_ip())
utils.print_err("Error: Packet: " + packet["raw_packet"])
self.__master_data_lock.acquire()
worker.set_status("outdated")
self.__master_data_lock.release()
sock.close()
return False
except Exception as e:
self.__sync_slots_used = self.__sync_slots_used - 1
utils.print_err("Error: Unable to talk to worker at " +
worker_address[0] + ": " + str(e))
sock.close()
self.__master_data_lock.acquire()
worker.set_status("error")
self.__master_data_lock.release()
return False
# get the worker's parameters
worker_uuid = packet["worker_uuid"]
port = packet["port"]
rsync_mod = packet["module"]
# worker_uuid, port, rsync_mod = data[3:].split()
# double check the worker's supplied params.
if worker_uuid == "":
self.__sync_slots_used = self.__sync_slots_used - 1
utils.print_err("Error: Received bad packet from " +
worker_sync.get_ip())
utils.print_err("Error: Packet: " + packet["raw_packet"])
self.__master_data_lock.acquire()
worker.set_status("error")
self.__master_data_lock.release()
sock.close()
return False
if port == "":
self.__sync_slots_used = self.__sync_slots_used - 1
utils.print_err("Error: Received bad packet from " +
worker_sync.get_ip())
utils.print_err("Error: Packet: " + packet["raw_packet"])
self.__master_data_lock.acquire()
worker.set_status("error")
self.__master_data_lock.release()
sock.close()
return False
if rsync_mod == "":
self.__master_data_lock.acquire()
utils.print_err("Error: Received bad packet from " +
worker_sync.get_ip())
utils.print_err("Error: Packet: " + packet["raw_packet"])
worker.set_status("error")
self.__master_data_lock.release()
sock.close()
return False
sock.close()
# if the worker is ready, start the sync
# create a temporary file with the password
fd, file_path = mkstemp()
pass_file = open(file_path, "w")
pass_file.write(worker_uuid)
pass_file.close()
os.close(fd)
if not os.path.isdir("/tmp/mprov/"):
# mprov tmp dir doesn't exist.
os.mkdir("/tmp/mprov", 0700)
# open an rsync log for logging.
rsync_log = open("/tmp/mprov/master_sync_" + rsync_mod + ".log", "w+")
rsync_args = [
"/usr/bin/rsync", "-avx", "--delete", "--delete-after",
"--progress", "--exclude=/tmp", "--exclude=/proc",
"--exclude=/sys", "--exclude=/run", "--exclude=/dev",
"--password-file=" + file_path, "--port=" + port,
self.__path + "/", "root@" + worker.get_ip() + "::" + rsync_mod
]
#print "Run: " + " ".join(rsync_args)
rsync_log.write("Run: " + " ".join(rsync_args) + "\n")
# wait a couple of seconds for the worker to set up the sync
sleep(5)
#print("Run Rsync")
rsync_proc = subprocess.Popen(rsync_args,
stdout=rsync_log,
stderr=rsync_log)
#print("Call Communicate")
# wait for the rsync to finish.
rsync_proc.communicate()
#print("Grab Return Code")
return_code = rsync_proc.returncode
rsync_log.close()
result_pass = False
# now, let's check the return code of the rsync process and see if it was an error.
if return_code != 0 and return_code != 24:
utils.print_err("Error: rsync returned '" + str(return_code) + "'")
utils.print_err(
"Error: marking worker as status 'error' for worker: " +
worker.get_ip())
self.__master_data_lock.acquire()
worker.set_status("error")
self.__master_data_lock.release()
self.__sync_slots_used = self.__sync_slots_used - 1
else:
self.__master_data_lock.acquire()
worker.set_status("updated")
self.__sync_slots_used = self.__sync_slots_used - 1
if self.__sync_slots_used < 0:
self.__sync_slots_used = 0
worker.set_last_sync(time())
self.__master_data_lock.release()
os.remove(file_path)
result_pass = True
# connect to the worker and tell it to close up the rsync channel.
# create a new socket for the worker connection.
sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
worker_address = (worker.get_ip(), 4018)
# worker should reply in 30 seconds or so...
sock2.settimeout(30)
try:
sock2.connect(worker_address)
# send it the signal to shut-down.
sock2.sendall("stop master repo_uuid=" + self.__UUID)
except Exception as e:
utils.print_err(str(e))
self.__master_data_lock.acquire()
self.__sync_slots_used = self.__sync_slots_used - 1
worker.set_status("error")
self.__master_data_lock.release()
sock2.close()
return False
sock2.close()
# once the sync is done, exit our self.
return result_pass
def _handle_worker_sync(self, connection, address):
"""
handle a new worker request for a sync.
:param connection:
:param address:
:return:
"""
print "Worker to worker sync requested from: " + address[0]
# tell the other end to proceed.
connection.sendall("ok repo_uuid=" + self.__my_uuid)
# now we wait for the worker to tell us it's ready.
connection.settimeout(60)
packet_worker = None # type: dict
try:
packet_worker = utils.parse_packet(connection.recv(
1024)) # this will block until the worker is ready.
except Exception as e:
utils.print_err(
"Error: Client didn't reply back. Stopping request.")
utils.print_err("Error: Exception: " + e.message)
connection.close()
return
if packet_worker is None:
utils.print_err(
"Error: Client didn't reply back. Stopping request.")
connection.close()
return
if "ok" not in packet_worker:
utils.print_err("Error: Client wasn't ready.")
utils.log("Error: Client wasn't ready.",
self.__config.get_conf_val("ms"), 4017)
connection.close()
return
# handle incoming worker sync request.
# check for available slots.
if self.__slots_in_use > 0:
if self.__slots_in_use >= self.__worker_slots:
utils.print_err(
"Error: Worker Full, but client asked for sync. Shouldn't happen."
)
# no slots available. Disconnect
connection.close()
return
# parse the "ok" packet
rsync_uuid = packet_worker["worker_uuid"]
sync_port = packet_worker["port"]
mod_name = packet_worker["module"]
# and set up the rsync.
# if the client is ready, start the sync
# create a temporary file with the password
fd, file_path = mkstemp()
pass_file = open(file_path, "w")
pass_file.write(rsync_uuid)
pass_file.close()
os.close(fd)
rsync_args = [
"/usr/bin/rsync", "-avx", "--delete", "--delete-after",
"--progress", "--exclude=/tmp", "--exclude=/proc",
"--exclude=/sys", "--exclude=/run", "--exclude=/dev",
"--port=" + sync_port, "--password-file=" + file_path,
self.__path + "/", "root@" + address[0] + "::" + mod_name
]
print " ".join(rsync_args)
# logging of the rsync.
if not os.path.isdir("/tmp/mprov/"):
# mprov tmp dir doesn't exist.
os.mkdir("/tmp/mprov", 0700)
# open an rsync log for logging.
rsync_log = open("/tmp/mprov/worker_sync_" + mod_name + ".log", "w+")
# we are about to run but give the worker a couple of seconds to make sure the
# rsync is set up on their end
sleep(10)
rsync_proc = subprocess.Popen(rsync_args,
stdout=rsync_log,
stderr=rsync_log)
# wait for the rsync to finish.
rsync_proc.communicate()
rsync_log.close()
# examine return code and log.... something...
return_code = rsync_proc.returncode
if return_code != 0 and return_code != 24:
utils.print_err("Error: Worker rsync to " + address[0] +
" died prematurely! RC=" + str(return_code))
else:
os.remove(file_path)
# connect to the worker and tell it to close up the rsync channel.
# create a new socket for the worker connection.
sock2 = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
worker_address = (address[0], 4018)
# worker should reply in 30 seconds or so...
sock2.settimeout(30)
try:
sock2.connect(worker_address)
# send it the signal to shut-down.
sock2.sendall("stop master repo_uuid=" + self.__my_uuid)
except Exception as e:
print e
sock2.close()
return False
sock2.close()
connection.close()
return
def _handle_client_sync(self, connection, address, req_uuid):
"""
handle a new client request for a sync.
:param connection:
:param address:
:param req_uuid:
:return:
"""
# first check with the master for the information about the client.
# create a new socket for the master connection.
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
master_address = (self.__config.get_conf_val("ms"), 4017)
sock.settimeout(30)
packet = None # type: dict
try:
sock.connect(master_address)
sock.sendall("verify uuid=" + req_uuid)
packet = utils.parse_packet(sock.recv(1024))
except Exception as e:
utils.print_err("Error: Network issue communicating to master.")
utils.print_err("Error: Exception: " + str(e))
sock.close()
if packet is None:
utils.print_err("Error: Master didn't verify sync request.")
self._cleanup_client_req(req_uuid)
return
if packet["raw_packet"] == "" or "err" in packet:
utils.print_err("Error: Master didn't verify sync request.")
self._cleanup_client_req(req_uuid)
return
print packet["raw_packet"]
cli_uuid = packet["client_uuid"]
req_mod = packet["image"]
cli_ip = packet["ip"]
req_uuid = packet["uuid"]
print(
"Master verified the request. Telling the client and waiting until it's ready."
)
# make an WorkerServerSync object and stick it in the list.
cli_req = WorkerServerSync(req_uuid, cli_uuid, address[0], req_mod,
True)
self.__sync_requests.append(cli_req)
# now we have a valid client object, so tell the client to proceed.
connection.sendall("ok")
# now we wait for the client to tell us it's ready.
connection.settimeout(60)
packet_client = None # type: dict
try:
packet_client = utils.parse_packet(connection.recv(
1024)) # this will block until the client is ready.
except Exception as e:
utils.print_err(
"Error: Client didn't reply back. Stopping request.")
utils.print_err("Error: Exception: " + e.message)
connection.close()
cli_req.set_sync_active(False)
return
if packet_client is None:
utils.print_err(
"Error: Client didn't reply back. Stopping request.")
connection.close()
cli_req.set_sync_active(False)
return
if "ok" not in packet_client:
#utils.print_err("Error: Client wasn't ready.")
utils.log("Error: Client wasn't ready.",
self.__config.get_conf_val("ms"), 4017)
connection.close()
cli_req.set_sync_active(False)
return
# handle incoming client sync request.
# check for available slots.
if self.__slots_in_use > 0:
if self.__slots_in_use >= self.__worker_slots:
utils.print_err(
"Error: Worker Full, but client asked for sync. Shouldn't happen."
)
# no slots available. Disconnect
cli_req.set_sync_active(False)
connection.close()
return
# parse the "ok" packet
rsync_uuid = packet_client["client_uuid"]
sync_port = packet_client["port"]
mod_name = packet_client["module"]
if rsync_uuid != cli_req.get_client_uuid():
utils.print_err("Error: request ID mismatch!")
cli_req.set_sync_active(False)
connection.sendall("err")
connection.close()
return
# and set up the rsync.
# if the client is ready, start the sync
# create a temporary file with the password
fd, file_path = mkstemp()
pass_file = open(file_path, "w")
pass_file.write(rsync_uuid)
pass_file.close()
os.close(fd)
rsync_args = [
"/usr/bin/rsync", "-avx", "--progress", "--port=" + sync_port,
"--exclude=/tmp", "--exclude=/proc", "--exclude=/sys",
"--exclude=/run", "--exclude=/dev", "--password-file=" + file_path,
self.__path + "/" + cli_req.get_req_mod() + "/",
"root@" + address[0] + "::" + mod_name
]
print " ".join(rsync_args)
# logging of the rsync.
if not os.path.isdir("/tmp/mprov/"):
# mprov tmp dir doesn't exist.
os.mkdir("/tmp/mprov", 0700)
# open an rsync log for logging.
rsync_log = open("/tmp/mprov/client_sync_" + mod_name + ".log", "w+")
# we are about to run but give the client a couple of seconds to make sure the
# rsync is set up on their end
sleep(5)
rsync_proc = subprocess.Popen(rsync_args,
stdout=rsync_log,
stderr=rsync_log)
# wait for the rsync to finish.
rsync_proc.communicate()
rsync_log.close()
# examine return code and log.... something...
return_code = rsync_proc.returncode
if return_code != 0 and return_code != 24:
utils.print_err("Error: Client rsync died prematurely! RC=" +
str(return_code))
connection.sendall("ok result=error")
else:
connection.sendall("ok result=pass")
os.remove(file_path)
# mark the sync as done.
cli_req.set_sync_active(False)
connection.close()
return
def _handle_master_sync(self, connection, packet):
"""
handle a sync request from the master server.
:param connection: the socket connection to communicate on
:param packet: the incoming packet
:type connection: socket.socket
:type packet: dict
:return:
"""
# let the master server redirect us to another worker that has an updated
# copy of the image repo
print "Repository Sync"
self.__state = "syncing"
sync_connection = None
if "repo_uuid" in packet:
self.__repo_sync_uuid = packet["repo_uuid"]
# if the master server is sending us a worker to sync from, then connect to the worker.
if "worker" in packet:
worker_address = packet["worker"]
#connect to the worker
sync_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
worker_address = (worker_address, 4018)
try:
sync_connection.connect(worker_address)
sync_connection.sendall("sync worker")
packet_reply = utils.parse_packet(sync_connection.recv(1024))
if "retry" in packet_reply:
print("Worker not redy, will retry")
sync_connection.close()
connection.close()
return
if "ok" not in packet_reply:
utils.print_err(
"Error in worker to worker sync. bad reply from: " +
worker_address[0])
utils.print_err("Error Packet: " +
packet_reply["raw_packet"])
sync_connection.close()
connection.close()
return
if "repo_uuid" in packet_reply:
self.__repo_sync_uuid = packet_reply["repo_uuid"]
except Exception as e:
utils.print_err("Error in worker to worker sync from: " +
worker_address[0])
sync_connection.close()
connection.close()
return
else:
sync_connection = connection
self.__last_master_sync_ip = sync_connection.getpeername()[0]
# generate the module_name
module_name = str(uuid.uuid4())
sync_port = "8970" # TODO: make this dynamic, and random.
# output the rsyncd config and a secret file
rsyncd_fd, rsyncd_path = mkstemp()
secrets_fd, secrets_path = mkstemp()
rsyncd_file = open(rsyncd_path, "w")
file_contents = "[" + module_name + "]\n" + \
"\tpath = " + self.__path + "\n" + \
"\tread only = no\n" + \
"\tauth users = root\n" + \
"\tsecrets file = " + secrets_path + "\n" + \
"\tuid = 0\n" + \
"\tgid = 0\n" + \
"\tpost-xfer exec = /usr/bin/pkill rsync\n"
rsyncd_file.write(file_contents)
rsyncd_file.close()
os.close(rsyncd_fd)
# output the rsyncd secrets file.
rsyncd_file = open(secrets_path, "w")
rsyncd_file.write("root:" + self.__my_uuid)
rsyncd_file.close()
os.close(secrets_fd)
# before we run, kill off any other rsync daemons running
rsyncd_running = subprocess.Popen(
["/usr/bin/pgrep", "-f", "rsync --daemon"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(pgrep_stdout, pgrep_stderr) = rsyncd_running.communicate()
utils.print_err("")
if pgrep_stdout != "" and pgrep_stdout is not None:
for pid in pgrep_stdout.split("\n"):
if pid != "":
os.kill(int(pid), 9)
# setup the rsyncd command.
rsyncd_proc = subprocess.Popen([
"/usr/bin/rsync", "--daemon", "--port=" + sync_port, "--no-detach",
"-v", "-4", "--config=" + rsyncd_path
],
shell=False)
self.__rsyncd_pid = rsyncd_proc.pid
sync_connection.sendall("ok worker_uuid=" + self.__my_uuid + " port=" +
sync_port + " module=" + module_name)
rsyncd_proc.communicate()
reply = "ok"
self.__state = "updated"
# check the return code here!
if rsyncd_proc.returncode != 0 and rsyncd_proc.returncode != 24 and rsyncd_proc.returncode != 20:
utils.print_err("Error: rsync from " +
sync_connection.getpeername()[0] +
" died unexpectedly with RC=" +
str(rsyncd_proc.returncode) + "!!!")
reply = "err"
self.__state = "err"
# if we used a different connection for the sync, then tell the master we are done.
if sync_connection != connection:
connection.sendall(reply)
connection.close()
# clean up the tmp files.
os.remove(rsyncd_path)
os.remove(secrets_path)
sync_connection.close()
print "Sync Complete."
def _handle_connection(self, connection, address):
"""
Handle the incoming connection
:param connection: socket to communicate on.
:type connection: socket.socket
:type address: List[str]
:param address: the address that is connected to us.
:return: True if we close OK, False if we don't. I don't think this matters.
"""
size = 1024
while True:
try:
data = connection.recv(size)
packet = utils.parse_packet(data)
if packet is not None:
if "sync" in packet:
if "master" in packet:
self.__master_sync_active = True
self._handle_master_sync(connection, packet)
self.__master_sync_active = False
elif "client" in packet:
if self.__master_sync_active:
connection.sendall("retry")
connection.close()
return False
self._handle_client_sync(connection, address,
packet["uuid"])
elif "worker" in packet:
if self.__master_sync_active:
connection.sendall("retry")
connection.close()
return False
self._handle_worker_sync(connection, address)
elif "stop" in packet:
if "master" in packet:
if "repo_uuid" in packet:
if packet[
"repo_uuid"] == self.__repo_sync_uuid:
# we are getting a stop from where we expect.
self._handle_stop_master_sync(
connection, address)
return True
else:
utils.print_err(
"Error: stop command unknown repo_uuid: "
+ packet["repo_uuid"])
else:
utils.print_err(
"Error: stop command from source with no UUID."
)
utils.print_err("Error: should be: " +
self.__repo_sync_uuid)
utils.print_err("Error: this is probably bad.")
connection.close()
return False
elif "client" in packet:
# purge a stale client request, kill any rsyncs running, and open the slot.
if address[0] == self.__config.get_conf_val("ms"):
self._handle_stop_client_sync(
connection, address, packet["uuid"])
connection.close()
return False
elif "control" in packet:
# a control connection request
if "client" in packet:
# this is a client connection request. Run the handler.
self._handle_client_control(
connection, packet["uuid"])
else:
utils.print_err("Unrecognized Packet: " + data)
connection.close()
return False
# done parsing, close the connection
connection.close()
return True
except Exception as e:
utils.print_err("Error: Unhandled exception thrown: " + str(e))
utils.print_err("Error: Packet generating error: " + data)
traceback.print_exc(file=sys.stderr)
connection.close()
return False
def __init__(self, info):
packet = utils.parse_packet(info)
self.__name = packet["name"]
self.__UUID = packet["worker_uuid"]
self.__slots_total = packet["slots"]
self.__ip = packet["ip"]
