class LockServer:

    def _lock_handler(self, conn, host, port, f):
        status = self._lock_list.check_file(f)
        if status is None:
            self._lock_list.add(f, host, port)
            conn.send(Response.OK)
        elif status[0] == host and status[1] == port:
            conn.send(Response.OK)
        else:
            conn.send(Response.LOCK_TAKEN)

    def _unlock_handler(self, conn, host, port, f):
        status = self._lock_list.check_file(f)
        if status is None:
            conn.send(Response.LOCK_FREE)
        elif status[0] == host and status[1] == port:
            self._lock_list.remove(f, host, port)
            conn.send(Response.OK)
        else:
            conn.send(Response.LOCK_TAKEN)

    def _request_handler(self, conn):
        try:
            # no initial request can be longer than 8096 bytes
            data = conn.recv(8096)
            input = data.split(" ")

            # invoke respective handlers for the input command
            if input[0] == "LOCK":
                self._lock_handler(conn, input[1], input[2], input[3])
                print "Received LOCK from "+input[1]+":"+input[2]+" for: "+input[3]
            elif input[0] == "UNLOCK":
                self._unlock_handler(conn, input[1], input[2], input[3])
            else:
                conn.send("INVALID_COMMAND")
            conn.close()
        except Exception as e:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
            print exc_type.__name__ + " " + fname + ":" + str(exc_tb.tb_lineno) + " " + str(e)
            conn.send(Response.ERROR)
            conn.close()

    def __init__(self, port):
        self._port = port
        self._server = TCPServer(self._port, 10, self._request_handler)
        self._lock_list = LockList()
        self._server.start()
class ReplicationManager:

    def _advertise_handler(self, conn, host, port):
        location = Location(host, port)
        self._replication_controller.add(location)
        conn.send(Response.OK)

    def _lookup_handler(self, conn, host, port):
        location = Location(host, port)
        set = self._replication_controller.lookup(location)
        for i in range(0, len(set)):
            if set[i].compare(location):
                del set[i]      # don't include the requesting node in the response, it already knows it's in the set
                break
        str_set = ""            # a string representation of the response set
        for item in set:
            # convert each item in the response set to the string representation (easier to parse for the client)
            str_set += item.get_string() + " "
        conn.send(Response.OK + " " + str_set.rstrip(" "))

    def _request_handler(self, conn):
        try:
            # no initial request can be longer than 8096 bytes
            data = conn.recv(8096)
            input = data.split(" ")

            # invoke respective handlers for the input command
            if input[0] == "ADVERTISE":
                self._advertise_handler(conn, input[1], input[2])
                print "Received ADVERTISE from "+input[1]+":"+input[2]
            elif input[0] == "LOOKUP":
                self._lookup_handler(conn, input[1], input[2])
            else:
                conn.send("INVALID_COMMAND")
            conn.close()
        except Exception as e:
            print str(e)
            conn.send(Response.ERROR)
            conn.close()

    def __init__(self, port):
        self._port = port
        self._server = TCPServer(self._port, 10, self._request_handler)
        self._server.start()
        self._replication_controller = ReplicationController()
class Node(object):

    # convert a fully qualified path to a relative path
    def _relative_path(self, base, path):
        _base = base.strip('/')
        _path = path.strip('/')
        if _path.startswith(_base):
            _path = _path[len(_base):]
        return _path.strip('/')

    # GET downloads a file to the client
    def _get_handler(self, conn, input):
        filename = self._dir + input
        if not os.path.isfile(filename):
            conn.send(Response.NO_EXIST)
        else:
            f = open(filename, "rb")
            conn.send_file(f)
            f.close()
            conn.shutdown(socket.SHUT_WR)
            conn.close()

    # XPUT uploads a file (either adding a new file or overwriting an existing one) WITHOUT forwarding it to the
    # replication set. this is used to stop PUT messages circling through the network forever
    def _xput_handler(self, conn, input):
        filepath = os.path.dirname(input)
        if not os.path.isdir(self._dir + filepath):
            conn.send(Response.NO_EXIST)
            return False
        conn.send(Response.OK)
        # check if the file existed or not before overwriting
        exists = os.path.isfile(self._dir + input)
        f = open(self._dir + input, "wb")
        conn.recv_file(f)
        f.close()
        conn.send(Response.OK)
        if not exists:
            self._advertise_buffer.add(input, ObjectBuffer.Type.file)
        return True

    # PUT uploads a file (either adding a new file or overwriting an existing one)
    def _put_handler(self, conn, input):
        # perform the usual replication-less xput first
        success = self._xput_handler(conn, input)
        # now add this put operation to the replication buffer if it was a successful xput (ie the client didn't try
        # to put a file to an invalid directory)
        if success:
            self._replication_buffer.add(input, ObjectBuffer.Type.file)

    # XMKDIR creates a directory in the node WITHOUT forwarding it to the replication set
    def _xmkdir_handler(self, conn, input):
        newdir = str(self._dir + input.strip('/'))
        basedir = os.path.dirname(newdir)
        if not os.path.isdir(basedir):
            conn.send(Response.NO_EXIST)
            return False
        else:
            try:
                os.makedirs(newdir)
                conn.send(Response.OK)
                self._advertise_buffer.add(input.strip('/'), ObjectBuffer.Type.directory)
                return True
            except Exception as e:
                if e.errno != errno.EEXIST:
                    raise
                else:
                    conn.send(Response.EXISTS)
                return False

    # MKDIR creates a directory in the node
    def _mkdir_handler(self, conn, input):
        success = self._xmkdir_handler(conn, input)
        if success:
            self._replication_buffer.add(input.strip('/'), ObjectBuffer.Type.directory)

    # XDELETE deletes a file or directory *recursively* in the node WITHOUT forwarding it to the replication set
    def _xdelete_handler(self, conn, input):
        object = str(self._dir + input).rstrip('/')
        if os.path.isdir(object):
            shutil.rmtree(object)
            conn.send(Response.OK)
            self._advertise_buffer.add(input, ObjectBuffer.Type.deleteDirectory)
        elif os.path.isfile(object):
            os.remove(object)
            conn.send(Response.OK)
            self._advertise_buffer.add(input, ObjectBuffer.Type.deleteFile)
        else:
            conn.send(Response.NO_EXIST)
            return False
        return True

    #DELETE deletes a file or directory recursively
    def _delete_handler(self, conn, input):
        success = self._xdelete_handler(conn, input)
        if success:
            isfile = os.path.isfile(self._dir + input)
            repl_type = ObjectBuffer.Type.deleteFile if isfile else ObjectBuffer.Type.deleteDirectory
            self._replication_buffer.add(input, repl_type)

    # called whenever the server receives data
    def _request_handler(self, conn):
        try:
            # no initial request can be longer than 8096 bytes
            data = conn.recv(8096)
            input = data.split()

            # invoke respective handlers for the input command
            if input[0] == "GET":
                self._get_handler(conn, input[1])
            elif input[0] == "XPUT":
                self._xput_handler(conn, input[1])
            elif input[0] == "PUT":
                self._put_handler(conn, input[1])
            elif input[0] == "XMKDIR":
                self._xmkdir_handler(conn, input[1])
            elif input[0] == "MKDIR":
                self._mkdir_handler(conn, input[1])
            elif input[0] == "XDELETE":
                self._xdelete_handler(conn, input[1])
            elif input[0] == "DELETE":
                self._delete_handler(conn, input[1])
            else:
                conn.send(Response.INVALID_COMMAND)
            conn.close()
        except Exception as e:
            conn.send(Response.ERROR + " " + str(e))
            conn.close()

    def _init_advertise(self, server, host, port, adv_host, adv_port):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((adv_host, adv_port))
        except Exception as e:
            if e.errno == errno.ECONNREFUSED:
                print "*!* ERROR: Unable to connect to " + server + "!"
                print "*!* Node and " + server + " are *NOT* in sync!"
            else:
                print e
            return None
        s.send("ADVERTISE "+host+" "+str(port))
        data = s.recv(1024)
        if data != Response.OK:
            print "Received unusual response from " + server + ": " + data + ". Attempting to continue anyway."
        return s

    # advertise *all* files in the Nodes filesystem to the DirectoryServer
    def _directory_server_advertisement(self, host, port, ds_host, ds_port):
        print "Advertising to Directory Server..."
        s = self._init_advertise("Directory Server", host, port, ds_host, ds_port)
        if s is None:
            return
        # repeatedly send advertisement messages of the structure of the server filesystem
        for (dirpath, dirnames, filenames) in os.walk(self._dir):
            advertisement = Advertisement(self._relative_path(self._dir, dirpath), dirnames, filenames)
            s.send(advertisement.to_json())
            data = s.recv(1024)
            if data != Response.OK:
                print "Received unusual response from Directory Server: "+data+". Attempting to continue anyway."
        s.close()
        print "Advertisement complete: Node and Directory Server in sync."

    # advertise this Node to the replication manager, letting it know that this Node exists
    def _replication_manager_advertisement(self, host, port, rm_host, rm_port):
        print "Advertising to Replication Manager..."
        s = self._init_advertise("Replication Manager", host, port, rm_host, rm_port)
        if s is None:
            return
        s.close()
        print "Advertisement complete: Node and Replication Manager in sync."

    # perform a full advertise to the Directory Server and Replication Manager. this is done at startup
    def _full_advertise(self, host, port, ds_host, ds_port, rm_host, rm_port):
        self._directory_server_advertisement(host, port, ds_host, ds_port)
        self._replication_manager_advertisement(host, port, rm_host, rm_port)

    # this manages batch updates of advertisements to the directory server. it runs in its own thread, and will sleep
    # on a condition variable until data becomes available in the buffer to send to the directory server
    def _incremental_advertise(self, host, port, ds_host, ds_port):
        while True:
            # sleep on condition variable
            while self._advertise_buffer.is_empty():
                self._advertise_cv.acquire()
                self._advertise_cv.wait()
                self._advertise_cv.release()
                time.sleep(Interval.ADVERTISE)
            # if the pc reaches here, this thread has been woken to send an incremental advertise and clear the buffer
            s = self._init_advertise("Directory Server", host, port, ds_host, ds_port)
            if s is None:
                return
            self._advertise_cv.acquire()
            messages = self._advertise_buffer.get_all()
            for message in messages:
                s.send(message.to_json())
                data = s.recv(1024)
                if data != Response.OK:
                    print "Received unusual response from Directory Server: "+data+". Attempting to continue anyway."
            self._advertise_buffer.clear()
            self._advertise_cv.release()
            s.close()
            print "Advertisement complete: Node and Directory Server in sync."

    # find out which other Nodes are in the same replication set as this Node
    def _get_replication_set(self, host, port, rm_host, rm_port):
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            s.connect((rm_host, rm_port))
        except Exception as e:
            if e.errno == errno.ECONNREFUSED:
                print "*!* ERROR: Unable to connect to ReplicationManager!"
                print "*!* Node and ReplicationManager are *NOT* in sync!"
            else:
                print e
            return None
        s.send("LOOKUP "+host+" "+str(port))
        data = s.recv(1024)
        s.close()
        if len(data):
            arr = data.split(" ")[1:]   # remove the first element ("OK"), the rest of the elements are the replicants
            locations = []
            for item in arr:
                parts = item.split(":")
                loc = Location(parts[0], parts[1])
                locations.append(loc)
            return locations
        return None

    # incrementally send batch updates to each Node in the replication set, *peer to peer*
    def _replication_manager_update(self, host, port, rm_host, rm_port):
        while True:
            # sleep on condition variable
            while self._replication_buffer.is_empty():
                self._replication_cv.acquire()
                self._replication_cv.wait()
                self._replication_cv.release()
                time.sleep(Interval.ADVERTISE)
            self._replication_cv.acquire()
            # flush the buffer and then release the lock so request handler threads waiting on the lock can continue
            adv_list = self._replication_buffer.get_all()
            self._replication_buffer.clear()
            self._replication_cv.release()

            # get the replication set
            set = self._get_replication_set(host, port, rm_host, rm_port)
            if set is None:
                continue

            for loc in set:
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                s.connect((loc.host, int(loc.port)))
                conn = ConnectionHelper(s)
                for item in adv_list:
                    for dir in item.dirnames:
                        s.send("XMKDIR "+dir)
                        data = s.recv(1024)
                        if data != Response.OK:
                            print "Received unusual response from replication peer "+loc.host+":"+loc.port
                    for file in item.filenames:
                        s.send("XPUT "+file)
                        data = s.recv(1024)
                        if data != Response.OK:
                            print "Received unusual response from replication peer "+loc.host+":"+loc.port
                            continue    # abort XPUT operation
                        f = open(self._dir + item.dirpath + "/" + file, "rb")
                        conn.send_file(f)
                    for i in item.deletelist:
                        s.send("XDELETE "+i)
                        data = s.recv(1024)
                        if data != Response.OK:
                            print "Received unusual response from replication peer "+loc.host+":"+loc.port
                conn.close()

    def __init__(self, dir, host, port, ds_host, ds_port, rm_host, rm_port):
        self._dir = dir
        # append a slash to the end of the home directory if it's not passed in
        if self._dir[len(self._dir)-1] != '/':
            self._dir += '/'
        # initialise the condition variable and buffer for the incremental advertise thread
        self._advertise_cv = threading.Condition()
        self._advertise_buffer = ObjectBuffer(self._advertise_cv)
        # initialise the condition variable and buffer for the replication forwarder
        self._replication_cv = threading.Condition()
        self._replication_buffer = ObjectBuffer(self._replication_cv)
        # do an initial full advertisement (to the directory server and replication manager) before the threadpool init
        self._full_advertise(host, port, ds_host, ds_port, rm_host, rm_port)
        # now initialise the node's listening threadpool with 10 threads
        self._server = TCPServer(port, 10, self._request_handler)
        t = threading.Thread(target=self._incremental_advertise, args=(host, port, ds_host, ds_port,))
        t.daemon = True
        t.start()
        t = threading.Thread(target=self._replication_manager_update, args=(host, port, rm_host, rm_port,))
        t.daemon = True
        t.start()
        self._server.start()
        print "Node server started successfully."
class DirectoryServer:

    # keeps directory location names consistent throughout the directory structure
    def _sanitise_location(self, loc):
        if not loc.startswith('/'):
            loc = '/' + loc
        return loc.rstrip('/')

    # respond to advertisements, which are updates from Nodes. Nodes send these updates when their filesystems have
    # changed
    def _advertise_handler(self, conn, host, port):
        conn.send(Response.OK)
        data = conn.recv(8096)
        while data:
            data = json.loads(data)
            self._tree.add(host, port, data["dirnames"], data["filenames"], data["dirpath"], data["deletelist"])
            conn.send(Response.OK)
            data = conn.recv(8096)

    # respond to a request from a client to get the location of a file. it will try to respond with a random location
    # (if the file is stored in multiple Nodes) to try to balance load between Nodes.
    def _get_handler(self, conn, location):
        node = self._tree.find(location)
        if node is None:
            conn.send(Response.NO_EXIST)
        elif isinstance(node, DT.Directory):
            conn.send(Response.IS_DIRECTORY)
        else:
            conn.send(Response.OK + " " + node.random_loc()+" "+location)

    # respond to a request from a client who wants to upload a new file. the DirectoryServer will choose a server for
    # the client to upload the file to.
    def _put_handler(self, conn, location):
        location = self._sanitise_location(location)
        parent = os.path.dirname(location)
        pnode = self._tree.find(parent)
        if pnode is None:
            conn.send(Response.ERROR)
            return

        child = self._tree.find(location)
        if child is None:
            # pick the server with the least amount of objects in the hierarchy
            loc = pnode.hlocs[len(pnode.hlocs)-1]
            conn.send(Response.OK + " " + loc.get_string())
            return

        # pick a random (existing) child location
        conn.send(Response.OK + " " + child.random_loc())

    # respond to a request from a client who wants to create a new directory. the DirectoryServer will choose a server
    # for the client to create the new directory in.
    def _mkdir_handler(self, conn, location):
        location = self._sanitise_location(location)
        parent = os.path.dirname(location)
        child = location[len(parent)-1:].strip('/')
        pnode = self._tree.find(parent)
        if pnode is None:
            conn.send(Response.ERROR)
            return

        if pnode.get_child(child) is not None:
            conn.send(Response.EXISTS)
            return

        # pick the server with the least amount of objects in the hierarchy
        loc = pnode.hlocs[len(pnode.hlocs)-1]
        conn.send(Response.OK + " " + loc.get_string())

    # respond to a client request to list all items in a current location. similar to 'ls' in linux.
    def _list_handler(self, conn, location):
        node = self._tree.find(location)
        if node is None:
            conn.send(Response.NO_EXIST)
        elif not isinstance(node, DT.Directory):
            conn.send(Response.CANT_LIST)
        else:
            children = []
            for child in node.children:
                children.append(child.name)
            conn.send(str(children))

    def _request_handler(self, conn):
        try:
            # no initial request can be longer than 8096 bytes
            data = conn.recv(8096)
            input = data.split(" ")

            # invoke respective handlers for the input command
            if input[0] == "ADVERTISE":
                self._advertise_handler(conn, input[1], input[2])
                print "Received ADVERTISE from "+input[1]+":"+input[2]
            elif input[0] == "GET":
                self._get_handler(conn, input[1])
            elif input[0] == "PUT":
                self._put_handler(conn, input[1])
            elif input[0] == "MKDIR":
                self._mkdir_handler(conn, input[1])
            elif input[0] == "LIST":
                self._list_handler(conn, input[1])
            elif input[0] == "PRINT":
                self._tree.pretty_print(input[1])
            else:
                conn.send("INVALID_COMMAND")
            conn.close()
        except Exception as e:
            exc_type, exc_obj, exc_tb = sys.exc_info()
            fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
            print exc_type.__name__ + " " + fname + ":" + str(exc_tb.tb_lineno) + " " + str(e)
            conn.send(Response.ERROR)
            conn.close()

    def __init__(self, port):
        self._port = port
        self._server = TCPServer(self._port, 10, self._request_handler)
        self._tree = DT.DirectoryTree()
        self._server.start()