コード例 #1
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
    def getAllLocations(self):
        locations = database.getChunkLocations(self.msg[1])
        msg = ""
        for item in locations:
            msg += item + "|"

        fL.send(self.s, msg)
コード例 #2
0
    def getAllLocations(self):
        locations = database.getChunkLocations(self.msg[1])
        msg = ""
        for item in locations:
            msg += item + "|"

        fL.send(self.s, msg)
コード例 #3
0
    def delete(self, filename):
        # Create socket connection to the master
        m = self.createSocket()
        # if there was an error making the socket, exit the function
        if not m:
            return 0

        #send DELETE request to the master
        try:
            fL.send(m, "DELETE|" + filename)
        except:
            print "ERROR: COULD NOT SEND DELETE REQUEST TO MASTER"
        #receive acks from the master
        self.data = fL.recv(m)
        #tell the user whether the file was successfully marked or not
        if self.data == "FAILED1":
            print "ERROR: The file could not be marked for deletion."
            return -1
        elif self.data == "FAILED2":
            print "ERROR: The given file name does not exist."
            return -2
        elif self.data == "FAILED3":
            print "The file has already been marked for deletion."
            return -3
        elif self.data == "MARKED":
            print "File successfully marked for deletion."
        m.close()
コード例 #4
0
ファイル: API.py プロジェクト: BenningtonCS/GFS
	def delete(self, filename):
		# Create socket connection to the master
		m = self.createSocket()
		# if there was an error making the socket, exit the function
		if not m:
			return 0

		#send DELETE request to the master
		try:
			fL.send(m, "DELETE|" + filename)
		except:
			print "ERROR: COULD NOT SEND DELETE REQUEST TO MASTER"
		#receive acks from the master
		self.data = fL.recv(m)
		#tell the user whether the file was successfully marked or not
		if self.data == "FAILED1":
			print "ERROR: The file could not be marked for deletion."
			return -1
		elif self.data == "FAILED2":
			print "ERROR: The given file name does not exist."
			return -2
		elif self.data == "FAILED3":
			print "The file has already been marked for deletion."
			return -3
		elif self.data == "MARKED":
			print "File successfully marked for deletion."
		m.close()
コード例 #5
0
    def append(self):
        # Visual confirmation for debugging: confirm init of append()
        logging.debug('Gathering metadata for chunk append')

        # We know that we will only be appending to the lastest chunk, since a new
        # chunk should only be created when an old chunk fills up, so we find the
        # handle of the latest chunk for a given file.
        latestChunkHandle = database.findLatestChunk(self.fileName)
        # Then we get the locations where that chunk is stored
        locations = database.getChunkLocations(latestChunkHandle)

        # Define an empty string that will hold the message we send back to the client
        appendMessage = ''

        # Parse the locations list into a pipe separated string
        for item in locations:
            appendMessage += item + '|'

        # Add the chunk handle to the message we will send to the client
        appendMessage += str(latestChunkHandle)

        #send our message
        fL.send(self.s, appendMessage)
        # Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle
        logging.debug('SENT == ' + str(appendMessage))
        # Visual confirmation for debugging: confirm success of append()
        logging.debug('Append successfully handled')
コード例 #6
0
 def createChunk(self):
     self.lock.acquire()
     chunkHandle = database.getChunkHandle()
     self.lock.release()
     newChunk = database.createNewChunk(self.msg[1], self.msg[2],
                                        chunkHandle)
     fL.send(self.s, newChunk)
コード例 #7
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
    def append(self):
        # Visual confirmation for debugging: confirm init of append()
        logging.debug("Gathering metadata for chunk append")

        # We know that we will only be appending to the lastest chunk, since a new
        # chunk should only be created when an old chunk fills up, so we find the
        # handle of the latest chunk for a given file.
        latestChunkHandle = database.findLatestChunk(self.fileName)
        # Then we get the locations where that chunk is stored
        locations = database.getChunkLocations(latestChunkHandle)

        # Define an empty string that will hold the message we send back to the client
        appendMessage = ""

        # Parse the locations list into a pipe separated string
        for item in locations:
            appendMessage += item + "|"

            # Add the chunk handle to the message we will send to the client
        appendMessage += str(latestChunkHandle)

        # send our message
        fL.send(self.s, appendMessage)
        # Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle
        logging.debug("SENT == " + str(appendMessage))
        # Visual confirmation for debugging: confirm success of append()
        logging.debug("Append successfully handled")
コード例 #8
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
 def getDeleteData(self):
     # Get the list of files from the database
     toDelete = database.toDelete
     # Create an empty message which will hold the data
     msg = ""
     # For each item in the list, add it to the message string
     for item in toDelete:
         msg += item + "|"
         # Send back the string of delete data
     fL.send(self.s, msg)
コード例 #9
0
 def getDeleteData(self):
     # Get the list of files from the database
     toDelete = database.toDelete
     # Create an empty message which will hold the data
     msg = ""
     # For each item in the list, add it to the message string
     for item in toDelete:
         msg += item + "|"
     # Send back the string of delete data
     fL.send(self.s, msg)
コード例 #10
0
    def read(self, filename, byteOffset, bytesToRead, newName):
        # Create socket connection to the master
        m = self.createSocket()
        # if there was an error making the socket, exit the function
        if not m:
            return 0

        #send READ request to the master
        try:
            fL.send(
                m, "READ|" + filename + "|" + str(byteOffset) + "|" +
                str(bytesToRead))
        except:
            print "ERROR: COULD NOT SEND READ REQUEST TO MASTER"
        #recieve data from the master
        self.data = fL.recv(m)
        #close connection to master
        m.close()
        #split the data into a list
        self.splitdata = self.data.split("|")
        #remove the first element of the list because it is irrelevant
        self.splitdata = self.splitdata[1:]
        #iterate through the list
        fromChunks = ""
        for q in self.splitdata:
            #split the list into smaller parts
            secondSplit = q.split("*")
            #set the location...
            location = secondSplit[0]
            #...and the chunk handle
            cH = secondSplit[1]
            #...and the offset
            offset = secondSplit[2]
            #connect to the chunk server
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            try:
                s.connect((location, self.TCP_PORT))
            except:
                print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location
                return 0
            #send READ request to chunk server
            fL.send(
                s,
                "READ|" + str(cH) + "|" + str(offset) + "|" + str(bytesToRead))
            #receive and print the contents of the file
            #fromChunks += "." + str(cH)
            dat = fL.recv(s)

            #close connection to chunk server
            s.close()

            with open(newName, "ab") as e:
                e.write(dat)

        return 1
コード例 #11
0
ファイル: API.py プロジェクト: BenningtonCS/GFS
	def read(self, filename, byteOffset, bytesToRead, newName):
		# Create socket connection to the master
		m = self.createSocket()
		# if there was an error making the socket, exit the function
		if not m:
			return 0

		#send READ request to the master
		try:
			fL.send(m, "READ|" + filename + "|" + str(byteOffset) + "|" + str(bytesToRead))
		except:
			print "ERROR: COULD NOT SEND READ REQUEST TO MASTER"
		#recieve data from the master
		self.data = fL.recv(m)
		#close connection to master
		m.close()
		#split the data into a list
		self.splitdata = self.data.split("|")
		#remove the first element of the list because it is irrelevant
		self.splitdata = self.splitdata[1:]
		#iterate through the list
		fromChunks = ""
		for q in self.splitdata:
			#split the list into smaller parts
			secondSplit = q.split("*")
			#set the location...
			location = secondSplit[0]
			#...and the chunk handle
			cH = secondSplit[1]
			#...and the offset
			offset = secondSplit[2]
			#connect to the chunk server
			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			try:
				s.connect((location,self.TCP_PORT))
			except:
				print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location
				return 0
			#send READ request to chunk server
			fL.send(s, "READ|" + str(cH) + "|" + str(offset) + "|" + str(bytesToRead))
			#receive and print the contents of the file
			#fromChunks += "." + str(cH)
			dat = fL.recv(s)
					
			#close connection to chunk server
			s.close()
               	
			with open(newName,"ab") as e:
				e.write(dat)
		
		return 1
コード例 #12
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
    def delete(self):
        logging.debug("Begin updating delete flag to True")

        # Make sure the file is not already marked for delete
        if self.fileName not in database.toDelete:
            # Make sure the file you wish to delete is actually a file in the system
            if self.fileName in database.data.keys():
                try:
                    # Change the delete flag for the specified file
                    database.flagDelete(self.fileName)
                    # Confirm that file has been marked for deletion
                    fL.send(self.s, "MARKED")

                    logging.debug("Delete Flags Updated")

                except:
                    logging.error("File could not be marked for deletion")
                    fL.send(self.s, "FAILED1")

            else:
                logging.debug("The file, " + self.fileName + ", does not exist.")
                fL.send(self.s, "FAILED2")

        else:
            logging.debug("The file, " + self.fileName + ", is already marked for delete")
            fL.send(self.s, "FAILED3")
コード例 #13
0
    def delete(self):
        logging.debug('Begin updating delete flag to True')

        # Make sure the file is not already marked for delete
        if self.fileName not in database.toDelete:
            # Make sure the file you wish to delete is actually a file in the system
            if self.fileName in database.data.keys():
                try:
                    # Change the delete flag for the specified file
                    database.flagDelete(self.fileName)
                    # Confirm that file has been marked for deletion
                    fL.send(self.s, "MARKED")

                    logging.debug('Delete Flags Updated')

                except:
                    logging.error("File could not be marked for deletion")
                    fL.send(self.s, "FAILED1")

            else:
                logging.debug('The file, ' + self.fileName +
                              ', does not exist.')
                fL.send(self.s, "FAILED2")

        else:
            logging.debug('The file, ' + self.fileName +
                          ', is already marked for delete')
            fL.send(self.s, "FAILED3")
コード例 #14
0
    def create(self):
        # Visual confirmation for debugging: confirm init of create()
        logging.debug('Creating chunk metadata')
        # Acquire a lock so the chunk handle counter can not be accessed simultaneously
        self.lock.acquire()
        # Get a new chunkhandle
        chunkHandle = database.getChunkHandle()
        # Release the lock so others can access the chunk handle counter
        self.lock.release()

        # Create a new file, and store the return flag
        createFileFlag = database.createNewFile(self.fileName, chunkHandle)

        # If the return flag was an error flag, alert the logger and API of the error
        if createFileFlag == -1:
            logging.error("Got a duplicate file name, sending FAIL to API")
            fL.send(self.s, "FAIL1")
            return -1

        elif createFileFlag == -2:
            logging.error("No file exists for a chunk to be created for")
            fL.send(self.s, "FAIL2")

        elif createFileFlag == -3:
            logging.error(
                "Chunk is not the latest chunk. New chunk has been created that can be appended to."
            )
            fL.send(self.s, "FAIL3")

        # Get the locations for a specified chunk
        locations = database.data[self.fileName].chunks[chunkHandle].locations

        # Parse the locations list retreived above into a pipe-separated list
        hosts = ""
        for item in locations:
            hosts += item + "|"

        # Visual confirmation for debugging: confirm success of create()
        logging.debug('Chunk metadata successfully created')

        try:
            # Send the API a string containing the location and chunkHandle information
            fL.send(self.s, str(hosts) + str(chunkHandle))

        except socket.error:
            logging.warning('Socket Connection Broken')
        # Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle
        logging.debug('SENT ==> ' + str(hosts) + str(chunkHandle))

        # Receieve an ack to affirm that the chunk was successfully created
        ack = fL.recv(self.s)

        if ack == "CREATED":
            logging.debug("successful creation")
        elif ack == "FAILED":
            logging.error("unsuccessful creation")
        else:
            logging.error("unknown ack")
コード例 #15
0
ファイル: heartBeat.py プロジェクト: ttanay/GFS
    def heartBeat(self, IP):
        logging.debug('HEARTBEAT: Initiating heartBeat protocol')
        # Get the list of all active chunk server IPs
        activeServers = self.getActiveChunkServers()

        try:
            # Create a TCP socket instance
            self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            # Set the timeout of the socket
            self.s.settimeout(self.SOCK_TIMEOUT)
            # Allow the socket to re-use address
            self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            # Connect to the chunkserver over the specified port
            self.s.connect((IP, self.PORT))
            # Send the chunk server a heart (ping)
            fL.send(self.s, "<3?")
            # Get the chunk server response
            data = fL.recv(self.s)
            print data
            # Close the connection to allow future connections
            self.s.close()
            # If the chunk server responds with a heart, add it to activehosts
            if data == "<3!":
                # If the chunkserver IP is not in the activeServers list, add it to the list
                if IP not in activeServers:
                    with open(self.AHOSTS, "a") as file:
                        file.write(IP + "\n")
            # Return a 1 so any script calling heartBeat will know the heartbeat was successful
            return 1

    # Handle the timeout (chunk server alive but not responding) and connection (server dead) errors
        except (socket.timeout, socket.error):
            print "</3"
            logging.debug('HEARTBEAT: Could not connect to ' + IP)
            # Check to see if the chunkserver is in the list of active IPs
            if IP in activeServers:
                # If it is, remove it from the list
                activeServers.remove(IP)
                # Clear the previous activehosts.txt file and replace it with the list of active servers,
                # which now excludes the failed chunkserver
                with open(self.AHOSTS, "w") as file:
                    newList = ""
                    for item in activeServers:
                        newList += item + "\n"
                    file.write(newList)
# Return a -1 so any script calling heartbeat will know the heartbeat failed.
            return -1
コード例 #16
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
    def create(self):
        # Visual confirmation for debugging: confirm init of create()
        logging.debug("Creating chunk metadata")
        # Acquire a lock so the chunk handle counter can not be accessed simultaneously
        self.lock.acquire()
        # Get a new chunkhandle
        chunkHandle = database.getChunkHandle()
        # Release the lock so others can access the chunk handle counter
        self.lock.release()

        # Create a new file, and store the return flag
        createFileFlag = database.createNewFile(self.fileName, chunkHandle)

        # If the return flag was an error flag, alert the logger and API of the error
        if createFileFlag == -1:
            logging.error("Got a duplicate file name, sending FAIL to API")
            fL.send(self.s, "FAIL1")
            return -1

        elif createFileFlag == -2:
            logging.error("No file exists for a chunk to be created for")
            fL.send(self.s, "FAIL2")

        elif createFileFlag == -3:
            logging.error("Chunk is not the latest chunk. New chunk has been created that can be appended to.")
            fL.send(self.s, "FAIL3")

            # Get the locations for a specified chunk
        locations = database.data[self.fileName].chunks[chunkHandle].locations

        # Parse the locations list retreived above into a pipe-separated list
        hosts = ""
        for item in locations:
            hosts += item + "|"

            # Visual confirmation for debugging: confirm success of create()
        logging.debug("Chunk metadata successfully created")

        try:
            # Send the API a string containing the location and chunkHandle information
            fL.send(self.s, str(hosts) + str(chunkHandle))

        except socket.error:
            logging.warning("Socket Connection Broken")
            # Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle
        logging.debug("SENT ==> " + str(hosts) + str(chunkHandle))

        # Receieve an ack to affirm that the chunk was successfully created
        ack = fL.recv(self.s)

        if ack == "CREATED":
            logging.debug("successful creation")
        elif ack == "FAILED":
            logging.error("unsuccessful creation")
        else:
            logging.error("unknown ack")
コード例 #17
0
ファイル: heartBeat.py プロジェクト: BenningtonCS/GFS
        def heartBeat(self, IP):
        	logging.debug('HEARTBEAT: Initiating heartBeat protocol')
                # Get the list of all active chunk server IPs
                activeServers = self.getActiveChunkServers()

                try:
                        # Create a TCP socket instance
                        self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			# Set the timeout of the socket
                        self.s.settimeout(self.SOCK_TIMEOUT)
			# Allow the socket to re-use address
                        self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
			# Connect to the chunkserver over the specified port
                        self.s.connect((IP, self.PORT))
			# Send the chunk server a heart (ping)
                        fL.send(self.s, "<3?")
			# Get the chunk server response
                        data = fL.recv(self.s)
                        print data
			# Close the connection to allow future connections
                        self.s.close()
                        # If the chunk server responds with a heart, add it to activehosts
                        if data == "<3!":
                                # If the chunkserver IP is not in the activeServers list, add it to the list
                                if IP not in activeServers:
                                        with open(self.AHOSTS, "a") as file:
                                                file.write(IP + "\n")
                        # Return a 1 so any script calling heartBeat will know the heartbeat was successful
                        return 1
		# Handle the timeout (chunk server alive but not responding) and connection (server dead) errors
		except (socket.timeout, socket.error):
			print "</3"
                        logging.debug('HEARTBEAT: Could not connect to ' + IP)
			# Check to see if the chunkserver is in the list of active IPs
                        if IP in activeServers:
				# If it is, remove it from the list
                                activeServers.remove(IP)
        			# Clear the previous activehosts.txt file and replace it with the list of active servers, 
        			# which now excludes the failed chunkserver
                                with open(self.AHOSTS, "w") as file:
                                        newList = ""
                                        for item in activeServers:
                                                newList += item + "\n"
                                        file.write(newList)
                        # Return a -1 so any script calling heartbeat will know the heartbeat failed.
                        return -1
コード例 #18
0
ファイル: API.py プロジェクト: BenningtonCS/GFS
	def fileList(self):
		# Create socket connection to the master
		m = self.createSocket()
		# if there was an error making the socket, exit the function
		if not m:
			return 0

		try:
			# Send the master a request for the list of files the system contains
			fL.send(m, "FILELIST|x")
			# Recieve the file list from the master
			data = fL.recv(m)
			# Close the TCP connection to the master
			m.close()
			return data
		except Exception as e:
			raise e
			logging.error("Unable to get file list from master.")
			return 0
コード例 #19
0
    def fileList(self):
        # Create socket connection to the master
        m = self.createSocket()
        # if there was an error making the socket, exit the function
        if not m:
            return 0

        try:
            # Send the master a request for the list of files the system contains
            fL.send(m, "FILELIST|x")
            # Recieve the file list from the master
            data = fL.recv(m)
            # Close the TCP connection to the master
            m.close()
            return data
        except Exception as e:
            raise e
            logging.error("Unable to get file list from master.")
            return 0
コード例 #20
0
ファイル: scrubber.py プロジェクト: ttanay/GFS
	def cleanLocation(self, location, handle):
		# Connect to the chunk server
		self.connect(location, 0)
		try:
			# Send the chunk server a SANITIZE message with the chunk handle
			# so it knows which chunk it is deleting
			fL.send(self.s, 'SANITIZE|' + str(handle))
			logging.debug('SCRUBBER: Sent SANITIZE message to chunkserver')
			# Wait for a response back from the chunk server to verify that
			# the chunk was removed
			data = fL.recv(self.s)
			logging.debug('SCRUBBER: Received response from chunkserver')
			return data
		except (socket.timeout, socket.error) as e:
			logging.error("SCRUBBER: Connection to chunkserver failed with error: " + str(e) + ". Unable to continue for location: " + str(location))
			# Set data to be empty so the deletion process will not continue to 
			# try and delete a file it received no information about.
			data = ""
			return data
コード例 #21
0
    def undelete(self, filename):
        # Create socket connection to the master
        m = self.createSocket()
        # if there was an error making the socket, exit the function
        if not m:
            return 0

        #send UNDELETE request to master
        try:
            fL.send(m, "UNDELETE|" + filename)
        except:
            print "ERROR COULD NOT SEND UNDELETE REQUEST TO MASTER"
        #receive acks from the master
        self.data = fL.recv(m)
        #tell the user whether the file was successfully unmarked or not
        if self.data == "FAILED1":
            print "ERROR: COULD NOT UNDELETE FILE"
            return -1
        elif self.data == "FAILED2":
            print "File was not flagged for deletion."
            return -2
        elif self.data == "MARKED":
            print "File successfully unmarked for deletion."
        m.close()
コード例 #22
0
ファイル: API.py プロジェクト: BenningtonCS/GFS
	def undelete(self, filename):
		# Create socket connection to the master
		m = self.createSocket()
		# if there was an error making the socket, exit the function
		if not m:
			return 0

		#send UNDELETE request to master
		try:
			fL.send(m, "UNDELETE|" + filename)
		except:
			print "ERROR COULD NOT SEND UNDELETE REQUEST TO MASTER"
		#receive acks from the master
		self.data = fL.recv(m)
		#tell the user whether the file was successfully unmarked or not
		if self.data == "FAILED1":
			print "ERROR: COULD NOT UNDELETE FILE"
			return -1
		elif self.data == "FAILED2":
			print "File was not flagged for deletion."
			return -2
		elif self.data == "MARKED":
			print "File successfully unmarked for deletion."
		m.close()
コード例 #23
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
    def undelete(self):
        logging.debug("Begin updating delete flag to False")

        # Make sure the file is already marked for delete
        if self.fileName in database.toDelete:

            try:
                # Change the delete flag for the specified file
                database.flagUndelete(self.fileName)
                # Confirm that file has been marked for deletion
                fL.send(self.s, "MARKED")

                logging.debug("Delete Flags Updated")

            except:
                logging.error("File could not be unmarked for deletion")
                fL.send(self.s, "FAILED1")

                # If the file is not already marked for delete, you can't undelete it..
        else:
            logging.debug("The file, " + self.fileName + ", was not marked for deletion to begin with.")
            fL.send(self.s, "FAILED2")
コード例 #24
0
    def undelete(self):
        logging.debug('Begin updating delete flag to False')

        # Make sure the file is already marked for delete
        if self.fileName in database.toDelete:

            try:
                # Change the delete flag for the specified file
                database.flagUndelete(self.fileName)
                # Confirm that file has been marked for deletion
                fL.send(self.s, "MARKED")

                logging.debug('Delete Flags Updated')

            except:
                logging.error("File could not be unmarked for deletion")
                fL.send(self.s, "FAILED1")

        # If the file is not already marked for delete, you can't undelete it..
        else:
            logging.debug('The file, ' + self.fileName +
                          ', was not marked for deletion to begin with.')
            fL.send(self.s, "FAILED2")
コード例 #25
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
 def getAllChunks(self):
     chunks = database.allChunks(self.fileName)
     fL.send(self.s, chunks)
コード例 #26
0
 def fileNames(self):
     names = str(database.getFileNames())
     fL.send(self.s, names)
コード例 #27
0
import socket, config
import functionLibrary as fL

port = int(raw_input('What port to connect over? : '))

# Define a socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Allow socket address reuse
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Connect to the server over the specified IP and port
s.connect(('', port))

message = str(raw_input('What message would you like to send? : '))
ending = str(raw_input('Include proper EOT character? (y/n): ')).lower()

# Send the message with a proper EOT termination
if ending == "y":
    fL.send(s, message)

# Send the message without the proper EOT termination
elif ending == "n":
    s.send(message)

# Receive data back from the server
data = fL.recv(s)

# Print the data to console
print "DATA == ", data

s.close()
コード例 #28
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
 def createChunk(self):
     self.lock.acquire()
     chunkHandle = database.getChunkHandle()
     self.lock.release()
     newChunk = database.createNewChunk(self.msg[1], self.msg[2], chunkHandle)
     fL.send(self.s, newChunk)
コード例 #29
0
ファイル: timingTests.py プロジェクト: BenningtonCS/GFS
	#try:
	# Create a TCP socket instance
	s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
	# Set the timeout of the socket
	s.settimeout(3)
	# Allow the socket to re-use address
	s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
	# Connect to the chunkserver over the specified port
	s.connect((config.masterip, config.port))
	#except:
	#	print "CONNECTION ERROR"
	#	errorList.append('SCRUBBER - Unable to connect to master')
	print "connected"

	fL.send(s, "GETDELETEDATA|*")

	data = fL.recv(s)
	
	s.close()
	print "recvd to delete list"
	toDelete = []

	for item in data.split("|"):
		if item != "":
			toDelete.append(item)

	# Create an instance of the Scrubber object, and initiate it.
	scrub = scrubber.Scrubber(toDelete)

	scrubStart = time.time()
コード例 #30
0
ファイル: serverEmulator.py プロジェクト: ttanay/GFS
# Listen for client connections
s.listen(1)
print "Listening..."
# Accept the incoming connection
conn, addr = s.accept()
print "accepted connection"
# Receive the data
data = fL.recv(conn)

# If data was received
if data:
    print data
    # If the option to send back an eot was yes, send the message
    # with the eot
    if ending == "y":
        fL.send(conn, "Received successfully")
    # If the option to send back an eot was no, send the message
    # without the eot
    elif ending == "n":
        conn.send("Received successfully")

# If no data was received
if not data:
    print "No data received!"
    # If the option to send back an eot was yes, send the message
    # with the eot
    if ending == "y":
        fL.send(conn, "Received unsuccessfully")
    # If the option to send back an eot was no, send the message
    # without the eot
    elif ending == "n":
コード例 #31
0
ファイル: chunkserver.py プロジェクト: ttanay/GFS
    def run(self):
        c = fL.recv(self.connection)  # listens for a command on
        # the connection handed
        # down from the main
        # thread
        com = c.split(
            '|')  # if c has multiple parts, they will be seperated by
        # pipes. This will put each part into a list
        command = com[
            0]  # the command should be the first part of the message,
        # thus the first part of the list
        logging.debug("Recieved command " + command)

        # next the distributor hands the connection to the appropriate
        # worker based on the command given, invalid commands simply fall
        # through

        if command == "<3?":
            # heartbeat to see if each chunkserver is running. If it is, it will
            # send back a confirmation of <3!
            try:
                # in this and each other if/elif statement the correct
                # worker thread is started for a given command
                fL.send(self.connection, "<3!")
                logging.debug("Send heart beat back")
                self.connection.close()
                logging.debug("Closed connection.")
            except socket.error as e:
                logging.error(e)

        elif command == "CHUNKSPACE?":
            try:
                #				fL.send(self.connection, "CONTINUE") # after receiving the connection
                #								 # the thread confirms that it is
                #								 # ready to receive arguments
                #				logging.debug("send a continue")
                #				chunkHandle = fL.recv(self.connection) # it listens on its
                #									 # connection for a chunkhandle
                chunkHandle = com[1]  # name of the chunkhandle
                logging.debug("recieved name of the chunkhandle: " +
                              chunkHandle)
                emptySpace = str(
                    mg64 - os.stat(chunkPath + "/" +
                                   chunkHandle).st_size)  # then checks the
                # difference
                # between the
                # file's size and
                # 64mg (the max
                # chunk size)
                fL.send(self.connection,
                        str(emptySpace))  # and returns the amount of space
                # left to the API
                logging.debug("Send the space remaining")


#				self.connection.close() # closes the connection
#				logging.debug("Closed the connection")
            except socket.error as e:
                logging.error(e)
            except IOError as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)
            except Exception as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)

        elif command == "READ":
            # read data from a chunk
            try:
                #				fL.send(self.connection, "CONTINUE") # confirms readiness for data
                #				logging.debug("sent continue #1")
                #				chunkHandle = fL.recv(self.connection) # listens for chunkHandle
                chunkHandle = com[1]
                #				logging.debug("recieved name of the chunkhandle: " + chunkHandle)
                #				fL.send(self.connection, "CONTINUE") # confirms ready state
                #				logging.debug("sent continue #2")
                byteOffSet = int(com[2])  # listens for a byte
                # offset to read from
                # (relative to the
                # beginning of the
                # given chunk)
                #				byteOffSet
                #				logging.debug("recieved the byte offset number.")
                #				fL.send(self.connection, "CONTINUE") # confirms the desire for EVEN MORE data
                #				logging.debug("sent continue #3")
                bytesToRead = int(
                    com[3])  #int(fL.recv(self.connection)) # listens for the
                # number of bytes to read
                logging.debug("recieved the number of bytes to read")
                chunk = open(
                    chunkPath + "/" +
                    chunkHandle)  # opens the designated chunk to read from
                chunk.seek(
                    int(byteOffSet))  # goes to the specified byte offset
                fileContent = chunk.read(
                    bytesToRead)  # stuffs all the stuff to be
                # read into a variable
                fL.send(self.connection, fileContent)
                logging.debug("send the file content")
                chunk.close()  # closes the chunk
                logging.debug("closed the connection")
                self.connection.close()  # closes the connection
            except socket.error as e:
                logging.error(e)
            except IOError as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)
            except Exception as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)

        elif command == "CONTENTS?":
            try:
                output = ""
                for files in os.walk(chunkPath):  # read every file
                    output = str('|'.join(
                        item
                        for item in files[-1]))  # turn the list into a string
                    print output
                if output == "":  # if there is nothing in the dir
                    print "output is empty"
                    fL.send(self.connection, " ")  # send an empty string
                    logging.debug(
                        "Sent an empty string which should be the output")
                else:  # otherwise
                    print "output is not empty"
                    fL.send(self.connection,
                            output)  # send everything as a string
                    logging.debug("sent the output")
            except socket.error as e:
                logging.error(e)
            except IOError as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)
            except Exception as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)

        elif command == "CREATE":
            # create a new chunk
            try:
                #				fL.send(self.connection, "CONTINUE")
                #				logging.debug("Sent continue")
                chunkHandle = com[
                    1]  #fL.recv(self.connection) # get the name of the chunk
                logging.debug("recieved name of the chunk")
                open(chunkPath + "/" + chunkHandle,
                     'w').close()  # create the file
            except IOError as e:
                logging.error(e)
                fL.send(self.connection, "FAILED")
            except Exception as e:
                logging.error(e)
                fL.send(self.connection, "FAILED")
            else:
                fL.send(self.connection, "CREATED")

        elif command == "APPEND":
            # append new data to a chunk
            try:
                #				fL.send(self.connection, "CONTINUE")
                #				logging.debug("sent continue #1")
                #				chunkHandle = fL.recv(self.connection) # name of the chunk
                chunkHandle = com[1]
                #				logging.debug("Recieved name of the chunk")
                #				fL.send(self.connection, "CONTINUE")
                #				logging.debug("Sent continue #2")
                #				data = fL.recv(self.connection)    # data being added
                data = "|".join(com[2:])
                length = str(len(data))
                logging.error(length)
                logging.debug("Recieved the data to be added")
                with open(chunkPath + "/" + chunkHandle,
                          'a') as a:  # open the chunk
                    #for item in data:
                    a.write(data)  # add the data to it
            except socket.error as e:
                logging.error(e)
            except IOError as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)
            except Exception as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)
            else:
                fL.send(self.connection, "SUCCESS")

        elif command == "SANITIZE":
            # recieves SANITIZE from the scrubber which tells the chunkserver to delete a chunk
            try:
                chunkHandle = com[1]  # name of the chunk
                logging.debug("Recieved name of the chunk")
                try:
                    os.remove(chunkPath + '/' +
                              chunkHandle)  # remove file from directory
                    logging.debug("Removed chunk handle.")
                # If there is an error thrown that the chunk does not exist, we return success
                # because sanitize is called to remove a chunk from a location. If it does not
                # exist at that location, then removal is technially achieved.
                except OSError:
                    fL.send(self.connection, "SUCCESS")
                    logging.debug(
                        "chunk already did not exist. sending success")

                fL.send(self.connection, "SUCCESS")  # send a success
                logging.debug("removal successfull!")

            except socket.error as e:
                logging.error(e)
            except IOError as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)
            except Exception as e:
                fL.send(self.connection, "FAILED")
                logging.error(e)

        else:
            error = "Received invalid command: " + command
            logging.error(error)
コード例 #32
0
    def create(self, filename):

        logging.debug("API: Starting create function.")

        #return an error if some wise guy tries to put a pipe in the file name.
        if "|" in filename:
            print "Invalid character, '|', in filename. No action taken."
            return 0

        logging.debug("API: Creating socket.")

        # Create socket connection to the master
        m = self.createSocket()
        # if there was an error making the socket, exit the function
        if not m:
            return 0

        #send a CREATE request to the master
        try:
            logging.debug("API: Attempting to send CREATE| " + filename)
            fL.send(m, "CREATE|" + filename)
        except:
            logging.error("ERROR: COULD NOT SEND CREATE REQUEST TO MASTER")
        #receive data back from the master
        self.data = fL.recv(m)
        #error if the file trying to be created already exists
        logging.debug("API: Received message: " + self.data)
        if self.data == "FAIL1":
            print "THAT FILE EXISTS ALREADY... EXITING API"
            return 0
        elif self.data == "FAIL2":
            print "NO SUCH FILE EXISTS FOR CHUNK CREATION"
            return 0
        elif self.data == "FAIL3":
            print "CHUNK IS NOT THE LATEST CHUNK"
            return 0

        #parse the received data into locations, and chunk handle
        self.splitdata = self.data.split("|")
        dataLength = len(self.splitdata)
        chunkHandle = self.splitdata[-1]
        global ack
        logging.debug("API: about to begin for loop, " + str(dataLength - 1) +
                      "iterations")
        #iterate through each IP address received from the master
        for n in range(dataLength - 1):
            logging.debug("API: For loop, iteration number " + str(n))
            #designate the IP for this iteration
            location = self.splitdata[n]
            #create a socket to be used to connect to chunk server
            s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            #attempt to connect to the chunk server at the current location
            try:
                s.connect((location, self.TCP_PORT))
            except:
                logging.error("ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ",
                              location)
                continue
            #send CREATE request to the chunk server at the current location
            fL.send(s, "CREATE|" + chunkHandle)
            #wait to receive a CONTINUE from chunk server to proceed
            ack = fL.recv(s)
            #close connection to current chunk server.
            s.close()

        if ack == "FAILED":
            print "ERROR: FILE CREATION FAILED"
            fL.send(m, "FAILED")
        elif ack == "CREATED":
            print "File creation successful!"
            fL.send(m, "CREATED")
            return 1
        m.close()
コード例 #33
0
ファイル: testCodeForAPI.py プロジェクト: BenningtonCS/GFS
TCP_PORT2 = 9666

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((TCP_ADDRESS, TCP_PORT1))

s.listen(1)

while 1:
	conn, addr = s.accept()
	data = fL.recv(conn)
	print data
	splitdata = data.split("|")
	#s.connect((API_ADDRESS, TCP_PORT2))
	if splitdata[0] == "CREATE":
		fL.send(conn, "10.10.117.104|01")
	elif splitdata[0] == "APPEND":
		fL.send(conn, "10.10.117.104|01")
	elif splitdata[0] == "READ":
		fL.send(conn, "READ|10.10.117.104*01*0")
	elif (splitdata[1] == 01):
		fL.send(conn, "poop")
	elif splitdata[0] == "CHUNKSPACE?":
		fL.send(conn, "10")
	elif splitdata[0] == "CREATECHUNK":
		fL.send(conn, "10.10.117.104|01")
	elif splitdata[0] == "DELETE":
		fL.send(conn, "MARKED")
	elif splitdata[0] == "UNDELETE":
		fL.send(conn, "MARKED")
	
コード例 #34
0
ファイル: API.py プロジェクト: BenningtonCS/GFS
	def create(self,filename):

		logging.debug("API: Starting create function.")

		#return an error if some wise guy tries to put a pipe in the file name.
		if "|" in filename:
			print "Invalid character, '|', in filename. No action taken."
			return 0


		logging.debug("API: Creating socket.")

		# Create socket connection to the master
		m = self.createSocket()
		# if there was an error making the socket, exit the function
		if not m:
			return 0

		#send a CREATE request to the master
		try:
			logging.debug("API: Attempting to send CREATE| " + filename)
			fL.send(m, "CREATE|" + filename)
		except: 
			logging.error("ERROR: COULD NOT SEND CREATE REQUEST TO MASTER")
		#receive data back from the master 
		self.data = fL.recv(m)
		#error if the file trying to be created already exists 
		logging.debug("API: Received message: " + self.data)
		if self.data == "FAIL1":
			print "THAT FILE EXISTS ALREADY... EXITING API"
			return 0
		elif self.data == "FAIL2":
			print "NO SUCH FILE EXISTS FOR CHUNK CREATION"
			return 0
		elif self.data == "FAIL3":
			print "CHUNK IS NOT THE LATEST CHUNK"
			return 0

		#parse the received data into locations, and chunk handle
		self.splitdata = self.data.split("|")
		dataLength = len(self.splitdata)
		chunkHandle = self.splitdata[-1]
		global ack
		logging.debug("API: about to begin for loop, " + str(dataLength -1) + "iterations")
		#iterate through each IP address received from the master
		for n in range(dataLength-1):
			logging.debug("API: For loop, iteration number " + str(n))
			#designate the IP for this iteration
			location = self.splitdata[n]
			#create a socket to be used to connect to chunk server
			s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			#attempt to connect to the chunk server at the current location
			try:
				s.connect((location,self.TCP_PORT))
			except: 
				logging.error("ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location)
				continue
			#send CREATE request to the chunk server at the current location
			fL.send(s, "CREATE|" + chunkHandle)
			#wait to receive a CONTINUE from chunk server to proceed
			ack = fL.recv(s)
			#close connection to current chunk server.
			s.close()

		if ack == "FAILED":
			print "ERROR: FILE CREATION FAILED"
			fL.send(m, "FAILED")
		elif ack == "CREATED":
			print "File creation successful!"
			fL.send(m, "CREATED")
			return 1
		m.close()
コード例 #35
0
    def interrogateChunkServer(self, IP, retry):
        logging.debug('Initialize interrogateChunkServer()')

        # Create an instance of a TCP socket
        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        # Define the variable that will hold the the data received from the chunkservers
        data = " "
        try:
            s.connect((IP, chunkPort))
            logging.debug('Connection Established: ' + str(IP) + ' on port ' +
                          str(chunkPort))
            # Request the chunk contents of the specified chunkserver
            fL.send(s, 'CONTENTS?')
            logging.debug('Sent chunkserver a CONTENTS? message')
            # Received the chunk contents of the speicified chunkserver
            data = fL.recv(s)
            # Close the socket connection
            s.close()

        # If the database is unable to connect to the chunkservers, retry the connection
        # If the retry fails, remove it from the list of activehosts and move on.
        except:
            # Make sure the socket connection is closed before doing anything, so if
            # a retry occurs, the socket will not already be in use.
            s.close()
            # heartBeat the chunkserver. If that indicates it is alive, try to
            # interrogate the chunkserver again. If not, remove if from the active
            # hosts list and move on.
            if retry < 3:
                # It the heartBeat indicated the chunkserver is still alive, try again.
                if hB.heartBeat(IP) == 1:
                    self.intterogateChunkServer(IP, retry + 1)
                    logging.debug(
                        'Retry connect to chunkserver for interrogation')
                else:
                    self.remFromAhosts(IP)
                    logging.warning(
                        'Heartbeat indicates chunkserver dead. Not interrogating, moving on.'
                    )
                    return -1

            else:
                self.remFromAhosts(IP)
                # Log the fact that we were unable to connect to a chunkserver
                logging.error("interrogateChunkServer failed to connect to " +
                              IP)
                return -1

        logging.debug('Received response from chunkserver')

        # If the IP is not already in the location lookup, add it!
        if IP not in self.locDict.keys():
            self.locDict[IP] = []

        # If the chunkserver has nothing on it, it should return whitespace. If this is the case,
        # then nothing in the database can be updated, so it will continue onto the next IP.
        # If the chunkserver returns something other than whitespace (a message formatted chunk1|chunk2|... )
        # then the data can be processed.
        if data != " ":
            # Convert the pipe separated string into a list
            chunkData = data.split('|')
            # In the event that data is formatted poorly with additional | characters, we want to get rid of null elements
            chunkData = filter(None, chunkData)

            # For every chunk handle in that list, update that chunk objects locations list
            for chunk in chunkData:

                # If the IP key is not already in the location lookup, add it!
                if IP not in self.locDict.keys():
                    self.locDict[IP] = []
                    # Add the chunk to the list of values for the IP key
                    self.locDict[IP].append(chunk)

                # If the location does already exist, append the current chunk to its list of chunk values
                else:
                    assChnks = self.locDict[IP]
                    # But first, make sure that chunk isn't already in the values, so you don't get
                    # multiple copies of the same chunk in the lookup.
                    if chunk not in assChnks:
                        # Add the chunk to the list of values for the IP key
                        self.locDict[IP].append(chunk)

                try:
                    # Find which file the chunk is associated with in the lookup dictionary
                    fileName = self.lookup[chunk]

                    # From the file name we found the chunk to be associated with in the
                    # lookup, we can append the current IP to the list of chunk locations
                    # in the chunk object within the file object dictionary.
                    if IP not in self.data[fileName].chunks[chunk].locations:
                        self.data[fileName].chunks[chunk].locations.append(IP)
                        logging.debug('Appended location to chunk ' +
                                      str(chunk) + ' location list')

                # If the chunk is not recognized in the master's database, the chunk is an orphan (does not
                # belong to a file). In this case, the chunk should be removed.
                except KeyError:
                    # Create an instance of a TCP socket
                    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                    s.connect((IP, chunkPort))
                    logging.debug('Connection Established: ' + str(IP) +
                                  ' on port ' + str(chunkPort))
                    # Tell the chunkserver to sanitize the chunk
                    fL.send(s, 'SANITIZE|' + str(chunk))
                    logging.debug('Sent chunkserver a SANITIZE message')
                    # Received the chunk contents of the speicified chunkserver
                    data = fL.recv(s)

                    if data == "SUCCESS":
                        logging.debug("Orphan removal successful")
                        self.locDict[IP].remove(chunk)
                    elif data == "FAILED":
                        logging.debug("Orphan removal failed.")
                    # Close the socket connection
                    s.close()

        logging.debug('interrogateChunkServer() complete')
コード例 #36
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
 def fileList(self):
     # call the database object's returnData method
     list = str(database.getFiles())
     fL.send(self.s, list)
コード例 #37
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
 def fileNames(self):
     names = str(database.getFileNames())
     fL.send(self.s, names)
コード例 #38
0
    def append(self, filename, newData, flag):
        # Create socket connection to the master
        m = self.createSocket()
        # if there was an error making the socket, exit the function
        if not m:
            return 0

        #send APPEND request to master
        try:
            fL.send(m, "APPEND|" + filename)
        except:
            print "COULD NOT SEND APPEND REQUEST TO MASTER"
            return 0
        #receive data back from master
        self.data = fL.recv(m)
        #close connection to master
        m.close()
        #some error handling
        if (self.data == "FAILED"):
            print "ERROR: MASTER SENT FAIL MESSAGE exiting..."
            return 0
        elif (self.data == "OPEN"):
            print "ERROR: FILE " + filename + " ALREADY OPEN"
            return 0
        #parse the data into useful parts
        self.splitdata = self.data.split("|")
        dataLength = len(self.splitdata)
        cH = self.splitdata[-1]
        #get length of the requested new data to use for append across chunks
        if flag == 1:
            with open(newData, "rb") as da:
                newData = da.read()

        dataSize = len(newData)

        lenNewData = int(dataSize)

        for n in range(0, dataLength - 1):
            location = self.splitdata[n]
            #create socket to connect to chunk server at location
            self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            #attempt to connect to chunk server at location
            try:
                self.s.connect((location, self.TCP_PORT))
            except:
                print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location
            #ask chunk server how much room is left on latest chunk
            fL.send(self.s, "CHUNKSPACE?|" + cH)
            #the response is stored in remainingSpace
            remainingSpace = fL.recv(self.s)
            self.s.close()
            #some error handling
            if remainingSpace == "FAILED":
                print "CHUNKSPACE REQUEST FAILED. exiting..."
                return 0
            #make remainingSpace an integer
            remainingSpace = int(remainingSpace)

            #if the length of the new data is greater than the room left in the chunk...
            if (lenNewData > remainingSpace):
                #...split the data into two parts, the first part being equal to the
                #amount of room left in the current chunk. the second part being the
                #rest of the data.
                cut = remainingSpace
                newData1 = newData[0:cut]
                print "Sending data of length:" + str(len(newData1))
                newData2 = newData[cut:]
                #tell the chunk server to append the first part of the new data that
                #will fill up the rest of the remaining space on a chunk
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                try:
                    s.connect((location, self.TCP_PORT))
                except:
                    print "ERROR: COULD NOT REOPEN SOCKET"
                fL.send(s, "APPEND|" + cH + "|" + newData1)
                print "first append"
                SoF = fL.recv(s)
                #close connection to chunk server
                s.close()
                #error handling
                if SoF == "FAILED":
                    print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..."
                    return 0

            elif (lenNewData <= remainingSpace):
                t = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                t.connect((location, self.TCP_PORT))
                try:
                    fL.send(t, "APPEND|" + cH + "|" + newData)
                except:
                    print "ERROR: COULD NOT SEND APPEND TO CHUNK SERVER"
                SoF = fL.recv(t)
                t.close()
                #error handling/acks
                if SoF == "FAILED":
                    print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..."
                    return 0

        ###################
        if lenNewData > remainingSpace:
            # Create socket connection to the master
            m = self.createSocket()
            # if there was an error making the socket, exit the function
            if not m:
                return 0

            #tell the master to create a new chunk for the remaining data
            try:
                fL.send(m, "CREATECHUNK|" + filename + "|" + cH)
            except:
                print "ERROR: COULD NOT CREATE NEW CHUNK TO APPEND TO"
            #receive data back from master
            cData = fL.recv(m)
            #parse this data and handle it very similarly as the in the create function
            if self.data == "FAIL2":
                print "NO SUCH FILE EXISTS FOR CHUNK CREATION"
                exit(0)
            splitcData = cData.split("|")
            cDataLength = len(splitcData)
            cH = splitcData[-1]
            #close the connection to the master so we can connect to the chunk servers
            m.close()
            #iterate through each IP address received from the master
            for n in range(0, cDataLength - 1):
                #create a socket to be used to connect to chunk server
                s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                #designate the IP for this iteration
                location = splitcData[n]
                print location
                #attempt to connect to the chunk server at the current location
                try:
                    s.connect((location, self.TCP_PORT))
                except:
                    print "ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location
                    continue
                    #send CREATE request to the chunk server at the current location
                    fL.send(s, "CREATE|" + cH)
                global ack
                ack = fL.recv(s)
                ################
                #close connection to current chunk server.
                s.close()
                #do some acks
                if ack == "FAILED":
                    print "ERROR: CHUNK CREATION FAILED"
                    #fL.send(m, "FAILED")
                elif ack == "CREATED":
                    print "Chunk creation successful!"
                    #fL.send(m, "CREATED")
                #m.close()

            #now that the new chunk has been created on all of the servers...
            #...run append again with the second part of the new data
            #self.s.close()
            try:
                self.append(filename, newData2, False)
            except UnboundLocalError:
                pass
        return 1
コード例 #39
0
	def run(self):
		
		if self.role == 1:
			print "1"
			self.socket.connect((self.remoteAddr,self.port))
			print "2"
			fL.send(self.socket, "<3?")
			print "3"
			print fL.recv(self.socket)
			print "4"

		elif self.role == 2:
			self.socket.connect((self.remoteAddr,self.port))
			fL.send(self.socket, "CHUNKSPACE?|"+chunkHandle)
			print "chunkspace :" + fL.recv(self.socket)

		elif self.role == 3:
			self.socket.connect((self.remoteAddr,self.port))
			fL.send(self.socket, "READ|"+chunkHandle+"|"+byteOffSet+"|"+bytesToRead)
			print "Reading.... " + fL.recv(self.socket)


		elif self.role == 4:
			self.socket.connect((self.remoteAddr,self.port))
			fL.send(self.socket, "CONTENTS?")
			print "contents :" + fL.recv(self.socket)

		elif self.role == 5:
			self.socket.connect((self.remoteAddr,self.port))
			fL.send(self.socket, "CREATE|"+chunkHandle)
			print fL.recv(self.socket)


		elif self.role == 6:
			self.socket.connect((self.remoteAddr,self.port))
			fL.send(self.socket, "APPEND|"+chunkHandle+"|"+appendData)

		elif self.role == 7:
			self.socket.connect((self.remoteAddr,self.port))
			fL.send(self.socket, "SANITIZE|"+chunkHandle)
			print fL.recv(self.socket)
コード例 #40
0
    def read(self):
        # Get the byte offset and bytes to read from the received message
        try:
            byteOffset = int(self.msg[2])
            bytesToRead = int(self.msg[3])
        # If there is an index error (the values do not exist in the message)
        # alert the client and end the read function.
        except IndexError:
            fL.send(self.s, "READ command not given proper parameters")
            return

        logging.debug('parsed byte offset and bytes to read')

        # Get the size of a chunk from the config file
        maxChunkSize = config.chunkSize
        # Find the sequence of the chunk in the given file by using integer division
        # (divide and take the floor)
        startSequence = byteOffset // maxChunkSize

        if startSequence > len(database.data[self.fileName].chunks.keys()):
            logging.debug('No such byte offset exists for the given file')
            fL.send(self.s, "FAILED, NO SUCH BYTE OFFSET EXISTS FOR THIS FILE")
            return

        # Get the offset of the read-start within its given chunk
        chunkByteOffset = byteOffset % maxChunkSize

        logging.debug('start sequence # == ' + str(startSequence))
        logging.debug('chunk byte offset == ' + str(chunkByteOffset))

        # If the user inputs a bytes to read of -1, the read will go until the end
        # of the file.
        if bytesToRead == -1:
            # The ending offset will be the max file size
            endOffset = maxChunkSize
            # The end sequence can be found be knowing how many chunks a file has, and
            # subtracting by 1 because the sequence numbers start at 0, not 1.
            endSequence = len(database.data[self.fileName].chunks.keys()) - 1

        else:
            # To find where the user wants to read ending, add the number of bytes they want
            # to read to their starting point, the byteOffest. This will give the byte offset
            # of the end of the read
            endReadOffset = byteOffset + bytesToRead

            # Find the sequence of the chunk in which the end of the read will terminate
            endSequence = endReadOffset // maxChunkSize

            # Get the offset of the read-end within its given chunk
            endOffset = endReadOffset % maxChunkSize

        logging.debug('end sequence # == ' + str(endSequence))
        logging.debug('end read offset == ' + str(endOffset))

        # Create an empty string to hold the message that will be sent to the client
        responseMessage = "READFROM"

        # For each sequence number that exists between (and including) the read-start chunk
        # and the read-end chunk, get the file's chunk with the appropriate sequence number,
        # and append to the response message, a location it is stored at, its chunk handle,
        # the byte offset from within that chunk to begin reading from, and the byte offset
        # to end reading from
        for sequence in range(startSequence, (endSequence + 1)):
            #	try:
            # Get the chunkHandles associated with the given file, and sort the chunkHandles from
            # least to greatest in the list. This will put them in their sequence order where the
            # 0th element is now the 0th sequence, 1st element the 1st sequence, etc.
            logging.debug(sorted(database.data[self.fileName].chunks.keys()))
            associatedChunkHandles = sorted(
                database.data[self.fileName].chunks.keys())

            # Append a location of where the start-sequence chunk is stored to the message
            logging.debug(database.data[self.fileName].chunks[
                associatedChunkHandles[sequence]].locations)
            responseMessage += "|" + str(database.data[self.fileName].chunks[
                associatedChunkHandles[sequence]].locations[0])

            # Append the chunk handle to the message
            responseMessage += "*" + str(associatedChunkHandles[sequence])

            # Append the byte offset to start reading from to the message
            responseMessage += "*" + str(chunkByteOffset)

            # If there are multiple chunks that will be read over, the next chunk will start
            # the read from the beginning
            chunkByteOffset = 0

            # Check to see if the READ will take place in the same chunk. If it does, append the
            # endOffset to the message so the client will know where to end reading
            if startSequence == endSequence:
                responseMessage += "*" + str(endOffset)
            # If the READ takes place over multiple chunks, write the end of read for the current
            # chunk to be the end of the chunk, and then increase the start sequence number so when the
            # metadata for the last chunk is processed, it will be caught by the if statement above
            # and send the appropriate ending offset.
            elif startSequence < endSequence:
                responseMessage += "*" + str(maxChunkSize)
                startSequence += 1

        #	except:
        #		logging.error("Unable to generate proper READ response message.")

        logging.debug('RESPONSE MESSAGE == ' + str(responseMessage))
        #send our message
        fL.send(self.s, responseMessage)
        logging.debug('SENT == ' + responseMessage)
        # Visual confirmation for debugging: confirm success of read()
        logging.debug('Read successfully handled')
コード例 #41
0
TCP_PORT2 = 9666

s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((TCP_ADDRESS, TCP_PORT1))

s.listen(1)

while 1:
    conn, addr = s.accept()
    data = fL.recv(conn)
    print data
    splitdata = data.split("|")
    #s.connect((API_ADDRESS, TCP_PORT2))
    if splitdata[0] == "CREATE":
        fL.send(conn, "10.10.117.104|01")
    elif splitdata[0] == "APPEND":
        fL.send(conn, "10.10.117.104|01")
    elif splitdata[0] == "READ":
        fL.send(conn, "READ|10.10.117.104*01*0")
    elif (splitdata[1] == 01):
        fL.send(conn, "poop")
    elif splitdata[0] == "CHUNKSPACE?":
        fL.send(conn, "10")
    elif splitdata[0] == "CREATECHUNK":
        fL.send(conn, "10.10.117.104|01")
    elif splitdata[0] == "DELETE":
        fL.send(conn, "MARKED")
    elif splitdata[0] == "UNDELETE":
        fL.send(conn, "MARKED")
コード例 #42
0
ファイル: database.py プロジェクト: BenningtonCS/GFS
	def interrogateChunkServer(self, IP, retry):
		logging.debug('Initialize interrogateChunkServer()')

		# Create an instance of a TCP socket
		s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
		s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
		# Define the variable that will hold the the data received from the chunkservers
		data = " "
		try:
			s.connect((IP, chunkPort))
			logging.debug('Connection Established: ' + str(IP) + ' on port ' + str(chunkPort))
			# Request the chunk contents of the specified chunkserver
			fL.send(s, 'CONTENTS?')
			logging.debug('Sent chunkserver a CONTENTS? message')
			# Received the chunk contents of the speicified chunkserver
			data = fL.recv(s)
			# Close the socket connection
			s.close()

		# If the database is unable to connect to the chunkservers, retry the connection
		# If the retry fails, remove it from the list of activehosts and move on.
		except:
			# Make sure the socket connection is closed before doing anything, so if 
			# a retry occurs, the socket will not already be in use.
			s.close()
			# heartBeat the chunkserver. If that indicates it is alive, try to 
			# interrogate the chunkserver again. If not, remove if from the active 
			# hosts list and move on.
			if retry < 3:
				# It the heartBeat indicated the chunkserver is still alive, try again.
				if hB.heartBeat(IP) == 1:
					self.intterogateChunkServer(IP, retry + 1)
					logging.debug('Retry connect to chunkserver for interrogation')
				else:
					self.remFromAhosts(IP)
					logging.warning('Heartbeat indicates chunkserver dead. Not interrogating, moving on.')
					return -1

			else:
				self.remFromAhosts(IP)
				# Log the fact that we were unable to connect to a chunkserver
				logging.error("interrogateChunkServer failed to connect to " + IP)
				return -1


		
		
		logging.debug('Received response from chunkserver')


		# If the IP is not already in the location lookup, add it!
		if IP not in self.locDict.keys():
			self.locDict[IP] = []


		# If the chunkserver has nothing on it, it should return whitespace. If this is the case, 
		# then nothing in the database can be updated, so it will continue onto the next IP.
		# If the chunkserver returns something other than whitespace (a message formatted chunk1|chunk2|... )
		# then the data can be processed.
		if data != " ":
			# Convert the pipe separated string into a list
			chunkData = data.split('|')
			# In the event that data is formatted poorly with additional | characters, we want to get rid of null elements
			chunkData = filter(None, chunkData)

			# For every chunk handle in that list, update that chunk objects locations list
			for chunk in chunkData:

				# If the IP key is not already in the location lookup, add it!
				if IP not in self.locDict.keys():
					self.locDict[IP] = []
					# Add the chunk to the list of values for the IP key
					self.locDict[IP].append(chunk)

				# If the location does already exist, append the current chunk to its list of chunk values
				else:
					assChnks = self.locDict[IP]
					# But first, make sure that chunk isn't already in the values, so you don't get
					# multiple copies of the same chunk in the lookup.
					if chunk not in assChnks:
						# Add the chunk to the list of values for the IP key
						self.locDict[IP].append(chunk)

				try:
					# Find which file the chunk is associated with in the lookup dictionary
					fileName = self.lookup[chunk]

					# From the file name we found the chunk to be associated with in the
					# lookup, we can append the current IP to the list of chunk locations
					# in the chunk object within the file object dictionary.
					if IP not in self.data[fileName].chunks[chunk].locations:
						self.data[fileName].chunks[chunk].locations.append(IP)
						logging.debug('Appended location to chunk ' + str(chunk) + ' location list')

				# If the chunk is not recognized in the master's database, the chunk is an orphan (does not
				# belong to a file). In this case, the chunk should be removed.
				except KeyError:
					# Create an instance of a TCP socket
					s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
					s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
					s.connect((IP, chunkPort))
					logging.debug('Connection Established: ' + str(IP) + ' on port ' + str(chunkPort))
					# Tell the chunkserver to sanitize the chunk
					fL.send(s, 'SANITIZE|' + str(chunk))
					logging.debug('Sent chunkserver a SANITIZE message')
					# Received the chunk contents of the speicified chunkserver
					data = fL.recv(s)

					if data == "SUCCESS":
						logging.debug("Orphan removal successful")
						self.locDict[IP].remove(chunk)
					elif data == "FAILED":
						logging.debug("Orphan removal failed.")
					# Close the socket connection
					s.close()
		

		logging.debug('interrogateChunkServer() complete')
コード例 #43
0
ファイル: timingTests.py プロジェクト: ttanay/GFS
    #try:
    # Create a TCP socket instance
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    # Set the timeout of the socket
    s.settimeout(3)
    # Allow the socket to re-use address
    s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
    # Connect to the chunkserver over the specified port
    s.connect((config.masterip, config.port))
    #except:
    #	print "CONNECTION ERROR"
    #	errorList.append('SCRUBBER - Unable to connect to master')
    print "connected"

    fL.send(s, "GETDELETEDATA|*")

    data = fL.recv(s)

    s.close()
    print "recvd to delete list"
    toDelete = []

    for item in data.split("|"):
        if item != "":
            toDelete.append(item)

    # Create an instance of the Scrubber object, and initiate it.
    scrub = scrubber.Scrubber(toDelete)

    scrubStart = time.time()
コード例 #44
0
ファイル: serverEmulator.py プロジェクト: BenningtonCS/GFS
# Listen for client connections
s.listen(1)
print "Listening..."
# Accept the incoming connection
conn, addr = s.accept()
print "accepted connection"
# Receive the data
data = fL.recv(conn)

# If data was received
if data:
	print data
	# If the option to send back an eot was yes, send the message
	# with the eot
	if ending == "y":
		fL.send(conn, "Received successfully")
	# If the option to send back an eot was no, send the message
	# without the eot
	elif ending == "n":
		conn.send("Received successfully")

# If no data was received
if not data:
	print "No data received!"
	# If the option to send back an eot was yes, send the message
	# with the eot
	if ending == "y":
		fL.send(conn, "Received unsuccessfully")
	# If the option to send back an eot was no, send the message
	# without the eot
	elif ending == "n":
コード例 #45
0
ファイル: API.py プロジェクト: BenningtonCS/GFS
	def append(self, filename, newData,flag):
		# Create socket connection to the master
		m = self.createSocket()
		# if there was an error making the socket, exit the function
		if not m:
			return 0

		#send APPEND request to master
		try:
			fL.send(m, "APPEND|" + filename)
		except:
			print "COULD NOT SEND APPEND REQUEST TO MASTER"
			return 0
		#receive data back from master
		self.data = fL.recv(m)
		#close connection to master
		m.close()
		#some error handling
		if (self.data == "FAILED"):
			print "ERROR: MASTER SENT FAIL MESSAGE exiting..."
			return 0
		elif (self.data == "OPEN"):
			print "ERROR: FILE " +filename+" ALREADY OPEN"
			return 0
		#parse the data into useful parts
		self.splitdata = self.data.split("|")
		dataLength = len(self.splitdata)
		cH = self.splitdata[-1]
		#get length of the requested new data to use for append across chunks
		if flag == 1:
			with open(newData,"rb") as da:
				newData = da.read()
			
			
	
		dataSize = len(newData)
		
		
		lenNewData = int(dataSize)
		
		for n in range(0, dataLength-1):
			location = self.splitdata[n]
			#create socket to connect to chunk server at location
			self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
			#attempt to connect to chunk server at location
			try:
				self.s.connect((location,self.TCP_PORT))
			except:
				print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location
			#ask chunk server how much room is left on latest chunk
			fL.send(self.s, "CHUNKSPACE?|" + cH)
			#the response is stored in remainingSpace
			remainingSpace = fL.recv(self.s)
			self.s.close()
			#some error handling
			if remainingSpace == "FAILED":
				print "CHUNKSPACE REQUEST FAILED. exiting..."
				return 0
			#make remainingSpace an integer
			remainingSpace = int(remainingSpace)
			
			#if the length of the new data is greater than the room left in the chunk...
			if (lenNewData > remainingSpace):   
				#...split the data into two parts, the first part being equal to the
				#amount of room left in the current chunk. the second part being the 
				#rest of the data.
				cut = remainingSpace
				newData1 = newData[0:cut]
				print "Sending data of length:" + str(len(newData1))
				newData2 = newData[cut:]
				#tell the chunk server to append the first part of the new data that
				#will fill up the rest of the remaining space on a chunk
				s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
				try:
					s.connect((location, self.TCP_PORT))
				except:
					print "ERROR: COULD NOT REOPEN SOCKET"
				fL.send(s, "APPEND|" + cH + "|" + newData1)
				print "first append"
				SoF = fL.recv(s)
				#close connection to chunk server
				s.close()
				#error handling
				if SoF == "FAILED":
					print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..."
					return 0
				
				
			elif (lenNewData <= remainingSpace):
				t = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
				t.connect((location, self.TCP_PORT))
				try:
					fL.send(t, "APPEND|" + cH + "|" + newData)
				except:
					print "ERROR: COULD NOT SEND APPEND TO CHUNK SERVER"			
				SoF = fL.recv(t)
				t.close()
				#error handling/acks
				if SoF == "FAILED":
					print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..."
					return 0

		###################
		if lenNewData > remainingSpace:
			# Create socket connection to the master
			m = self.createSocket()
			# if there was an error making the socket, exit the function
			if not m:
				return 0

			#tell the master to create a new chunk for the remaining data
			try:
				fL.send(m, "CREATECHUNK|" + filename + "|" + cH)
			except:
				print "ERROR: COULD NOT CREATE NEW CHUNK TO APPEND TO"
			#receive data back from master
			cData = fL.recv(m)
			#parse this data and handle it very similarly as the in the create function
			if self.data == "FAIL2":
				print "NO SUCH FILE EXISTS FOR CHUNK CREATION"
				exit(0)
			splitcData = cData.split("|")
			cDataLength = len(splitcData)
			cH = splitcData[-1]
			#close the connection to the master so we can connect to the chunk servers
			m.close()
			#iterate through each IP address received from the master
			for n in range(0, cDataLength-1):
				#create a socket to be used to connect to chunk server
				s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
				#designate the IP for this iteration
				location = splitcData[n]
				print location
				#attempt to connect to the chunk server at the current location
				try:
					s.connect((location,self.TCP_PORT))
				except:
					print "ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location
					continue
					#send CREATE request to the chunk server at the current location
					fL.send(s, "CREATE|" + cH)
				global ack
				ack = fL.recv(s)
				################
				#close connection to current chunk server.
				s.close()
				#do some acks
				if ack == "FAILED":
					print "ERROR: CHUNK CREATION FAILED"
					#fL.send(m, "FAILED")
				elif ack == "CREATED":
					print "Chunk creation successful!"
					#fL.send(m, "CREATED")
				#m.close()
			
			#now that the new chunk has been created on all of the servers...
			#...run append again with the second part of the new data
			#self.s.close()
			try:
				self.append(filename, newData2,False)
			except UnboundLocalError:
				pass
		return 1
コード例 #46
0
ファイル: chunkserver.py プロジェクト: BenningtonCS/GFS
	def run(self):
		c = fL.recv(self.connection) # listens for a command on 
							# the connection handed 
							# down from the main 
							# thread
		com = c.split('|') # if c has multiple parts, they will be seperated by
				   # pipes. This will put each part into a list
		command = com[0]   # the command should be the first part of the message,
				   # thus the first part of the list
		logging.debug("Recieved command " + command)

		# next the distributor hands the connection to the appropriate 
		# worker based on the command given, invalid commands simply fall 
		# through

		if command == "<3?":
			# heartbeat to see if each chunkserver is running. If it is, it will
			# send back a confirmation of <3!
			try:
				# in this and each other if/elif statement the correct 
				# worker thread is started for a given command
				fL.send(self.connection, "<3!")
				logging.debug("Send heart beat back")
				self.connection.close()
				logging.debug("Closed connection.")
			except socket.error as e:
				logging.error(e)

		elif command == "CHUNKSPACE?":
			try:
#				fL.send(self.connection, "CONTINUE") # after receiving the connection 
#								 # the thread confirms that it is 
#								 # ready to receive arguments
#				logging.debug("send a continue")
#				chunkHandle = fL.recv(self.connection) # it listens on its 
#									 # connection for a chunkhandle
				chunkHandle = com[1] # name of the chunkhandle
				logging.debug("recieved name of the chunkhandle: " + chunkHandle)
				emptySpace = str(mg64 - os.stat(chunkPath + "/" + chunkHandle).st_size) # then checks the 
									         # difference 
									         # between the 
									         # file's size and 
									         # 64mg (the max 
									         # chunk size)
				fL.send(self.connection, str(emptySpace)) # and returns the amount of space 
								 # left to the API
				logging.debug("Send the space remaining")
#				self.connection.close() # closes the connection
#				logging.debug("Closed the connection")
			except socket.error as e:
				logging.error(e)
			except IOError as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)
			except Exception as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)

		elif command == "READ":
			# read data from a chunk
			try:
#				fL.send(self.connection, "CONTINUE") # confirms readiness for data
#				logging.debug("sent continue #1")
#				chunkHandle = fL.recv(self.connection) # listens for chunkHandle
				chunkHandle = com[1]
#				logging.debug("recieved name of the chunkhandle: " + chunkHandle)
#				fL.send(self.connection, "CONTINUE") # confirms ready state
#				logging.debug("sent continue #2")
				byteOffSet = int(com[2]) # listens for a byte 
									     # offset to read from 
									     # (relative to the 
									     # beginning of the 
									     # given chunk)
#				byteOffSet 
#				logging.debug("recieved the byte offset number.")
#				fL.send(self.connection, "CONTINUE") # confirms the desire for EVEN MORE data
#				logging.debug("sent continue #3") 
				bytesToRead = int(com[3]) #int(fL.recv(self.connection)) # listens for the 
									      # number of bytes to read
				logging.debug("recieved the number of bytes to read")
				chunk = open(chunkPath+"/"+chunkHandle) # opens the designated chunk to read from
				chunk.seek(int(byteOffSet)) # goes to the specified byte offset
				fileContent = chunk.read(bytesToRead) # stuffs all the stuff to be 
								      # read into a variable
				fL.send(self.connection, fileContent)
				logging.debug("send the file content")
				chunk.close() # closes the chunk
				logging.debug("closed the connection")
				self.connection.close() # closes the connection
			except socket.error as e:
				logging.error(e)
			except IOError as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)
			except Exception as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)

		elif command == "CONTENTS?":
			try:
				output = ""
				for files in os.walk(chunkPath): # read every file
					output = str('|'.join(item for item in files[-1])) # turn the list into a string
					print output
				if output == "":		     # if there is nothing in the dir
					print "output is empty"
					fL.send(self.connection, " ")    # send an empty string
					logging.debug("Sent an empty string which should be the output")
				else:				     # otherwise
					print "output is not empty"
					fL.send(self.connection, output) # send everything as a string
					logging.debug("sent the output")
			except socket.error as e:
				logging.error(e)
			except IOError as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)
			except Exception as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)

		elif command == "CREATE":
			# create a new chunk
			try:
#				fL.send(self.connection, "CONTINUE")
#				logging.debug("Sent continue")
	                	chunkHandle = com[1] #fL.recv(self.connection) # get the name of the chunk
				logging.debug("recieved name of the chunk")
	                	open(chunkPath + "/" + chunkHandle, 'w').close() # create the file
	        	except IOError as e:
				logging.error(e)
				fL.send(self.connection, "FAILED")
			except Exception as e:
				logging.error(e)
				fL.send(self.connection, "FAILED")
			else:
				fL.send(self.connection, "CREATED")
			

		elif command == "APPEND":	
			# append new data to a chunk
			try:
#				fL.send(self.connection, "CONTINUE")
#				logging.debug("sent continue #1")
#				chunkHandle = fL.recv(self.connection) # name of the chunk
				chunkHandle = com[1]
#				logging.debug("Recieved name of the chunk")
#				fL.send(self.connection, "CONTINUE") 
#				logging.debug("Sent continue #2") 
#				data = fL.recv(self.connection)    # data being added
				data = "|".join(com[2:])
				length = str(len(data))
				logging.error(length)
				logging.debug("Recieved the data to be added")
	                	with open(chunkPath+"/"+chunkHandle, 'a') as a: # open the chunk
	                        	#for item in data:
					a.write(data)		 # add the data to it
	                except socket.error as e:
				logging.error(e)
			except IOError as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)
			except Exception as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)
			else:
				fL.send(self.connection,"SUCCESS")

		elif command == "SANITIZE":
			# recieves SANITIZE from the scrubber which tells the chunkserver to delete a chunk
			try:
				chunkHandle = com[1] # name of the chunk
				logging.debug("Recieved name of the chunk")
				try:
					os.remove(chunkPath + '/' + chunkHandle) # remove file from directory
					logging.debug("Removed chunk handle.")
				# If there is an error thrown that the chunk does not exist, we return success
				# because sanitize is called to remove a chunk from a location. If it does not
				# exist at that location, then removal is technially achieved.
				except OSError:
					fL.send(self.connection, "SUCCESS")
					logging.debug("chunk already did not exist. sending success")
					
				fL.send(self.connection, "SUCCESS") # send a success
				logging.debug("removal successfull!")
				
			except socket.error as e:
				logging.error(e)
			except IOError as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)
			except Exception as e:
				fL.send(self.connection,"FAILED")
				logging.error(e)



 		else:
 			error = "Received invalid command: " + command
 			logging.error(error)
コード例 #47
0
 def fileList(self):
     # call the database object's returnData method
     list = str(database.getFiles())
     fL.send(self.s, list)
コード例 #48
0
ファイル: clientEmulator.py プロジェクト: BenningtonCS/GFS
port = int(raw_input('What port to connect over? : '))

# Define a socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Allow socket address reuse
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Connect to the server over the specified IP and port 
s.connect(('', port))


message = str(raw_input('What message would you like to send? : '))
ending = str(raw_input('Include proper EOT character? (y/n): ')).lower()

# Send the message with a proper EOT termination
if ending == "y":
	fL.send(s, message)

# Send the message without the proper EOT termination
elif ending == "n":
	s.send(message)

# Receive data back from the server
data = fL.recv(s)

# Print the data to console
print "DATA == ", data


s.close()
コード例 #49
0
 def getAllChunks(self):
     chunks = database.allChunks(self.fileName)
     fL.send(self.s, chunks)
コード例 #50
0
ファイル: scrubber.py プロジェクト: ttanay/GFS
	def clean(self):
		logging.debug("SCRUBBER: Commencing Clean")

		# For all of the files in the toDelete list
		for fileName in self.data:


			# Create a TCP connection to the master
			self.connect(config.masterip, 0)
			try:
				# Request all the chunks associated with a given file
				fL.send(self.s, "GETALLCHUNKS|" + fileName)
				# Receive all the chunks associated with a file
				data = fL.recv(self.s)

			except (socket.timeout, socket.error) as e:
				logging.error("SCRUBBER: Connection to master failed with error: " + str(e) + ". Unable to continue for file: " + str(fileName))
				# Set data to be empty so the deletion process will not continue to 
				# try and delete a file it received no information about.
				data = ""

			# Be sure the socket is closed.
			self.s.close()


			# Convert the pipe-separated string of chunk handles into a list
			chunkHandles = data.split("|")
			# Ensure there are no "" elements in the list
			chunkHandles = filter(None, chunkHandles)
			# Create a counter for successful chunk deletions
			successfulChunkDelete = 0


			# For each of those chunk handles
			for handle in chunkHandles:

				# Create a counter for successful deletions from a chunkserver
				successDeleteFromCS = 0
				# Create a TCP connection to the master
				self.connect(config.masterip, 0)
				try:
					# Request all the locations associated with a given chunk
					fL.send(self.s, "GETLOCATIONS|" + handle)
					# Receive all the locations associated with a chunk
					data = fL.recv(self.s)

				except (socket.timeout, socket.error) as e:
					logging.error("SCRUBBER: Connection to master failed with error: " + str(e) + ". Unable to continue for chunk: " + str(handle))
					# Set data to be empty so the deletion process will not continue to 
					# try and delete a file it received no information about.
					data = ""

				# Be sure the socket is closed
				self.s.close()


				# Convert the pipe-separated string of locations into a list
				locations = data.split("|")
				# Ensure there are no "" elements in the list
				locations = filter(None, locations)



				# For each location the chunk is stored on
				for location in locations:

					# Send a SANITIZE message to a specified location
					data = self.cleanLocation(location, handle)


					# If the chunk server responds with a success message, increment the success counter
					if data == "SUCCESS":
						logging.debug("SCRUBBER: Chunk successfully removed from chunkserver")
						successDeleteFromCS += 1
						

					# If the chunk server responds with a failure message, DO SOMETHING ELSE!
					elif data == "FAILED":
						retryAck = self.cleanLocation(location, handle)

						if retryAck == "SUCCESS":
							logging.debug("SCRUBBER: Chunk successfully removed from chunkserver")
							successDeleteFromCS += 1

						elif retryAck == "FAILED":
							logging.error("SCRUBBER: Received failure message on chunk delete. Chunkhandle : " + str(handle))

						else: 
							logging.error("SCRUBBER: Unexpected Receive: " + str(data) + " from chunkserver " + str(location))


					# If the chunk server responds with something other than SUCCESS or FAILED, something went wrong.
					else:
						logging.error("SCRUBBER: Unexpected Receive: " + str(data) + " from chunkserver " + str(location))

				# If the success counter is equal to the amount of all the IPs, then
				# all the IPs successfully deleted that chunk, so increment the 
				# successfulChunkDelete counter
				if len(locations) == successDeleteFromCS:
					successfulChunkDelete += 1
				else:
					# Improve error handling to maybe automatically retry
					logging.error("SCRUBBER: Not all chunk location deletes were successful")

			# If the number of successful chunk deletes is equal to the number of chunks
			# associated with the file, then all the chunks for that file have been deleted,
			# so the file entry can be deleted
			if len(chunkHandles) == successfulChunkDelete:
				# Call the database sanitize function, which removes the key/value pair
				# from the database.
				self.connect(config.masterip, 0)
				fL.send(self.s, "SANITIZE|" + fileName)

				#data = fL.recv(self.s)
				self.s.close()
				logging.debug("SCRUBBER: " + str(fileName) + 'successfully sanitized')
				
			else:
				# Improve error handling to automatically resolve problem
				logging.error("SCRUBBER: Not all chunk deletes were successful")
コード例 #51
0
ファイル: master.py プロジェクト: BenningtonCS/GFS
    def read(self):
        # Get the byte offset and bytes to read from the received message
        try:
            byteOffset = int(self.msg[2])
            bytesToRead = int(self.msg[3])
            # If there is an index error (the values do not exist in the message)
            # alert the client and end the read function.
        except IndexError:
            fL.send(self.s, "READ command not given proper parameters")
            return

        logging.debug("parsed byte offset and bytes to read")

        # Get the size of a chunk from the config file
        maxChunkSize = config.chunkSize
        # Find the sequence of the chunk in the given file by using integer division
        # (divide and take the floor)
        startSequence = byteOffset // maxChunkSize

        if startSequence > len(database.data[self.fileName].chunks.keys()):
            logging.debug("No such byte offset exists for the given file")
            fL.send(self.s, "FAILED, NO SUCH BYTE OFFSET EXISTS FOR THIS FILE")
            return

            # Get the offset of the read-start within its given chunk
        chunkByteOffset = byteOffset % maxChunkSize

        logging.debug("start sequence # == " + str(startSequence))
        logging.debug("chunk byte offset == " + str(chunkByteOffset))

        # If the user inputs a bytes to read of -1, the read will go until the end
        # of the file.
        if bytesToRead == -1:
            # The ending offset will be the max file size
            endOffset = maxChunkSize
            # The end sequence can be found be knowing how many chunks a file has, and
            # subtracting by 1 because the sequence numbers start at 0, not 1.
            endSequence = len(database.data[self.fileName].chunks.keys()) - 1

        else:
            # To find where the user wants to read ending, add the number of bytes they want
            # to read to their starting point, the byteOffest. This will give the byte offset
            # of the end of the read
            endReadOffset = byteOffset + bytesToRead

            # Find the sequence of the chunk in which the end of the read will terminate
            endSequence = endReadOffset // maxChunkSize

            # Get the offset of the read-end within its given chunk
            endOffset = endReadOffset % maxChunkSize

        logging.debug("end sequence # == " + str(endSequence))
        logging.debug("end read offset == " + str(endOffset))

        # Create an empty string to hold the message that will be sent to the client
        responseMessage = "READFROM"

        # For each sequence number that exists between (and including) the read-start chunk
        # and the read-end chunk, get the file's chunk with the appropriate sequence number,
        # and append to the response message, a location it is stored at, its chunk handle,
        # the byte offset from within that chunk to begin reading from, and the byte offset
        # to end reading from
        for sequence in range(startSequence, (endSequence + 1)):
            # 	try:
            # Get the chunkHandles associated with the given file, and sort the chunkHandles from
            # least to greatest in the list. This will put them in their sequence order where the
            # 0th element is now the 0th sequence, 1st element the 1st sequence, etc.
            logging.debug(sorted(database.data[self.fileName].chunks.keys()))
            associatedChunkHandles = sorted(database.data[self.fileName].chunks.keys())

            # Append a location of where the start-sequence chunk is stored to the message
            logging.debug(database.data[self.fileName].chunks[associatedChunkHandles[sequence]].locations)
            responseMessage += "|" + str(
                database.data[self.fileName].chunks[associatedChunkHandles[sequence]].locations[0]
            )

            # Append the chunk handle to the message
            responseMessage += "*" + str(associatedChunkHandles[sequence])

            # Append the byte offset to start reading from to the message
            responseMessage += "*" + str(chunkByteOffset)

            # If there are multiple chunks that will be read over, the next chunk will start
            # the read from the beginning
            chunkByteOffset = 0

            # Check to see if the READ will take place in the same chunk. If it does, append the
            # endOffset to the message so the client will know where to end reading
            if startSequence == endSequence:
                responseMessage += "*" + str(endOffset)
                # If the READ takes place over multiple chunks, write the end of read for the current
                # chunk to be the end of the chunk, and then increase the start sequence number so when the
                # metadata for the last chunk is processed, it will be caught by the if statement above
                # and send the appropriate ending offset.
            elif startSequence < endSequence:
                responseMessage += "*" + str(maxChunkSize)
                startSequence += 1

                # 	except:
                # 		logging.error("Unable to generate proper READ response message.")

        logging.debug("RESPONSE MESSAGE == " + str(responseMessage))
        # send our message
        fL.send(self.s, responseMessage)
        logging.debug("SENT == " + responseMessage)
        # Visual confirmation for debugging: confirm success of read()
        logging.debug("Read successfully handled")