def read(self, filename, byteOffset, bytesToRead, newName): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send READ request to the master try: fL.send( m, "READ|" + filename + "|" + str(byteOffset) + "|" + str(bytesToRead)) except: print "ERROR: COULD NOT SEND READ REQUEST TO MASTER" #recieve data from the master self.data = fL.recv(m) #close connection to master m.close() #split the data into a list self.splitdata = self.data.split("|") #remove the first element of the list because it is irrelevant self.splitdata = self.splitdata[1:] #iterate through the list fromChunks = "" for q in self.splitdata: #split the list into smaller parts secondSplit = q.split("*") #set the location... location = secondSplit[0] #...and the chunk handle cH = secondSplit[1] #...and the offset offset = secondSplit[2] #connect to the chunk server s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((location, self.TCP_PORT)) except: print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location return 0 #send READ request to chunk server fL.send( s, "READ|" + str(cH) + "|" + str(offset) + "|" + str(bytesToRead)) #receive and print the contents of the file #fromChunks += "." + str(cH) dat = fL.recv(s) #close connection to chunk server s.close() with open(newName, "ab") as e: e.write(dat) return 1
def read(self, filename, byteOffset, bytesToRead, newName): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send READ request to the master try: fL.send(m, "READ|" + filename + "|" + str(byteOffset) + "|" + str(bytesToRead)) except: print "ERROR: COULD NOT SEND READ REQUEST TO MASTER" #recieve data from the master self.data = fL.recv(m) #close connection to master m.close() #split the data into a list self.splitdata = self.data.split("|") #remove the first element of the list because it is irrelevant self.splitdata = self.splitdata[1:] #iterate through the list fromChunks = "" for q in self.splitdata: #split the list into smaller parts secondSplit = q.split("*") #set the location... location = secondSplit[0] #...and the chunk handle cH = secondSplit[1] #...and the offset offset = secondSplit[2] #connect to the chunk server s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((location,self.TCP_PORT)) except: print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location return 0 #send READ request to chunk server fL.send(s, "READ|" + str(cH) + "|" + str(offset) + "|" + str(bytesToRead)) #receive and print the contents of the file #fromChunks += "." + str(cH) dat = fL.recv(s) #close connection to chunk server s.close() with open(newName,"ab") as e: e.write(dat) return 1
def delete(self, filename): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send DELETE request to the master try: fL.send(m, "DELETE|" + filename) except: print "ERROR: COULD NOT SEND DELETE REQUEST TO MASTER" #receive acks from the master self.data = fL.recv(m) #tell the user whether the file was successfully marked or not if self.data == "FAILED1": print "ERROR: The file could not be marked for deletion." return -1 elif self.data == "FAILED2": print "ERROR: The given file name does not exist." return -2 elif self.data == "FAILED3": print "The file has already been marked for deletion." return -3 elif self.data == "MARKED": print "File successfully marked for deletion." m.close()
def create(self): # Visual confirmation for debugging: confirm init of create() logging.debug('Creating chunk metadata') # Acquire a lock so the chunk handle counter can not be accessed simultaneously self.lock.acquire() # Get a new chunkhandle chunkHandle = database.getChunkHandle() # Release the lock so others can access the chunk handle counter self.lock.release() # Create a new file, and store the return flag createFileFlag = database.createNewFile(self.fileName, chunkHandle) # If the return flag was an error flag, alert the logger and API of the error if createFileFlag == -1: logging.error("Got a duplicate file name, sending FAIL to API") fL.send(self.s, "FAIL1") return -1 elif createFileFlag == -2: logging.error("No file exists for a chunk to be created for") fL.send(self.s, "FAIL2") elif createFileFlag == -3: logging.error( "Chunk is not the latest chunk. New chunk has been created that can be appended to." ) fL.send(self.s, "FAIL3") # Get the locations for a specified chunk locations = database.data[self.fileName].chunks[chunkHandle].locations # Parse the locations list retreived above into a pipe-separated list hosts = "" for item in locations: hosts += item + "|" # Visual confirmation for debugging: confirm success of create() logging.debug('Chunk metadata successfully created') try: # Send the API a string containing the location and chunkHandle information fL.send(self.s, str(hosts) + str(chunkHandle)) except socket.error: logging.warning('Socket Connection Broken') # Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle logging.debug('SENT ==> ' + str(hosts) + str(chunkHandle)) # Receieve an ack to affirm that the chunk was successfully created ack = fL.recv(self.s) if ack == "CREATED": logging.debug("successful creation") elif ack == "FAILED": logging.error("unsuccessful creation") else: logging.error("unknown ack")
def create(self): # Visual confirmation for debugging: confirm init of create() logging.debug("Creating chunk metadata") # Acquire a lock so the chunk handle counter can not be accessed simultaneously self.lock.acquire() # Get a new chunkhandle chunkHandle = database.getChunkHandle() # Release the lock so others can access the chunk handle counter self.lock.release() # Create a new file, and store the return flag createFileFlag = database.createNewFile(self.fileName, chunkHandle) # If the return flag was an error flag, alert the logger and API of the error if createFileFlag == -1: logging.error("Got a duplicate file name, sending FAIL to API") fL.send(self.s, "FAIL1") return -1 elif createFileFlag == -2: logging.error("No file exists for a chunk to be created for") fL.send(self.s, "FAIL2") elif createFileFlag == -3: logging.error("Chunk is not the latest chunk. New chunk has been created that can be appended to.") fL.send(self.s, "FAIL3") # Get the locations for a specified chunk locations = database.data[self.fileName].chunks[chunkHandle].locations # Parse the locations list retreived above into a pipe-separated list hosts = "" for item in locations: hosts += item + "|" # Visual confirmation for debugging: confirm success of create() logging.debug("Chunk metadata successfully created") try: # Send the API a string containing the location and chunkHandle information fL.send(self.s, str(hosts) + str(chunkHandle)) except socket.error: logging.warning("Socket Connection Broken") # Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle logging.debug("SENT ==> " + str(hosts) + str(chunkHandle)) # Receieve an ack to affirm that the chunk was successfully created ack = fL.recv(self.s) if ack == "CREATED": logging.debug("successful creation") elif ack == "FAILED": logging.error("unsuccessful creation") else: logging.error("unknown ack")
def run(self): if self.role == 1: print "1" self.socket.connect((self.remoteAddr,self.port)) print "2" fL.send(self.socket, "<3?") print "3" print fL.recv(self.socket) print "4" elif self.role == 2: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "CHUNKSPACE?|"+chunkHandle) print "chunkspace :" + fL.recv(self.socket) elif self.role == 3: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "READ|"+chunkHandle+"|"+byteOffSet+"|"+bytesToRead) print "Reading.... " + fL.recv(self.socket) elif self.role == 4: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "CONTENTS?") print "contents :" + fL.recv(self.socket) elif self.role == 5: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "CREATE|"+chunkHandle) print fL.recv(self.socket) elif self.role == 6: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "APPEND|"+chunkHandle+"|"+appendData) elif self.role == 7: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "SANITIZE|"+chunkHandle) print fL.recv(self.socket)
def heartBeat(self, IP): logging.debug('HEARTBEAT: Initiating heartBeat protocol') # Get the list of all active chunk server IPs activeServers = self.getActiveChunkServers() try: # Create a TCP socket instance self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Set the timeout of the socket self.s.settimeout(self.SOCK_TIMEOUT) # Allow the socket to re-use address self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Connect to the chunkserver over the specified port self.s.connect((IP, self.PORT)) # Send the chunk server a heart (ping) fL.send(self.s, "<3?") # Get the chunk server response data = fL.recv(self.s) print data # Close the connection to allow future connections self.s.close() # If the chunk server responds with a heart, add it to activehosts if data == "<3!": # If the chunkserver IP is not in the activeServers list, add it to the list if IP not in activeServers: with open(self.AHOSTS, "a") as file: file.write(IP + "\n") # Return a 1 so any script calling heartBeat will know the heartbeat was successful return 1 # Handle the timeout (chunk server alive but not responding) and connection (server dead) errors except (socket.timeout, socket.error): print "</3" logging.debug('HEARTBEAT: Could not connect to ' + IP) # Check to see if the chunkserver is in the list of active IPs if IP in activeServers: # If it is, remove it from the list activeServers.remove(IP) # Clear the previous activehosts.txt file and replace it with the list of active servers, # which now excludes the failed chunkserver with open(self.AHOSTS, "w") as file: newList = "" for item in activeServers: newList += item + "\n" file.write(newList) # Return a -1 so any script calling heartbeat will know the heartbeat failed. return -1
def fileList(self): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 try: # Send the master a request for the list of files the system contains fL.send(m, "FILELIST|x") # Recieve the file list from the master data = fL.recv(m) # Close the TCP connection to the master m.close() return data except Exception as e: raise e logging.error("Unable to get file list from master.") return 0
def cleanLocation(self, location, handle): # Connect to the chunk server self.connect(location, 0) try: # Send the chunk server a SANITIZE message with the chunk handle # so it knows which chunk it is deleting fL.send(self.s, 'SANITIZE|' + str(handle)) logging.debug('SCRUBBER: Sent SANITIZE message to chunkserver') # Wait for a response back from the chunk server to verify that # the chunk was removed data = fL.recv(self.s) logging.debug('SCRUBBER: Received response from chunkserver') return data except (socket.timeout, socket.error) as e: logging.error("SCRUBBER: Connection to chunkserver failed with error: " + str(e) + ". Unable to continue for location: " + str(location)) # Set data to be empty so the deletion process will not continue to # try and delete a file it received no information about. data = "" return data
def run(self): self.lock.acquire() self.data = fL.recv(self.connection) dataSplit = self.data.split('|') print self.data msg = dataSplit[0] if (msg == "CREATE"): create = API.create(dataSplit[1]) if (create == 1): self.connection.send("CREATE|1") elif (msg == "APPEND"): print "append" append = API.append(dataSplit[1], dataSplit[2], True) if (append): self.connection.send("1") print "1 sent for append" elif (msg == "READ"): read = API.read(dataSplit[1], 0, -1, "/var/www/fileshare/download/" + dataSplit[1]) if (read): self.connection.send("1") elif (msg == "STREAM"): stream = API.stream(dataSplit[1], 0, -1, "/var/www/fileshare/download/" + dataSplit[1]) if (stream): self.connection.send("1") elif (msg == "FILELIST"): filelist = API.fileList() filelist = filelist[1:-1] split = filelist.split(', ') string = "" for i in split: string += str(i[1:-1]) + "|" if (filelist): self.connection.send(string) #elif(msg == "DELETE"): self.connection.close() self.lock.release()
def run(self): self.lock.acquire() self.data = fL.recv(self.connection) dataSplit = self.data.split('|') print self.data msg = dataSplit[0] if(msg == "CREATE"): create = API.create(dataSplit[1]) if(create == 1): self.connection.send("CREATE|1") elif(msg == "APPEND"): print "append" append = API.append(dataSplit[1], dataSplit[2], True) if(append): self.connection.send("1") print "1 sent for append" elif(msg == "READ"): read = API.read(dataSplit[1], 0, -1, "/var/www/fileshare/download/"+dataSplit[1]) if(read): self.connection.send("1") elif(msg == "STREAM"): stream = API.stream(dataSplit[1], 0, -1, "/var/www/fileshare/download/"+dataSplit[1]) if(stream): self.connection.send("1") elif(msg == "FILELIST"): filelist = API.fileList() filelist = filelist[1:-1] split = filelist.split(', ') string = "" for i in split: string += str(i[1:-1]) + "|" if(filelist): self.connection.send(string) #elif(msg == "DELETE"): self.connection.close() self.lock.release()
def undelete(self, filename): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send UNDELETE request to master try: fL.send(m, "UNDELETE|" + filename) except: print "ERROR COULD NOT SEND UNDELETE REQUEST TO MASTER" #receive acks from the master self.data = fL.recv(m) #tell the user whether the file was successfully unmarked or not if self.data == "FAILED1": print "ERROR: COULD NOT UNDELETE FILE" return -1 elif self.data == "FAILED2": print "File was not flagged for deletion." return -2 elif self.data == "MARKED": print "File successfully unmarked for deletion." m.close()
def run(self): c = fL.recv(self.connection) # listens for a command on # the connection handed # down from the main # thread com = c.split( '|') # if c has multiple parts, they will be seperated by # pipes. This will put each part into a list command = com[ 0] # the command should be the first part of the message, # thus the first part of the list logging.debug("Recieved command " + command) # next the distributor hands the connection to the appropriate # worker based on the command given, invalid commands simply fall # through if command == "<3?": # heartbeat to see if each chunkserver is running. If it is, it will # send back a confirmation of <3! try: # in this and each other if/elif statement the correct # worker thread is started for a given command fL.send(self.connection, "<3!") logging.debug("Send heart beat back") self.connection.close() logging.debug("Closed connection.") except socket.error as e: logging.error(e) elif command == "CHUNKSPACE?": try: # fL.send(self.connection, "CONTINUE") # after receiving the connection # # the thread confirms that it is # # ready to receive arguments # logging.debug("send a continue") # chunkHandle = fL.recv(self.connection) # it listens on its # # connection for a chunkhandle chunkHandle = com[1] # name of the chunkhandle logging.debug("recieved name of the chunkhandle: " + chunkHandle) emptySpace = str( mg64 - os.stat(chunkPath + "/" + chunkHandle).st_size) # then checks the # difference # between the # file's size and # 64mg (the max # chunk size) fL.send(self.connection, str(emptySpace)) # and returns the amount of space # left to the API logging.debug("Send the space remaining") # self.connection.close() # closes the connection # logging.debug("Closed the connection") except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection, "FAILED") logging.error(e) except Exception as e: fL.send(self.connection, "FAILED") logging.error(e) elif command == "READ": # read data from a chunk try: # fL.send(self.connection, "CONTINUE") # confirms readiness for data # logging.debug("sent continue #1") # chunkHandle = fL.recv(self.connection) # listens for chunkHandle chunkHandle = com[1] # logging.debug("recieved name of the chunkhandle: " + chunkHandle) # fL.send(self.connection, "CONTINUE") # confirms ready state # logging.debug("sent continue #2") byteOffSet = int(com[2]) # listens for a byte # offset to read from # (relative to the # beginning of the # given chunk) # byteOffSet # logging.debug("recieved the byte offset number.") # fL.send(self.connection, "CONTINUE") # confirms the desire for EVEN MORE data # logging.debug("sent continue #3") bytesToRead = int( com[3]) #int(fL.recv(self.connection)) # listens for the # number of bytes to read logging.debug("recieved the number of bytes to read") chunk = open( chunkPath + "/" + chunkHandle) # opens the designated chunk to read from chunk.seek( int(byteOffSet)) # goes to the specified byte offset fileContent = chunk.read( bytesToRead) # stuffs all the stuff to be # read into a variable fL.send(self.connection, fileContent) logging.debug("send the file content") chunk.close() # closes the chunk logging.debug("closed the connection") self.connection.close() # closes the connection except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection, "FAILED") logging.error(e) except Exception as e: fL.send(self.connection, "FAILED") logging.error(e) elif command == "CONTENTS?": try: output = "" for files in os.walk(chunkPath): # read every file output = str('|'.join( item for item in files[-1])) # turn the list into a string print output if output == "": # if there is nothing in the dir print "output is empty" fL.send(self.connection, " ") # send an empty string logging.debug( "Sent an empty string which should be the output") else: # otherwise print "output is not empty" fL.send(self.connection, output) # send everything as a string logging.debug("sent the output") except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection, "FAILED") logging.error(e) except Exception as e: fL.send(self.connection, "FAILED") logging.error(e) elif command == "CREATE": # create a new chunk try: # fL.send(self.connection, "CONTINUE") # logging.debug("Sent continue") chunkHandle = com[ 1] #fL.recv(self.connection) # get the name of the chunk logging.debug("recieved name of the chunk") open(chunkPath + "/" + chunkHandle, 'w').close() # create the file except IOError as e: logging.error(e) fL.send(self.connection, "FAILED") except Exception as e: logging.error(e) fL.send(self.connection, "FAILED") else: fL.send(self.connection, "CREATED") elif command == "APPEND": # append new data to a chunk try: # fL.send(self.connection, "CONTINUE") # logging.debug("sent continue #1") # chunkHandle = fL.recv(self.connection) # name of the chunk chunkHandle = com[1] # logging.debug("Recieved name of the chunk") # fL.send(self.connection, "CONTINUE") # logging.debug("Sent continue #2") # data = fL.recv(self.connection) # data being added data = "|".join(com[2:]) length = str(len(data)) logging.error(length) logging.debug("Recieved the data to be added") with open(chunkPath + "/" + chunkHandle, 'a') as a: # open the chunk #for item in data: a.write(data) # add the data to it except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection, "FAILED") logging.error(e) except Exception as e: fL.send(self.connection, "FAILED") logging.error(e) else: fL.send(self.connection, "SUCCESS") elif command == "SANITIZE": # recieves SANITIZE from the scrubber which tells the chunkserver to delete a chunk try: chunkHandle = com[1] # name of the chunk logging.debug("Recieved name of the chunk") try: os.remove(chunkPath + '/' + chunkHandle) # remove file from directory logging.debug("Removed chunk handle.") # If there is an error thrown that the chunk does not exist, we return success # because sanitize is called to remove a chunk from a location. If it does not # exist at that location, then removal is technially achieved. except OSError: fL.send(self.connection, "SUCCESS") logging.debug( "chunk already did not exist. sending success") fL.send(self.connection, "SUCCESS") # send a success logging.debug("removal successfull!") except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection, "FAILED") logging.error(e) except Exception as e: fL.send(self.connection, "FAILED") logging.error(e) else: error = "Received invalid command: " + command logging.error(error)
import socket, config import functionLibrary as fL port = int(raw_input('What port to connect over? : ')) # Define a socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Allow socket address reuse s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Connect to the server over the specified IP and port s.connect(('', port)) message = str(raw_input('What message would you like to send? : ')) ending = str(raw_input('Include proper EOT character? (y/n): ')).lower() # Send the message with a proper EOT termination if ending == "y": fL.send(s, message) # Send the message without the proper EOT termination elif ending == "n": s.send(message) # Receive data back from the server data = fL.recv(s) # Print the data to console print "DATA == ", data s.close()
port = int(raw_input('What port to connect over? : ')) # Define a socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # Allow socket address reuse s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Connect to the server over the specified IP and port s.connect(('', port)) message = str(raw_input('What message would you like to send? : ')) ending = str(raw_input('Include proper EOT character? (y/n): ')).lower() # Send the message with a proper EOT termination if ending == "y": fL.send(s, message) # Send the message without the proper EOT termination elif ending == "n": s.send(message) # Receive data back from the server data = fL.recv(s) # Print the data to console print "DATA == ", data s.close()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Bind the listener-server s.bind(('', port)) ending = str( raw_input('Include proper EOT character in response? (y/n): ')).lower() # Listen for client connections s.listen(1) print "Listening..." # Accept the incoming connection conn, addr = s.accept() print "accepted connection" # Receive the data data = fL.recv(conn) # If data was received if data: print data # If the option to send back an eot was yes, send the message # with the eot if ending == "y": fL.send(conn, "Received successfully") # If the option to send back an eot was no, send the message # without the eot elif ending == "n": conn.send("Received successfully") # If no data was received if not data:
def run(self): c = fL.recv(self.connection) # listens for a command on # the connection handed # down from the main # thread com = c.split('|') # if c has multiple parts, they will be seperated by # pipes. This will put each part into a list command = com[0] # the command should be the first part of the message, # thus the first part of the list logging.debug("Recieved command " + command) # next the distributor hands the connection to the appropriate # worker based on the command given, invalid commands simply fall # through if command == "<3?": # heartbeat to see if each chunkserver is running. If it is, it will # send back a confirmation of <3! try: # in this and each other if/elif statement the correct # worker thread is started for a given command fL.send(self.connection, "<3!") logging.debug("Send heart beat back") self.connection.close() logging.debug("Closed connection.") except socket.error as e: logging.error(e) elif command == "CHUNKSPACE?": try: # fL.send(self.connection, "CONTINUE") # after receiving the connection # # the thread confirms that it is # # ready to receive arguments # logging.debug("send a continue") # chunkHandle = fL.recv(self.connection) # it listens on its # # connection for a chunkhandle chunkHandle = com[1] # name of the chunkhandle logging.debug("recieved name of the chunkhandle: " + chunkHandle) emptySpace = str(mg64 - os.stat(chunkPath + "/" + chunkHandle).st_size) # then checks the # difference # between the # file's size and # 64mg (the max # chunk size) fL.send(self.connection, str(emptySpace)) # and returns the amount of space # left to the API logging.debug("Send the space remaining") # self.connection.close() # closes the connection # logging.debug("Closed the connection") except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection,"FAILED") logging.error(e) except Exception as e: fL.send(self.connection,"FAILED") logging.error(e) elif command == "READ": # read data from a chunk try: # fL.send(self.connection, "CONTINUE") # confirms readiness for data # logging.debug("sent continue #1") # chunkHandle = fL.recv(self.connection) # listens for chunkHandle chunkHandle = com[1] # logging.debug("recieved name of the chunkhandle: " + chunkHandle) # fL.send(self.connection, "CONTINUE") # confirms ready state # logging.debug("sent continue #2") byteOffSet = int(com[2]) # listens for a byte # offset to read from # (relative to the # beginning of the # given chunk) # byteOffSet # logging.debug("recieved the byte offset number.") # fL.send(self.connection, "CONTINUE") # confirms the desire for EVEN MORE data # logging.debug("sent continue #3") bytesToRead = int(com[3]) #int(fL.recv(self.connection)) # listens for the # number of bytes to read logging.debug("recieved the number of bytes to read") chunk = open(chunkPath+"/"+chunkHandle) # opens the designated chunk to read from chunk.seek(int(byteOffSet)) # goes to the specified byte offset fileContent = chunk.read(bytesToRead) # stuffs all the stuff to be # read into a variable fL.send(self.connection, fileContent) logging.debug("send the file content") chunk.close() # closes the chunk logging.debug("closed the connection") self.connection.close() # closes the connection except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection,"FAILED") logging.error(e) except Exception as e: fL.send(self.connection,"FAILED") logging.error(e) elif command == "CONTENTS?": try: output = "" for files in os.walk(chunkPath): # read every file output = str('|'.join(item for item in files[-1])) # turn the list into a string print output if output == "": # if there is nothing in the dir print "output is empty" fL.send(self.connection, " ") # send an empty string logging.debug("Sent an empty string which should be the output") else: # otherwise print "output is not empty" fL.send(self.connection, output) # send everything as a string logging.debug("sent the output") except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection,"FAILED") logging.error(e) except Exception as e: fL.send(self.connection,"FAILED") logging.error(e) elif command == "CREATE": # create a new chunk try: # fL.send(self.connection, "CONTINUE") # logging.debug("Sent continue") chunkHandle = com[1] #fL.recv(self.connection) # get the name of the chunk logging.debug("recieved name of the chunk") open(chunkPath + "/" + chunkHandle, 'w').close() # create the file except IOError as e: logging.error(e) fL.send(self.connection, "FAILED") except Exception as e: logging.error(e) fL.send(self.connection, "FAILED") else: fL.send(self.connection, "CREATED") elif command == "APPEND": # append new data to a chunk try: # fL.send(self.connection, "CONTINUE") # logging.debug("sent continue #1") # chunkHandle = fL.recv(self.connection) # name of the chunk chunkHandle = com[1] # logging.debug("Recieved name of the chunk") # fL.send(self.connection, "CONTINUE") # logging.debug("Sent continue #2") # data = fL.recv(self.connection) # data being added data = "|".join(com[2:]) length = str(len(data)) logging.error(length) logging.debug("Recieved the data to be added") with open(chunkPath+"/"+chunkHandle, 'a') as a: # open the chunk #for item in data: a.write(data) # add the data to it except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection,"FAILED") logging.error(e) except Exception as e: fL.send(self.connection,"FAILED") logging.error(e) else: fL.send(self.connection,"SUCCESS") elif command == "SANITIZE": # recieves SANITIZE from the scrubber which tells the chunkserver to delete a chunk try: chunkHandle = com[1] # name of the chunk logging.debug("Recieved name of the chunk") try: os.remove(chunkPath + '/' + chunkHandle) # remove file from directory logging.debug("Removed chunk handle.") # If there is an error thrown that the chunk does not exist, we return success # because sanitize is called to remove a chunk from a location. If it does not # exist at that location, then removal is technially achieved. except OSError: fL.send(self.connection, "SUCCESS") logging.debug("chunk already did not exist. sending success") fL.send(self.connection, "SUCCESS") # send a success logging.debug("removal successfull!") except socket.error as e: logging.error(e) except IOError as e: fL.send(self.connection,"FAILED") logging.error(e) except Exception as e: fL.send(self.connection,"FAILED") logging.error(e) else: error = "Received invalid command: " + command logging.error(error)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Bind the listener-server s.bind(('', port)) ending = str(raw_input('Include proper EOT character in response? (y/n): ')).lower() # Listen for client connections s.listen(1) print "Listening..." # Accept the incoming connection conn, addr = s.accept() print "accepted connection" # Receive the data data = fL.recv(conn) # If data was received if data: print data # If the option to send back an eot was yes, send the message # with the eot if ending == "y": fL.send(conn, "Received successfully") # If the option to send back an eot was no, send the message # without the eot elif ending == "n": conn.send("Received successfully") # If no data was received if not data:
def create(self, filename): logging.debug("API: Starting create function.") #return an error if some wise guy tries to put a pipe in the file name. if "|" in filename: print "Invalid character, '|', in filename. No action taken." return 0 logging.debug("API: Creating socket.") # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send a CREATE request to the master try: logging.debug("API: Attempting to send CREATE| " + filename) fL.send(m, "CREATE|" + filename) except: logging.error("ERROR: COULD NOT SEND CREATE REQUEST TO MASTER") #receive data back from the master self.data = fL.recv(m) #error if the file trying to be created already exists logging.debug("API: Received message: " + self.data) if self.data == "FAIL1": print "THAT FILE EXISTS ALREADY... EXITING API" return 0 elif self.data == "FAIL2": print "NO SUCH FILE EXISTS FOR CHUNK CREATION" return 0 elif self.data == "FAIL3": print "CHUNK IS NOT THE LATEST CHUNK" return 0 #parse the received data into locations, and chunk handle self.splitdata = self.data.split("|") dataLength = len(self.splitdata) chunkHandle = self.splitdata[-1] global ack logging.debug("API: about to begin for loop, " + str(dataLength - 1) + "iterations") #iterate through each IP address received from the master for n in range(dataLength - 1): logging.debug("API: For loop, iteration number " + str(n)) #designate the IP for this iteration location = self.splitdata[n] #create a socket to be used to connect to chunk server s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #attempt to connect to the chunk server at the current location try: s.connect((location, self.TCP_PORT)) except: logging.error("ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location) continue #send CREATE request to the chunk server at the current location fL.send(s, "CREATE|" + chunkHandle) #wait to receive a CONTINUE from chunk server to proceed ack = fL.recv(s) #close connection to current chunk server. s.close() if ack == "FAILED": print "ERROR: FILE CREATION FAILED" fL.send(m, "FAILED") elif ack == "CREATED": print "File creation successful!" fL.send(m, "CREATED") return 1 m.close()
def interrogateChunkServer(self, IP, retry): logging.debug('Initialize interrogateChunkServer()') # Create an instance of a TCP socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Define the variable that will hold the the data received from the chunkservers data = " " try: s.connect((IP, chunkPort)) logging.debug('Connection Established: ' + str(IP) + ' on port ' + str(chunkPort)) # Request the chunk contents of the specified chunkserver fL.send(s, 'CONTENTS?') logging.debug('Sent chunkserver a CONTENTS? message') # Received the chunk contents of the speicified chunkserver data = fL.recv(s) # Close the socket connection s.close() # If the database is unable to connect to the chunkservers, retry the connection # If the retry fails, remove it from the list of activehosts and move on. except: # Make sure the socket connection is closed before doing anything, so if # a retry occurs, the socket will not already be in use. s.close() # heartBeat the chunkserver. If that indicates it is alive, try to # interrogate the chunkserver again. If not, remove if from the active # hosts list and move on. if retry < 3: # It the heartBeat indicated the chunkserver is still alive, try again. if hB.heartBeat(IP) == 1: self.intterogateChunkServer(IP, retry + 1) logging.debug( 'Retry connect to chunkserver for interrogation') else: self.remFromAhosts(IP) logging.warning( 'Heartbeat indicates chunkserver dead. Not interrogating, moving on.' ) return -1 else: self.remFromAhosts(IP) # Log the fact that we were unable to connect to a chunkserver logging.error("interrogateChunkServer failed to connect to " + IP) return -1 logging.debug('Received response from chunkserver') # If the IP is not already in the location lookup, add it! if IP not in self.locDict.keys(): self.locDict[IP] = [] # If the chunkserver has nothing on it, it should return whitespace. If this is the case, # then nothing in the database can be updated, so it will continue onto the next IP. # If the chunkserver returns something other than whitespace (a message formatted chunk1|chunk2|... ) # then the data can be processed. if data != " ": # Convert the pipe separated string into a list chunkData = data.split('|') # In the event that data is formatted poorly with additional | characters, we want to get rid of null elements chunkData = filter(None, chunkData) # For every chunk handle in that list, update that chunk objects locations list for chunk in chunkData: # If the IP key is not already in the location lookup, add it! if IP not in self.locDict.keys(): self.locDict[IP] = [] # Add the chunk to the list of values for the IP key self.locDict[IP].append(chunk) # If the location does already exist, append the current chunk to its list of chunk values else: assChnks = self.locDict[IP] # But first, make sure that chunk isn't already in the values, so you don't get # multiple copies of the same chunk in the lookup. if chunk not in assChnks: # Add the chunk to the list of values for the IP key self.locDict[IP].append(chunk) try: # Find which file the chunk is associated with in the lookup dictionary fileName = self.lookup[chunk] # From the file name we found the chunk to be associated with in the # lookup, we can append the current IP to the list of chunk locations # in the chunk object within the file object dictionary. if IP not in self.data[fileName].chunks[chunk].locations: self.data[fileName].chunks[chunk].locations.append(IP) logging.debug('Appended location to chunk ' + str(chunk) + ' location list') # If the chunk is not recognized in the master's database, the chunk is an orphan (does not # belong to a file). In this case, the chunk should be removed. except KeyError: # Create an instance of a TCP socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.connect((IP, chunkPort)) logging.debug('Connection Established: ' + str(IP) + ' on port ' + str(chunkPort)) # Tell the chunkserver to sanitize the chunk fL.send(s, 'SANITIZE|' + str(chunk)) logging.debug('Sent chunkserver a SANITIZE message') # Received the chunk contents of the speicified chunkserver data = fL.recv(s) if data == "SUCCESS": logging.debug("Orphan removal successful") self.locDict[IP].remove(chunk) elif data == "FAILED": logging.debug("Orphan removal failed.") # Close the socket connection s.close() logging.debug('interrogateChunkServer() complete')
def interrogateChunkServer(self, IP, retry): logging.debug('Initialize interrogateChunkServer()') # Create an instance of a TCP socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # Define the variable that will hold the the data received from the chunkservers data = " " try: s.connect((IP, chunkPort)) logging.debug('Connection Established: ' + str(IP) + ' on port ' + str(chunkPort)) # Request the chunk contents of the specified chunkserver fL.send(s, 'CONTENTS?') logging.debug('Sent chunkserver a CONTENTS? message') # Received the chunk contents of the speicified chunkserver data = fL.recv(s) # Close the socket connection s.close() # If the database is unable to connect to the chunkservers, retry the connection # If the retry fails, remove it from the list of activehosts and move on. except: # Make sure the socket connection is closed before doing anything, so if # a retry occurs, the socket will not already be in use. s.close() # heartBeat the chunkserver. If that indicates it is alive, try to # interrogate the chunkserver again. If not, remove if from the active # hosts list and move on. if retry < 3: # It the heartBeat indicated the chunkserver is still alive, try again. if hB.heartBeat(IP) == 1: self.intterogateChunkServer(IP, retry + 1) logging.debug('Retry connect to chunkserver for interrogation') else: self.remFromAhosts(IP) logging.warning('Heartbeat indicates chunkserver dead. Not interrogating, moving on.') return -1 else: self.remFromAhosts(IP) # Log the fact that we were unable to connect to a chunkserver logging.error("interrogateChunkServer failed to connect to " + IP) return -1 logging.debug('Received response from chunkserver') # If the IP is not already in the location lookup, add it! if IP not in self.locDict.keys(): self.locDict[IP] = [] # If the chunkserver has nothing on it, it should return whitespace. If this is the case, # then nothing in the database can be updated, so it will continue onto the next IP. # If the chunkserver returns something other than whitespace (a message formatted chunk1|chunk2|... ) # then the data can be processed. if data != " ": # Convert the pipe separated string into a list chunkData = data.split('|') # In the event that data is formatted poorly with additional | characters, we want to get rid of null elements chunkData = filter(None, chunkData) # For every chunk handle in that list, update that chunk objects locations list for chunk in chunkData: # If the IP key is not already in the location lookup, add it! if IP not in self.locDict.keys(): self.locDict[IP] = [] # Add the chunk to the list of values for the IP key self.locDict[IP].append(chunk) # If the location does already exist, append the current chunk to its list of chunk values else: assChnks = self.locDict[IP] # But first, make sure that chunk isn't already in the values, so you don't get # multiple copies of the same chunk in the lookup. if chunk not in assChnks: # Add the chunk to the list of values for the IP key self.locDict[IP].append(chunk) try: # Find which file the chunk is associated with in the lookup dictionary fileName = self.lookup[chunk] # From the file name we found the chunk to be associated with in the # lookup, we can append the current IP to the list of chunk locations # in the chunk object within the file object dictionary. if IP not in self.data[fileName].chunks[chunk].locations: self.data[fileName].chunks[chunk].locations.append(IP) logging.debug('Appended location to chunk ' + str(chunk) + ' location list') # If the chunk is not recognized in the master's database, the chunk is an orphan (does not # belong to a file). In this case, the chunk should be removed. except KeyError: # Create an instance of a TCP socket s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) s.connect((IP, chunkPort)) logging.debug('Connection Established: ' + str(IP) + ' on port ' + str(chunkPort)) # Tell the chunkserver to sanitize the chunk fL.send(s, 'SANITIZE|' + str(chunk)) logging.debug('Sent chunkserver a SANITIZE message') # Received the chunk contents of the speicified chunkserver data = fL.recv(s) if data == "SUCCESS": logging.debug("Orphan removal successful") self.locDict[IP].remove(chunk) elif data == "FAILED": logging.debug("Orphan removal failed.") # Close the socket connection s.close() logging.debug('interrogateChunkServer() complete')
def append(self, filename, newData,flag): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send APPEND request to master try: fL.send(m, "APPEND|" + filename) except: print "COULD NOT SEND APPEND REQUEST TO MASTER" return 0 #receive data back from master self.data = fL.recv(m) #close connection to master m.close() #some error handling if (self.data == "FAILED"): print "ERROR: MASTER SENT FAIL MESSAGE exiting..." return 0 elif (self.data == "OPEN"): print "ERROR: FILE " +filename+" ALREADY OPEN" return 0 #parse the data into useful parts self.splitdata = self.data.split("|") dataLength = len(self.splitdata) cH = self.splitdata[-1] #get length of the requested new data to use for append across chunks if flag == 1: with open(newData,"rb") as da: newData = da.read() dataSize = len(newData) lenNewData = int(dataSize) for n in range(0, dataLength-1): location = self.splitdata[n] #create socket to connect to chunk server at location self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #attempt to connect to chunk server at location try: self.s.connect((location,self.TCP_PORT)) except: print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location #ask chunk server how much room is left on latest chunk fL.send(self.s, "CHUNKSPACE?|" + cH) #the response is stored in remainingSpace remainingSpace = fL.recv(self.s) self.s.close() #some error handling if remainingSpace == "FAILED": print "CHUNKSPACE REQUEST FAILED. exiting..." return 0 #make remainingSpace an integer remainingSpace = int(remainingSpace) #if the length of the new data is greater than the room left in the chunk... if (lenNewData > remainingSpace): #...split the data into two parts, the first part being equal to the #amount of room left in the current chunk. the second part being the #rest of the data. cut = remainingSpace newData1 = newData[0:cut] print "Sending data of length:" + str(len(newData1)) newData2 = newData[cut:] #tell the chunk server to append the first part of the new data that #will fill up the rest of the remaining space on a chunk s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((location, self.TCP_PORT)) except: print "ERROR: COULD NOT REOPEN SOCKET" fL.send(s, "APPEND|" + cH + "|" + newData1) print "first append" SoF = fL.recv(s) #close connection to chunk server s.close() #error handling if SoF == "FAILED": print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..." return 0 elif (lenNewData <= remainingSpace): t = socket.socket(socket.AF_INET, socket.SOCK_STREAM) t.connect((location, self.TCP_PORT)) try: fL.send(t, "APPEND|" + cH + "|" + newData) except: print "ERROR: COULD NOT SEND APPEND TO CHUNK SERVER" SoF = fL.recv(t) t.close() #error handling/acks if SoF == "FAILED": print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..." return 0 ################### if lenNewData > remainingSpace: # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #tell the master to create a new chunk for the remaining data try: fL.send(m, "CREATECHUNK|" + filename + "|" + cH) except: print "ERROR: COULD NOT CREATE NEW CHUNK TO APPEND TO" #receive data back from master cData = fL.recv(m) #parse this data and handle it very similarly as the in the create function if self.data == "FAIL2": print "NO SUCH FILE EXISTS FOR CHUNK CREATION" exit(0) splitcData = cData.split("|") cDataLength = len(splitcData) cH = splitcData[-1] #close the connection to the master so we can connect to the chunk servers m.close() #iterate through each IP address received from the master for n in range(0, cDataLength-1): #create a socket to be used to connect to chunk server s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #designate the IP for this iteration location = splitcData[n] print location #attempt to connect to the chunk server at the current location try: s.connect((location,self.TCP_PORT)) except: print "ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location continue #send CREATE request to the chunk server at the current location fL.send(s, "CREATE|" + cH) global ack ack = fL.recv(s) ################ #close connection to current chunk server. s.close() #do some acks if ack == "FAILED": print "ERROR: CHUNK CREATION FAILED" #fL.send(m, "FAILED") elif ack == "CREATED": print "Chunk creation successful!" #fL.send(m, "CREATED") #m.close() #now that the new chunk has been created on all of the servers... #...run append again with the second part of the new data #self.s.close() try: self.append(filename, newData2,False) except UnboundLocalError: pass return 1
def append(self, filename, newData, flag): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send APPEND request to master try: fL.send(m, "APPEND|" + filename) except: print "COULD NOT SEND APPEND REQUEST TO MASTER" return 0 #receive data back from master self.data = fL.recv(m) #close connection to master m.close() #some error handling if (self.data == "FAILED"): print "ERROR: MASTER SENT FAIL MESSAGE exiting..." return 0 elif (self.data == "OPEN"): print "ERROR: FILE " + filename + " ALREADY OPEN" return 0 #parse the data into useful parts self.splitdata = self.data.split("|") dataLength = len(self.splitdata) cH = self.splitdata[-1] #get length of the requested new data to use for append across chunks if flag == 1: with open(newData, "rb") as da: newData = da.read() dataSize = len(newData) lenNewData = int(dataSize) for n in range(0, dataLength - 1): location = self.splitdata[n] #create socket to connect to chunk server at location self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #attempt to connect to chunk server at location try: self.s.connect((location, self.TCP_PORT)) except: print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location #ask chunk server how much room is left on latest chunk fL.send(self.s, "CHUNKSPACE?|" + cH) #the response is stored in remainingSpace remainingSpace = fL.recv(self.s) self.s.close() #some error handling if remainingSpace == "FAILED": print "CHUNKSPACE REQUEST FAILED. exiting..." return 0 #make remainingSpace an integer remainingSpace = int(remainingSpace) #if the length of the new data is greater than the room left in the chunk... if (lenNewData > remainingSpace): #...split the data into two parts, the first part being equal to the #amount of room left in the current chunk. the second part being the #rest of the data. cut = remainingSpace newData1 = newData[0:cut] print "Sending data of length:" + str(len(newData1)) newData2 = newData[cut:] #tell the chunk server to append the first part of the new data that #will fill up the rest of the remaining space on a chunk s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: s.connect((location, self.TCP_PORT)) except: print "ERROR: COULD NOT REOPEN SOCKET" fL.send(s, "APPEND|" + cH + "|" + newData1) print "first append" SoF = fL.recv(s) #close connection to chunk server s.close() #error handling if SoF == "FAILED": print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..." return 0 elif (lenNewData <= remainingSpace): t = socket.socket(socket.AF_INET, socket.SOCK_STREAM) t.connect((location, self.TCP_PORT)) try: fL.send(t, "APPEND|" + cH + "|" + newData) except: print "ERROR: COULD NOT SEND APPEND TO CHUNK SERVER" SoF = fL.recv(t) t.close() #error handling/acks if SoF == "FAILED": print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..." return 0 ################### if lenNewData > remainingSpace: # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #tell the master to create a new chunk for the remaining data try: fL.send(m, "CREATECHUNK|" + filename + "|" + cH) except: print "ERROR: COULD NOT CREATE NEW CHUNK TO APPEND TO" #receive data back from master cData = fL.recv(m) #parse this data and handle it very similarly as the in the create function if self.data == "FAIL2": print "NO SUCH FILE EXISTS FOR CHUNK CREATION" exit(0) splitcData = cData.split("|") cDataLength = len(splitcData) cH = splitcData[-1] #close the connection to the master so we can connect to the chunk servers m.close() #iterate through each IP address received from the master for n in range(0, cDataLength - 1): #create a socket to be used to connect to chunk server s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #designate the IP for this iteration location = splitcData[n] print location #attempt to connect to the chunk server at the current location try: s.connect((location, self.TCP_PORT)) except: print "ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location continue #send CREATE request to the chunk server at the current location fL.send(s, "CREATE|" + cH) global ack ack = fL.recv(s) ################ #close connection to current chunk server. s.close() #do some acks if ack == "FAILED": print "ERROR: CHUNK CREATION FAILED" #fL.send(m, "FAILED") elif ack == "CREATED": print "Chunk creation successful!" #fL.send(m, "CREATED") #m.close() #now that the new chunk has been created on all of the servers... #...run append again with the second part of the new data #self.s.close() try: self.append(filename, newData2, False) except UnboundLocalError: pass return 1
def create(self,filename): logging.debug("API: Starting create function.") #return an error if some wise guy tries to put a pipe in the file name. if "|" in filename: print "Invalid character, '|', in filename. No action taken." return 0 logging.debug("API: Creating socket.") # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send a CREATE request to the master try: logging.debug("API: Attempting to send CREATE| " + filename) fL.send(m, "CREATE|" + filename) except: logging.error("ERROR: COULD NOT SEND CREATE REQUEST TO MASTER") #receive data back from the master self.data = fL.recv(m) #error if the file trying to be created already exists logging.debug("API: Received message: " + self.data) if self.data == "FAIL1": print "THAT FILE EXISTS ALREADY... EXITING API" return 0 elif self.data == "FAIL2": print "NO SUCH FILE EXISTS FOR CHUNK CREATION" return 0 elif self.data == "FAIL3": print "CHUNK IS NOT THE LATEST CHUNK" return 0 #parse the received data into locations, and chunk handle self.splitdata = self.data.split("|") dataLength = len(self.splitdata) chunkHandle = self.splitdata[-1] global ack logging.debug("API: about to begin for loop, " + str(dataLength -1) + "iterations") #iterate through each IP address received from the master for n in range(dataLength-1): logging.debug("API: For loop, iteration number " + str(n)) #designate the IP for this iteration location = self.splitdata[n] #create a socket to be used to connect to chunk server s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) #attempt to connect to the chunk server at the current location try: s.connect((location,self.TCP_PORT)) except: logging.error("ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location) continue #send CREATE request to the chunk server at the current location fL.send(s, "CREATE|" + chunkHandle) #wait to receive a CONTINUE from chunk server to proceed ack = fL.recv(s) #close connection to current chunk server. s.close() if ack == "FAILED": print "ERROR: FILE CREATION FAILED" fL.send(m, "FAILED") elif ack == "CREATED": print "File creation successful!" fL.send(m, "CREATED") return 1 m.close()
def clean(self): logging.debug("SCRUBBER: Commencing Clean") # For all of the files in the toDelete list for fileName in self.data: # Create a TCP connection to the master self.connect(config.masterip, 0) try: # Request all the chunks associated with a given file fL.send(self.s, "GETALLCHUNKS|" + fileName) # Receive all the chunks associated with a file data = fL.recv(self.s) except (socket.timeout, socket.error) as e: logging.error("SCRUBBER: Connection to master failed with error: " + str(e) + ". Unable to continue for file: " + str(fileName)) # Set data to be empty so the deletion process will not continue to # try and delete a file it received no information about. data = "" # Be sure the socket is closed. self.s.close() # Convert the pipe-separated string of chunk handles into a list chunkHandles = data.split("|") # Ensure there are no "" elements in the list chunkHandles = filter(None, chunkHandles) # Create a counter for successful chunk deletions successfulChunkDelete = 0 # For each of those chunk handles for handle in chunkHandles: # Create a counter for successful deletions from a chunkserver successDeleteFromCS = 0 # Create a TCP connection to the master self.connect(config.masterip, 0) try: # Request all the locations associated with a given chunk fL.send(self.s, "GETLOCATIONS|" + handle) # Receive all the locations associated with a chunk data = fL.recv(self.s) except (socket.timeout, socket.error) as e: logging.error("SCRUBBER: Connection to master failed with error: " + str(e) + ". Unable to continue for chunk: " + str(handle)) # Set data to be empty so the deletion process will not continue to # try and delete a file it received no information about. data = "" # Be sure the socket is closed self.s.close() # Convert the pipe-separated string of locations into a list locations = data.split("|") # Ensure there are no "" elements in the list locations = filter(None, locations) # For each location the chunk is stored on for location in locations: # Send a SANITIZE message to a specified location data = self.cleanLocation(location, handle) # If the chunk server responds with a success message, increment the success counter if data == "SUCCESS": logging.debug("SCRUBBER: Chunk successfully removed from chunkserver") successDeleteFromCS += 1 # If the chunk server responds with a failure message, DO SOMETHING ELSE! elif data == "FAILED": retryAck = self.cleanLocation(location, handle) if retryAck == "SUCCESS": logging.debug("SCRUBBER: Chunk successfully removed from chunkserver") successDeleteFromCS += 1 elif retryAck == "FAILED": logging.error("SCRUBBER: Received failure message on chunk delete. Chunkhandle : " + str(handle)) else: logging.error("SCRUBBER: Unexpected Receive: " + str(data) + " from chunkserver " + str(location)) # If the chunk server responds with something other than SUCCESS or FAILED, something went wrong. else: logging.error("SCRUBBER: Unexpected Receive: " + str(data) + " from chunkserver " + str(location)) # If the success counter is equal to the amount of all the IPs, then # all the IPs successfully deleted that chunk, so increment the # successfulChunkDelete counter if len(locations) == successDeleteFromCS: successfulChunkDelete += 1 else: # Improve error handling to maybe automatically retry logging.error("SCRUBBER: Not all chunk location deletes were successful") # If the number of successful chunk deletes is equal to the number of chunks # associated with the file, then all the chunks for that file have been deleted, # so the file entry can be deleted if len(chunkHandles) == successfulChunkDelete: # Call the database sanitize function, which removes the key/value pair # from the database. self.connect(config.masterip, 0) fL.send(self.s, "SANITIZE|" + fileName) #data = fL.recv(self.s) self.s.close() logging.debug("SCRUBBER: " + str(fileName) + 'successfully sanitized') else: # Improve error handling to automatically resolve problem logging.error("SCRUBBER: Not all chunk deletes were successful")