def getAllLocations(self):
locations = database.getChunkLocations(self.msg[1])
msg = ""
for item in locations:
msg += item + "|"
fL.send(self.s, msg)
def getAllLocations(self):
locations = database.getChunkLocations(self.msg[1])
msg = ""
for item in locations:
msg += item + "|"
fL.send(self.s, msg)
def delete(self, filename):
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#send DELETE request to the master
try:
fL.send(m, "DELETE|" + filename)
except:
print "ERROR: COULD NOT SEND DELETE REQUEST TO MASTER"
#receive acks from the master
self.data = fL.recv(m)
#tell the user whether the file was successfully marked or not
if self.data == "FAILED1":
print "ERROR: The file could not be marked for deletion."
return -1
elif self.data == "FAILED2":
print "ERROR: The given file name does not exist."
return -2
elif self.data == "FAILED3":
print "The file has already been marked for deletion."
return -3
elif self.data == "MARKED":
print "File successfully marked for deletion."
m.close()
def delete(self, filename): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send DELETE request to the master try: fL.send(m, "DELETE|" + filename) except: print "ERROR: COULD NOT SEND DELETE REQUEST TO MASTER" #receive acks from the master self.data = fL.recv(m) #tell the user whether the file was successfully marked or not if self.data == "FAILED1": print "ERROR: The file could not be marked for deletion." return -1 elif self.data == "FAILED2": print "ERROR: The given file name does not exist." return -2 elif self.data == "FAILED3": print "The file has already been marked for deletion." return -3 elif self.data == "MARKED": print "File successfully marked for deletion." m.close()
def append(self):
# Visual confirmation for debugging: confirm init of append()
logging.debug('Gathering metadata for chunk append')
# We know that we will only be appending to the lastest chunk, since a new
# chunk should only be created when an old chunk fills up, so we find the
# handle of the latest chunk for a given file.
latestChunkHandle = database.findLatestChunk(self.fileName)
# Then we get the locations where that chunk is stored
locations = database.getChunkLocations(latestChunkHandle)
# Define an empty string that will hold the message we send back to the client
appendMessage = ''
# Parse the locations list into a pipe separated string
for item in locations:
appendMessage += item + '|'
# Add the chunk handle to the message we will send to the client
appendMessage += str(latestChunkHandle)
#send our message
fL.send(self.s, appendMessage)
# Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle
logging.debug('SENT == ' + str(appendMessage))
# Visual confirmation for debugging: confirm success of append()
logging.debug('Append successfully handled')
def createChunk(self):
self.lock.acquire()
chunkHandle = database.getChunkHandle()
self.lock.release()
newChunk = database.createNewChunk(self.msg[1], self.msg[2],
chunkHandle)
fL.send(self.s, newChunk)
def append(self):
# Visual confirmation for debugging: confirm init of append()
logging.debug("Gathering metadata for chunk append")
# We know that we will only be appending to the lastest chunk, since a new
# chunk should only be created when an old chunk fills up, so we find the
# handle of the latest chunk for a given file.
latestChunkHandle = database.findLatestChunk(self.fileName)
# Then we get the locations where that chunk is stored
locations = database.getChunkLocations(latestChunkHandle)
# Define an empty string that will hold the message we send back to the client
appendMessage = ""
# Parse the locations list into a pipe separated string
for item in locations:
appendMessage += item + "|"
# Add the chunk handle to the message we will send to the client
appendMessage += str(latestChunkHandle)
# send our message
fL.send(self.s, appendMessage)
# Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle
logging.debug("SENT == " + str(appendMessage))
# Visual confirmation for debugging: confirm success of append()
logging.debug("Append successfully handled")
def getDeleteData(self):
# Get the list of files from the database
toDelete = database.toDelete
# Create an empty message which will hold the data
msg = ""
# For each item in the list, add it to the message string
for item in toDelete:
msg += item + "|"
# Send back the string of delete data
fL.send(self.s, msg)
def getDeleteData(self):
# Get the list of files from the database
toDelete = database.toDelete
# Create an empty message which will hold the data
msg = ""
# For each item in the list, add it to the message string
for item in toDelete:
msg += item + "|"
# Send back the string of delete data
fL.send(self.s, msg)
def read(self, filename, byteOffset, bytesToRead, newName):
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#send READ request to the master
try:
fL.send(
m, "READ|" + filename + "|" + str(byteOffset) + "|" +
str(bytesToRead))
except:
print "ERROR: COULD NOT SEND READ REQUEST TO MASTER"
#recieve data from the master
self.data = fL.recv(m)
#close connection to master
m.close()
#split the data into a list
self.splitdata = self.data.split("|")
#remove the first element of the list because it is irrelevant
self.splitdata = self.splitdata[1:]
#iterate through the list
fromChunks = ""
for q in self.splitdata:
#split the list into smaller parts
secondSplit = q.split("*")
#set the location...
location = secondSplit[0]
#...and the chunk handle
cH = secondSplit[1]
#...and the offset
offset = secondSplit[2]
#connect to the chunk server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((location, self.TCP_PORT))
except:
print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location
return 0
#send READ request to chunk server
fL.send(
s,
"READ|" + str(cH) + "|" + str(offset) + "|" + str(bytesToRead))
#receive and print the contents of the file
#fromChunks += "." + str(cH)
dat = fL.recv(s)
#close connection to chunk server
s.close()
with open(newName, "ab") as e:
e.write(dat)
return 1
def read(self, filename, byteOffset, bytesToRead, newName):
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#send READ request to the master
try:
fL.send(m, "READ|" + filename + "|" + str(byteOffset) + "|" + str(bytesToRead))
except:
print "ERROR: COULD NOT SEND READ REQUEST TO MASTER"
#recieve data from the master
self.data = fL.recv(m)
#close connection to master
m.close()
#split the data into a list
self.splitdata = self.data.split("|")
#remove the first element of the list because it is irrelevant
self.splitdata = self.splitdata[1:]
#iterate through the list
fromChunks = ""
for q in self.splitdata:
#split the list into smaller parts
secondSplit = q.split("*")
#set the location...
location = secondSplit[0]
#...and the chunk handle
cH = secondSplit[1]
#...and the offset
offset = secondSplit[2]
#connect to the chunk server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((location,self.TCP_PORT))
except:
print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location
return 0
#send READ request to chunk server
fL.send(s, "READ|" + str(cH) + "|" + str(offset) + "|" + str(bytesToRead))
#receive and print the contents of the file
#fromChunks += "." + str(cH)
dat = fL.recv(s)
#close connection to chunk server
s.close()
with open(newName,"ab") as e:
e.write(dat)
return 1
def delete(self):
logging.debug("Begin updating delete flag to True")
# Make sure the file is not already marked for delete
if self.fileName not in database.toDelete:
# Make sure the file you wish to delete is actually a file in the system
if self.fileName in database.data.keys():
try:
# Change the delete flag for the specified file
database.flagDelete(self.fileName)
# Confirm that file has been marked for deletion
fL.send(self.s, "MARKED")
logging.debug("Delete Flags Updated")
except:
logging.error("File could not be marked for deletion")
fL.send(self.s, "FAILED1")
else:
logging.debug("The file, " + self.fileName + ", does not exist.")
fL.send(self.s, "FAILED2")
else:
logging.debug("The file, " + self.fileName + ", is already marked for delete")
fL.send(self.s, "FAILED3")
def delete(self):
logging.debug('Begin updating delete flag to True')
# Make sure the file is not already marked for delete
if self.fileName not in database.toDelete:
# Make sure the file you wish to delete is actually a file in the system
if self.fileName in database.data.keys():
try:
# Change the delete flag for the specified file
database.flagDelete(self.fileName)
# Confirm that file has been marked for deletion
fL.send(self.s, "MARKED")
logging.debug('Delete Flags Updated')
except:
logging.error("File could not be marked for deletion")
fL.send(self.s, "FAILED1")
else:
logging.debug('The file, ' + self.fileName +
', does not exist.')
fL.send(self.s, "FAILED2")
else:
logging.debug('The file, ' + self.fileName +
', is already marked for delete')
fL.send(self.s, "FAILED3")
def create(self):
# Visual confirmation for debugging: confirm init of create()
logging.debug('Creating chunk metadata')
# Acquire a lock so the chunk handle counter can not be accessed simultaneously
self.lock.acquire()
# Get a new chunkhandle
chunkHandle = database.getChunkHandle()
# Release the lock so others can access the chunk handle counter
self.lock.release()
# Create a new file, and store the return flag
createFileFlag = database.createNewFile(self.fileName, chunkHandle)
# If the return flag was an error flag, alert the logger and API of the error
if createFileFlag == -1:
logging.error("Got a duplicate file name, sending FAIL to API")
fL.send(self.s, "FAIL1")
return -1
elif createFileFlag == -2:
logging.error("No file exists for a chunk to be created for")
fL.send(self.s, "FAIL2")
elif createFileFlag == -3:
logging.error(
"Chunk is not the latest chunk. New chunk has been created that can be appended to."
)
fL.send(self.s, "FAIL3")
# Get the locations for a specified chunk
locations = database.data[self.fileName].chunks[chunkHandle].locations
# Parse the locations list retreived above into a pipe-separated list
hosts = ""
for item in locations:
hosts += item + "|"
# Visual confirmation for debugging: confirm success of create()
logging.debug('Chunk metadata successfully created')
try:
# Send the API a string containing the location and chunkHandle information
fL.send(self.s, str(hosts) + str(chunkHandle))
except socket.error:
logging.warning('Socket Connection Broken')
# Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle
logging.debug('SENT ==> ' + str(hosts) + str(chunkHandle))
# Receieve an ack to affirm that the chunk was successfully created
ack = fL.recv(self.s)
if ack == "CREATED":
logging.debug("successful creation")
elif ack == "FAILED":
logging.error("unsuccessful creation")
else:
logging.error("unknown ack")
def heartBeat(self, IP):
logging.debug('HEARTBEAT: Initiating heartBeat protocol')
# Get the list of all active chunk server IPs
activeServers = self.getActiveChunkServers()
try:
# Create a TCP socket instance
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Set the timeout of the socket
self.s.settimeout(self.SOCK_TIMEOUT)
# Allow the socket to re-use address
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Connect to the chunkserver over the specified port
self.s.connect((IP, self.PORT))
# Send the chunk server a heart (ping)
fL.send(self.s, "<3?")
# Get the chunk server response
data = fL.recv(self.s)
print data
# Close the connection to allow future connections
self.s.close()
# If the chunk server responds with a heart, add it to activehosts
if data == "<3!":
# If the chunkserver IP is not in the activeServers list, add it to the list
if IP not in activeServers:
with open(self.AHOSTS, "a") as file:
file.write(IP + "\n")
# Return a 1 so any script calling heartBeat will know the heartbeat was successful
return 1
# Handle the timeout (chunk server alive but not responding) and connection (server dead) errors
except (socket.timeout, socket.error):
print "</3"
logging.debug('HEARTBEAT: Could not connect to ' + IP)
# Check to see if the chunkserver is in the list of active IPs
if IP in activeServers:
# If it is, remove it from the list
activeServers.remove(IP)
# Clear the previous activehosts.txt file and replace it with the list of active servers,
# which now excludes the failed chunkserver
with open(self.AHOSTS, "w") as file:
newList = ""
for item in activeServers:
newList += item + "\n"
file.write(newList)
# Return a -1 so any script calling heartbeat will know the heartbeat failed.
return -1
def create(self):
# Visual confirmation for debugging: confirm init of create()
logging.debug("Creating chunk metadata")
# Acquire a lock so the chunk handle counter can not be accessed simultaneously
self.lock.acquire()
# Get a new chunkhandle
chunkHandle = database.getChunkHandle()
# Release the lock so others can access the chunk handle counter
self.lock.release()
# Create a new file, and store the return flag
createFileFlag = database.createNewFile(self.fileName, chunkHandle)
# If the return flag was an error flag, alert the logger and API of the error
if createFileFlag == -1:
logging.error("Got a duplicate file name, sending FAIL to API")
fL.send(self.s, "FAIL1")
return -1
elif createFileFlag == -2:
logging.error("No file exists for a chunk to be created for")
fL.send(self.s, "FAIL2")
elif createFileFlag == -3:
logging.error("Chunk is not the latest chunk. New chunk has been created that can be appended to.")
fL.send(self.s, "FAIL3")
# Get the locations for a specified chunk
locations = database.data[self.fileName].chunks[chunkHandle].locations
# Parse the locations list retreived above into a pipe-separated list
hosts = ""
for item in locations:
hosts += item + "|"
# Visual confirmation for debugging: confirm success of create()
logging.debug("Chunk metadata successfully created")
try:
# Send the API a string containing the location and chunkHandle information
fL.send(self.s, str(hosts) + str(chunkHandle))
except socket.error:
logging.warning("Socket Connection Broken")
# Visual confirmation for debugging: confirm send of a list of storage hosts and chunk handle
logging.debug("SENT ==> " + str(hosts) + str(chunkHandle))
# Receieve an ack to affirm that the chunk was successfully created
ack = fL.recv(self.s)
if ack == "CREATED":
logging.debug("successful creation")
elif ack == "FAILED":
logging.error("unsuccessful creation")
else:
logging.error("unknown ack")
def heartBeat(self, IP):
logging.debug('HEARTBEAT: Initiating heartBeat protocol')
# Get the list of all active chunk server IPs
activeServers = self.getActiveChunkServers()
try:
# Create a TCP socket instance
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Set the timeout of the socket
self.s.settimeout(self.SOCK_TIMEOUT)
# Allow the socket to re-use address
self.s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Connect to the chunkserver over the specified port
self.s.connect((IP, self.PORT))
# Send the chunk server a heart (ping)
fL.send(self.s, "<3?")
# Get the chunk server response
data = fL.recv(self.s)
print data
# Close the connection to allow future connections
self.s.close()
# If the chunk server responds with a heart, add it to activehosts
if data == "<3!":
# If the chunkserver IP is not in the activeServers list, add it to the list
if IP not in activeServers:
with open(self.AHOSTS, "a") as file:
file.write(IP + "\n")
# Return a 1 so any script calling heartBeat will know the heartbeat was successful
return 1
# Handle the timeout (chunk server alive but not responding) and connection (server dead) errors
except (socket.timeout, socket.error):
print "</3"
logging.debug('HEARTBEAT: Could not connect to ' + IP)
# Check to see if the chunkserver is in the list of active IPs
if IP in activeServers:
# If it is, remove it from the list
activeServers.remove(IP)
# Clear the previous activehosts.txt file and replace it with the list of active servers,
# which now excludes the failed chunkserver
with open(self.AHOSTS, "w") as file:
newList = ""
for item in activeServers:
newList += item + "\n"
file.write(newList)
# Return a -1 so any script calling heartbeat will know the heartbeat failed.
return -1
def fileList(self):
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
try:
# Send the master a request for the list of files the system contains
fL.send(m, "FILELIST|x")
# Recieve the file list from the master
data = fL.recv(m)
# Close the TCP connection to the master
m.close()
return data
except Exception as e:
raise e
logging.error("Unable to get file list from master.")
return 0
def fileList(self):
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
try:
# Send the master a request for the list of files the system contains
fL.send(m, "FILELIST|x")
# Recieve the file list from the master
data = fL.recv(m)
# Close the TCP connection to the master
m.close()
return data
except Exception as e:
raise e
logging.error("Unable to get file list from master.")
return 0
def cleanLocation(self, location, handle):
# Connect to the chunk server
self.connect(location, 0)
try:
# Send the chunk server a SANITIZE message with the chunk handle
# so it knows which chunk it is deleting
fL.send(self.s, 'SANITIZE|' + str(handle))
logging.debug('SCRUBBER: Sent SANITIZE message to chunkserver')
# Wait for a response back from the chunk server to verify that
# the chunk was removed
data = fL.recv(self.s)
logging.debug('SCRUBBER: Received response from chunkserver')
return data
except (socket.timeout, socket.error) as e:
logging.error("SCRUBBER: Connection to chunkserver failed with error: " + str(e) + ". Unable to continue for location: " + str(location))
# Set data to be empty so the deletion process will not continue to
# try and delete a file it received no information about.
data = ""
return data
def undelete(self, filename):
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#send UNDELETE request to master
try:
fL.send(m, "UNDELETE|" + filename)
except:
print "ERROR COULD NOT SEND UNDELETE REQUEST TO MASTER"
#receive acks from the master
self.data = fL.recv(m)
#tell the user whether the file was successfully unmarked or not
if self.data == "FAILED1":
print "ERROR: COULD NOT UNDELETE FILE"
return -1
elif self.data == "FAILED2":
print "File was not flagged for deletion."
return -2
elif self.data == "MARKED":
print "File successfully unmarked for deletion."
m.close()
def undelete(self, filename): # Create socket connection to the master m = self.createSocket() # if there was an error making the socket, exit the function if not m: return 0 #send UNDELETE request to master try: fL.send(m, "UNDELETE|" + filename) except: print "ERROR COULD NOT SEND UNDELETE REQUEST TO MASTER" #receive acks from the master self.data = fL.recv(m) #tell the user whether the file was successfully unmarked or not if self.data == "FAILED1": print "ERROR: COULD NOT UNDELETE FILE" return -1 elif self.data == "FAILED2": print "File was not flagged for deletion." return -2 elif self.data == "MARKED": print "File successfully unmarked for deletion." m.close()
def undelete(self):
logging.debug("Begin updating delete flag to False")
# Make sure the file is already marked for delete
if self.fileName in database.toDelete:
try:
# Change the delete flag for the specified file
database.flagUndelete(self.fileName)
# Confirm that file has been marked for deletion
fL.send(self.s, "MARKED")
logging.debug("Delete Flags Updated")
except:
logging.error("File could not be unmarked for deletion")
fL.send(self.s, "FAILED1")
# If the file is not already marked for delete, you can't undelete it..
else:
logging.debug("The file, " + self.fileName + ", was not marked for deletion to begin with.")
fL.send(self.s, "FAILED2")
def undelete(self):
logging.debug('Begin updating delete flag to False')
# Make sure the file is already marked for delete
if self.fileName in database.toDelete:
try:
# Change the delete flag for the specified file
database.flagUndelete(self.fileName)
# Confirm that file has been marked for deletion
fL.send(self.s, "MARKED")
logging.debug('Delete Flags Updated')
except:
logging.error("File could not be unmarked for deletion")
fL.send(self.s, "FAILED1")
# If the file is not already marked for delete, you can't undelete it..
else:
logging.debug('The file, ' + self.fileName +
', was not marked for deletion to begin with.')
fL.send(self.s, "FAILED2")
def getAllChunks(self):
chunks = database.allChunks(self.fileName)
fL.send(self.s, chunks)
def fileNames(self):
names = str(database.getFileNames())
fL.send(self.s, names)
import socket, config
import functionLibrary as fL
port = int(raw_input('What port to connect over? : '))
# Define a socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Allow socket address reuse
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Connect to the server over the specified IP and port
s.connect(('', port))
message = str(raw_input('What message would you like to send? : '))
ending = str(raw_input('Include proper EOT character? (y/n): ')).lower()
# Send the message with a proper EOT termination
if ending == "y":
fL.send(s, message)
# Send the message without the proper EOT termination
elif ending == "n":
s.send(message)
# Receive data back from the server
data = fL.recv(s)
# Print the data to console
print "DATA == ", data
s.close()
def createChunk(self):
self.lock.acquire()
chunkHandle = database.getChunkHandle()
self.lock.release()
newChunk = database.createNewChunk(self.msg[1], self.msg[2], chunkHandle)
fL.send(self.s, newChunk)
#try:
# Create a TCP socket instance
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Set the timeout of the socket
s.settimeout(3)
# Allow the socket to re-use address
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Connect to the chunkserver over the specified port
s.connect((config.masterip, config.port))
#except:
# print "CONNECTION ERROR"
# errorList.append('SCRUBBER - Unable to connect to master')
print "connected"
fL.send(s, "GETDELETEDATA|*")
data = fL.recv(s)
s.close()
print "recvd to delete list"
toDelete = []
for item in data.split("|"):
if item != "":
toDelete.append(item)
# Create an instance of the Scrubber object, and initiate it.
scrub = scrubber.Scrubber(toDelete)
scrubStart = time.time()
# Listen for client connections
s.listen(1)
print "Listening..."
# Accept the incoming connection
conn, addr = s.accept()
print "accepted connection"
# Receive the data
data = fL.recv(conn)
# If data was received
if data:
print data
# If the option to send back an eot was yes, send the message
# with the eot
if ending == "y":
fL.send(conn, "Received successfully")
# If the option to send back an eot was no, send the message
# without the eot
elif ending == "n":
conn.send("Received successfully")
# If no data was received
if not data:
print "No data received!"
# If the option to send back an eot was yes, send the message
# with the eot
if ending == "y":
fL.send(conn, "Received unsuccessfully")
# If the option to send back an eot was no, send the message
# without the eot
elif ending == "n":
def run(self):
c = fL.recv(self.connection) # listens for a command on
# the connection handed
# down from the main
# thread
com = c.split(
'|') # if c has multiple parts, they will be seperated by
# pipes. This will put each part into a list
command = com[
0] # the command should be the first part of the message,
# thus the first part of the list
logging.debug("Recieved command " + command)
# next the distributor hands the connection to the appropriate
# worker based on the command given, invalid commands simply fall
# through
if command == "<3?":
# heartbeat to see if each chunkserver is running. If it is, it will
# send back a confirmation of <3!
try:
# in this and each other if/elif statement the correct
# worker thread is started for a given command
fL.send(self.connection, "<3!")
logging.debug("Send heart beat back")
self.connection.close()
logging.debug("Closed connection.")
except socket.error as e:
logging.error(e)
elif command == "CHUNKSPACE?":
try:
# fL.send(self.connection, "CONTINUE") # after receiving the connection
# # the thread confirms that it is
# # ready to receive arguments
# logging.debug("send a continue")
# chunkHandle = fL.recv(self.connection) # it listens on its
# # connection for a chunkhandle
chunkHandle = com[1] # name of the chunkhandle
logging.debug("recieved name of the chunkhandle: " +
chunkHandle)
emptySpace = str(
mg64 - os.stat(chunkPath + "/" +
chunkHandle).st_size) # then checks the
# difference
# between the
# file's size and
# 64mg (the max
# chunk size)
fL.send(self.connection,
str(emptySpace)) # and returns the amount of space
# left to the API
logging.debug("Send the space remaining")
# self.connection.close() # closes the connection
# logging.debug("Closed the connection")
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection, "FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection, "FAILED")
logging.error(e)
elif command == "READ":
# read data from a chunk
try:
# fL.send(self.connection, "CONTINUE") # confirms readiness for data
# logging.debug("sent continue #1")
# chunkHandle = fL.recv(self.connection) # listens for chunkHandle
chunkHandle = com[1]
# logging.debug("recieved name of the chunkhandle: " + chunkHandle)
# fL.send(self.connection, "CONTINUE") # confirms ready state
# logging.debug("sent continue #2")
byteOffSet = int(com[2]) # listens for a byte
# offset to read from
# (relative to the
# beginning of the
# given chunk)
# byteOffSet
# logging.debug("recieved the byte offset number.")
# fL.send(self.connection, "CONTINUE") # confirms the desire for EVEN MORE data
# logging.debug("sent continue #3")
bytesToRead = int(
com[3]) #int(fL.recv(self.connection)) # listens for the
# number of bytes to read
logging.debug("recieved the number of bytes to read")
chunk = open(
chunkPath + "/" +
chunkHandle) # opens the designated chunk to read from
chunk.seek(
int(byteOffSet)) # goes to the specified byte offset
fileContent = chunk.read(
bytesToRead) # stuffs all the stuff to be
# read into a variable
fL.send(self.connection, fileContent)
logging.debug("send the file content")
chunk.close() # closes the chunk
logging.debug("closed the connection")
self.connection.close() # closes the connection
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection, "FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection, "FAILED")
logging.error(e)
elif command == "CONTENTS?":
try:
output = ""
for files in os.walk(chunkPath): # read every file
output = str('|'.join(
item
for item in files[-1])) # turn the list into a string
print output
if output == "": # if there is nothing in the dir
print "output is empty"
fL.send(self.connection, " ") # send an empty string
logging.debug(
"Sent an empty string which should be the output")
else: # otherwise
print "output is not empty"
fL.send(self.connection,
output) # send everything as a string
logging.debug("sent the output")
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection, "FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection, "FAILED")
logging.error(e)
elif command == "CREATE":
# create a new chunk
try:
# fL.send(self.connection, "CONTINUE")
# logging.debug("Sent continue")
chunkHandle = com[
1] #fL.recv(self.connection) # get the name of the chunk
logging.debug("recieved name of the chunk")
open(chunkPath + "/" + chunkHandle,
'w').close() # create the file
except IOError as e:
logging.error(e)
fL.send(self.connection, "FAILED")
except Exception as e:
logging.error(e)
fL.send(self.connection, "FAILED")
else:
fL.send(self.connection, "CREATED")
elif command == "APPEND":
# append new data to a chunk
try:
# fL.send(self.connection, "CONTINUE")
# logging.debug("sent continue #1")
# chunkHandle = fL.recv(self.connection) # name of the chunk
chunkHandle = com[1]
# logging.debug("Recieved name of the chunk")
# fL.send(self.connection, "CONTINUE")
# logging.debug("Sent continue #2")
# data = fL.recv(self.connection) # data being added
data = "|".join(com[2:])
length = str(len(data))
logging.error(length)
logging.debug("Recieved the data to be added")
with open(chunkPath + "/" + chunkHandle,
'a') as a: # open the chunk
#for item in data:
a.write(data) # add the data to it
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection, "FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection, "FAILED")
logging.error(e)
else:
fL.send(self.connection, "SUCCESS")
elif command == "SANITIZE":
# recieves SANITIZE from the scrubber which tells the chunkserver to delete a chunk
try:
chunkHandle = com[1] # name of the chunk
logging.debug("Recieved name of the chunk")
try:
os.remove(chunkPath + '/' +
chunkHandle) # remove file from directory
logging.debug("Removed chunk handle.")
# If there is an error thrown that the chunk does not exist, we return success
# because sanitize is called to remove a chunk from a location. If it does not
# exist at that location, then removal is technially achieved.
except OSError:
fL.send(self.connection, "SUCCESS")
logging.debug(
"chunk already did not exist. sending success")
fL.send(self.connection, "SUCCESS") # send a success
logging.debug("removal successfull!")
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection, "FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection, "FAILED")
logging.error(e)
else:
error = "Received invalid command: " + command
logging.error(error)
def create(self, filename):
logging.debug("API: Starting create function.")
#return an error if some wise guy tries to put a pipe in the file name.
if "|" in filename:
print "Invalid character, '|', in filename. No action taken."
return 0
logging.debug("API: Creating socket.")
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#send a CREATE request to the master
try:
logging.debug("API: Attempting to send CREATE| " + filename)
fL.send(m, "CREATE|" + filename)
except:
logging.error("ERROR: COULD NOT SEND CREATE REQUEST TO MASTER")
#receive data back from the master
self.data = fL.recv(m)
#error if the file trying to be created already exists
logging.debug("API: Received message: " + self.data)
if self.data == "FAIL1":
print "THAT FILE EXISTS ALREADY... EXITING API"
return 0
elif self.data == "FAIL2":
print "NO SUCH FILE EXISTS FOR CHUNK CREATION"
return 0
elif self.data == "FAIL3":
print "CHUNK IS NOT THE LATEST CHUNK"
return 0
#parse the received data into locations, and chunk handle
self.splitdata = self.data.split("|")
dataLength = len(self.splitdata)
chunkHandle = self.splitdata[-1]
global ack
logging.debug("API: about to begin for loop, " + str(dataLength - 1) +
"iterations")
#iterate through each IP address received from the master
for n in range(dataLength - 1):
logging.debug("API: For loop, iteration number " + str(n))
#designate the IP for this iteration
location = self.splitdata[n]
#create a socket to be used to connect to chunk server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#attempt to connect to the chunk server at the current location
try:
s.connect((location, self.TCP_PORT))
except:
logging.error("ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ",
location)
continue
#send CREATE request to the chunk server at the current location
fL.send(s, "CREATE|" + chunkHandle)
#wait to receive a CONTINUE from chunk server to proceed
ack = fL.recv(s)
#close connection to current chunk server.
s.close()
if ack == "FAILED":
print "ERROR: FILE CREATION FAILED"
fL.send(m, "FAILED")
elif ack == "CREATED":
print "File creation successful!"
fL.send(m, "CREATED")
return 1
m.close()
TCP_PORT2 = 9666
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((TCP_ADDRESS, TCP_PORT1))
s.listen(1)
while 1:
conn, addr = s.accept()
data = fL.recv(conn)
print data
splitdata = data.split("|")
#s.connect((API_ADDRESS, TCP_PORT2))
if splitdata[0] == "CREATE":
fL.send(conn, "10.10.117.104|01")
elif splitdata[0] == "APPEND":
fL.send(conn, "10.10.117.104|01")
elif splitdata[0] == "READ":
fL.send(conn, "READ|10.10.117.104*01*0")
elif (splitdata[1] == 01):
fL.send(conn, "poop")
elif splitdata[0] == "CHUNKSPACE?":
fL.send(conn, "10")
elif splitdata[0] == "CREATECHUNK":
fL.send(conn, "10.10.117.104|01")
elif splitdata[0] == "DELETE":
fL.send(conn, "MARKED")
elif splitdata[0] == "UNDELETE":
fL.send(conn, "MARKED")
def create(self,filename):
logging.debug("API: Starting create function.")
#return an error if some wise guy tries to put a pipe in the file name.
if "|" in filename:
print "Invalid character, '|', in filename. No action taken."
return 0
logging.debug("API: Creating socket.")
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#send a CREATE request to the master
try:
logging.debug("API: Attempting to send CREATE| " + filename)
fL.send(m, "CREATE|" + filename)
except:
logging.error("ERROR: COULD NOT SEND CREATE REQUEST TO MASTER")
#receive data back from the master
self.data = fL.recv(m)
#error if the file trying to be created already exists
logging.debug("API: Received message: " + self.data)
if self.data == "FAIL1":
print "THAT FILE EXISTS ALREADY... EXITING API"
return 0
elif self.data == "FAIL2":
print "NO SUCH FILE EXISTS FOR CHUNK CREATION"
return 0
elif self.data == "FAIL3":
print "CHUNK IS NOT THE LATEST CHUNK"
return 0
#parse the received data into locations, and chunk handle
self.splitdata = self.data.split("|")
dataLength = len(self.splitdata)
chunkHandle = self.splitdata[-1]
global ack
logging.debug("API: about to begin for loop, " + str(dataLength -1) + "iterations")
#iterate through each IP address received from the master
for n in range(dataLength-1):
logging.debug("API: For loop, iteration number " + str(n))
#designate the IP for this iteration
location = self.splitdata[n]
#create a socket to be used to connect to chunk server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#attempt to connect to the chunk server at the current location
try:
s.connect((location,self.TCP_PORT))
except:
logging.error("ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location)
continue
#send CREATE request to the chunk server at the current location
fL.send(s, "CREATE|" + chunkHandle)
#wait to receive a CONTINUE from chunk server to proceed
ack = fL.recv(s)
#close connection to current chunk server.
s.close()
if ack == "FAILED":
print "ERROR: FILE CREATION FAILED"
fL.send(m, "FAILED")
elif ack == "CREATED":
print "File creation successful!"
fL.send(m, "CREATED")
return 1
m.close()
def interrogateChunkServer(self, IP, retry):
logging.debug('Initialize interrogateChunkServer()')
# Create an instance of a TCP socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Define the variable that will hold the the data received from the chunkservers
data = " "
try:
s.connect((IP, chunkPort))
logging.debug('Connection Established: ' + str(IP) + ' on port ' +
str(chunkPort))
# Request the chunk contents of the specified chunkserver
fL.send(s, 'CONTENTS?')
logging.debug('Sent chunkserver a CONTENTS? message')
# Received the chunk contents of the speicified chunkserver
data = fL.recv(s)
# Close the socket connection
s.close()
# If the database is unable to connect to the chunkservers, retry the connection
# If the retry fails, remove it from the list of activehosts and move on.
except:
# Make sure the socket connection is closed before doing anything, so if
# a retry occurs, the socket will not already be in use.
s.close()
# heartBeat the chunkserver. If that indicates it is alive, try to
# interrogate the chunkserver again. If not, remove if from the active
# hosts list and move on.
if retry < 3:
# It the heartBeat indicated the chunkserver is still alive, try again.
if hB.heartBeat(IP) == 1:
self.intterogateChunkServer(IP, retry + 1)
logging.debug(
'Retry connect to chunkserver for interrogation')
else:
self.remFromAhosts(IP)
logging.warning(
'Heartbeat indicates chunkserver dead. Not interrogating, moving on.'
)
return -1
else:
self.remFromAhosts(IP)
# Log the fact that we were unable to connect to a chunkserver
logging.error("interrogateChunkServer failed to connect to " +
IP)
return -1
logging.debug('Received response from chunkserver')
# If the IP is not already in the location lookup, add it!
if IP not in self.locDict.keys():
self.locDict[IP] = []
# If the chunkserver has nothing on it, it should return whitespace. If this is the case,
# then nothing in the database can be updated, so it will continue onto the next IP.
# If the chunkserver returns something other than whitespace (a message formatted chunk1|chunk2|... )
# then the data can be processed.
if data != " ":
# Convert the pipe separated string into a list
chunkData = data.split('|')
# In the event that data is formatted poorly with additional | characters, we want to get rid of null elements
chunkData = filter(None, chunkData)
# For every chunk handle in that list, update that chunk objects locations list
for chunk in chunkData:
# If the IP key is not already in the location lookup, add it!
if IP not in self.locDict.keys():
self.locDict[IP] = []
# Add the chunk to the list of values for the IP key
self.locDict[IP].append(chunk)
# If the location does already exist, append the current chunk to its list of chunk values
else:
assChnks = self.locDict[IP]
# But first, make sure that chunk isn't already in the values, so you don't get
# multiple copies of the same chunk in the lookup.
if chunk not in assChnks:
# Add the chunk to the list of values for the IP key
self.locDict[IP].append(chunk)
try:
# Find which file the chunk is associated with in the lookup dictionary
fileName = self.lookup[chunk]
# From the file name we found the chunk to be associated with in the
# lookup, we can append the current IP to the list of chunk locations
# in the chunk object within the file object dictionary.
if IP not in self.data[fileName].chunks[chunk].locations:
self.data[fileName].chunks[chunk].locations.append(IP)
logging.debug('Appended location to chunk ' +
str(chunk) + ' location list')
# If the chunk is not recognized in the master's database, the chunk is an orphan (does not
# belong to a file). In this case, the chunk should be removed.
except KeyError:
# Create an instance of a TCP socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.connect((IP, chunkPort))
logging.debug('Connection Established: ' + str(IP) +
' on port ' + str(chunkPort))
# Tell the chunkserver to sanitize the chunk
fL.send(s, 'SANITIZE|' + str(chunk))
logging.debug('Sent chunkserver a SANITIZE message')
# Received the chunk contents of the speicified chunkserver
data = fL.recv(s)
if data == "SUCCESS":
logging.debug("Orphan removal successful")
self.locDict[IP].remove(chunk)
elif data == "FAILED":
logging.debug("Orphan removal failed.")
# Close the socket connection
s.close()
logging.debug('interrogateChunkServer() complete')
def fileList(self):
# call the database object's returnData method
list = str(database.getFiles())
fL.send(self.s, list)
def fileNames(self):
names = str(database.getFileNames())
fL.send(self.s, names)
def append(self, filename, newData, flag):
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#send APPEND request to master
try:
fL.send(m, "APPEND|" + filename)
except:
print "COULD NOT SEND APPEND REQUEST TO MASTER"
return 0
#receive data back from master
self.data = fL.recv(m)
#close connection to master
m.close()
#some error handling
if (self.data == "FAILED"):
print "ERROR: MASTER SENT FAIL MESSAGE exiting..."
return 0
elif (self.data == "OPEN"):
print "ERROR: FILE " + filename + " ALREADY OPEN"
return 0
#parse the data into useful parts
self.splitdata = self.data.split("|")
dataLength = len(self.splitdata)
cH = self.splitdata[-1]
#get length of the requested new data to use for append across chunks
if flag == 1:
with open(newData, "rb") as da:
newData = da.read()
dataSize = len(newData)
lenNewData = int(dataSize)
for n in range(0, dataLength - 1):
location = self.splitdata[n]
#create socket to connect to chunk server at location
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#attempt to connect to chunk server at location
try:
self.s.connect((location, self.TCP_PORT))
except:
print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location
#ask chunk server how much room is left on latest chunk
fL.send(self.s, "CHUNKSPACE?|" + cH)
#the response is stored in remainingSpace
remainingSpace = fL.recv(self.s)
self.s.close()
#some error handling
if remainingSpace == "FAILED":
print "CHUNKSPACE REQUEST FAILED. exiting..."
return 0
#make remainingSpace an integer
remainingSpace = int(remainingSpace)
#if the length of the new data is greater than the room left in the chunk...
if (lenNewData > remainingSpace):
#...split the data into two parts, the first part being equal to the
#amount of room left in the current chunk. the second part being the
#rest of the data.
cut = remainingSpace
newData1 = newData[0:cut]
print "Sending data of length:" + str(len(newData1))
newData2 = newData[cut:]
#tell the chunk server to append the first part of the new data that
#will fill up the rest of the remaining space on a chunk
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((location, self.TCP_PORT))
except:
print "ERROR: COULD NOT REOPEN SOCKET"
fL.send(s, "APPEND|" + cH + "|" + newData1)
print "first append"
SoF = fL.recv(s)
#close connection to chunk server
s.close()
#error handling
if SoF == "FAILED":
print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..."
return 0
elif (lenNewData <= remainingSpace):
t = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
t.connect((location, self.TCP_PORT))
try:
fL.send(t, "APPEND|" + cH + "|" + newData)
except:
print "ERROR: COULD NOT SEND APPEND TO CHUNK SERVER"
SoF = fL.recv(t)
t.close()
#error handling/acks
if SoF == "FAILED":
print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..."
return 0
###################
if lenNewData > remainingSpace:
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#tell the master to create a new chunk for the remaining data
try:
fL.send(m, "CREATECHUNK|" + filename + "|" + cH)
except:
print "ERROR: COULD NOT CREATE NEW CHUNK TO APPEND TO"
#receive data back from master
cData = fL.recv(m)
#parse this data and handle it very similarly as the in the create function
if self.data == "FAIL2":
print "NO SUCH FILE EXISTS FOR CHUNK CREATION"
exit(0)
splitcData = cData.split("|")
cDataLength = len(splitcData)
cH = splitcData[-1]
#close the connection to the master so we can connect to the chunk servers
m.close()
#iterate through each IP address received from the master
for n in range(0, cDataLength - 1):
#create a socket to be used to connect to chunk server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#designate the IP for this iteration
location = splitcData[n]
print location
#attempt to connect to the chunk server at the current location
try:
s.connect((location, self.TCP_PORT))
except:
print "ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location
continue
#send CREATE request to the chunk server at the current location
fL.send(s, "CREATE|" + cH)
global ack
ack = fL.recv(s)
################
#close connection to current chunk server.
s.close()
#do some acks
if ack == "FAILED":
print "ERROR: CHUNK CREATION FAILED"
#fL.send(m, "FAILED")
elif ack == "CREATED":
print "Chunk creation successful!"
#fL.send(m, "CREATED")
#m.close()
#now that the new chunk has been created on all of the servers...
#...run append again with the second part of the new data
#self.s.close()
try:
self.append(filename, newData2, False)
except UnboundLocalError:
pass
return 1
def run(self): if self.role == 1: print "1" self.socket.connect((self.remoteAddr,self.port)) print "2" fL.send(self.socket, "<3?") print "3" print fL.recv(self.socket) print "4" elif self.role == 2: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "CHUNKSPACE?|"+chunkHandle) print "chunkspace :" + fL.recv(self.socket) elif self.role == 3: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "READ|"+chunkHandle+"|"+byteOffSet+"|"+bytesToRead) print "Reading.... " + fL.recv(self.socket) elif self.role == 4: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "CONTENTS?") print "contents :" + fL.recv(self.socket) elif self.role == 5: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "CREATE|"+chunkHandle) print fL.recv(self.socket) elif self.role == 6: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "APPEND|"+chunkHandle+"|"+appendData) elif self.role == 7: self.socket.connect((self.remoteAddr,self.port)) fL.send(self.socket, "SANITIZE|"+chunkHandle) print fL.recv(self.socket)
def read(self):
# Get the byte offset and bytes to read from the received message
try:
byteOffset = int(self.msg[2])
bytesToRead = int(self.msg[3])
# If there is an index error (the values do not exist in the message)
# alert the client and end the read function.
except IndexError:
fL.send(self.s, "READ command not given proper parameters")
return
logging.debug('parsed byte offset and bytes to read')
# Get the size of a chunk from the config file
maxChunkSize = config.chunkSize
# Find the sequence of the chunk in the given file by using integer division
# (divide and take the floor)
startSequence = byteOffset // maxChunkSize
if startSequence > len(database.data[self.fileName].chunks.keys()):
logging.debug('No such byte offset exists for the given file')
fL.send(self.s, "FAILED, NO SUCH BYTE OFFSET EXISTS FOR THIS FILE")
return
# Get the offset of the read-start within its given chunk
chunkByteOffset = byteOffset % maxChunkSize
logging.debug('start sequence # == ' + str(startSequence))
logging.debug('chunk byte offset == ' + str(chunkByteOffset))
# If the user inputs a bytes to read of -1, the read will go until the end
# of the file.
if bytesToRead == -1:
# The ending offset will be the max file size
endOffset = maxChunkSize
# The end sequence can be found be knowing how many chunks a file has, and
# subtracting by 1 because the sequence numbers start at 0, not 1.
endSequence = len(database.data[self.fileName].chunks.keys()) - 1
else:
# To find where the user wants to read ending, add the number of bytes they want
# to read to their starting point, the byteOffest. This will give the byte offset
# of the end of the read
endReadOffset = byteOffset + bytesToRead
# Find the sequence of the chunk in which the end of the read will terminate
endSequence = endReadOffset // maxChunkSize
# Get the offset of the read-end within its given chunk
endOffset = endReadOffset % maxChunkSize
logging.debug('end sequence # == ' + str(endSequence))
logging.debug('end read offset == ' + str(endOffset))
# Create an empty string to hold the message that will be sent to the client
responseMessage = "READFROM"
# For each sequence number that exists between (and including) the read-start chunk
# and the read-end chunk, get the file's chunk with the appropriate sequence number,
# and append to the response message, a location it is stored at, its chunk handle,
# the byte offset from within that chunk to begin reading from, and the byte offset
# to end reading from
for sequence in range(startSequence, (endSequence + 1)):
# try:
# Get the chunkHandles associated with the given file, and sort the chunkHandles from
# least to greatest in the list. This will put them in their sequence order where the
# 0th element is now the 0th sequence, 1st element the 1st sequence, etc.
logging.debug(sorted(database.data[self.fileName].chunks.keys()))
associatedChunkHandles = sorted(
database.data[self.fileName].chunks.keys())
# Append a location of where the start-sequence chunk is stored to the message
logging.debug(database.data[self.fileName].chunks[
associatedChunkHandles[sequence]].locations)
responseMessage += "|" + str(database.data[self.fileName].chunks[
associatedChunkHandles[sequence]].locations[0])
# Append the chunk handle to the message
responseMessage += "*" + str(associatedChunkHandles[sequence])
# Append the byte offset to start reading from to the message
responseMessage += "*" + str(chunkByteOffset)
# If there are multiple chunks that will be read over, the next chunk will start
# the read from the beginning
chunkByteOffset = 0
# Check to see if the READ will take place in the same chunk. If it does, append the
# endOffset to the message so the client will know where to end reading
if startSequence == endSequence:
responseMessage += "*" + str(endOffset)
# If the READ takes place over multiple chunks, write the end of read for the current
# chunk to be the end of the chunk, and then increase the start sequence number so when the
# metadata for the last chunk is processed, it will be caught by the if statement above
# and send the appropriate ending offset.
elif startSequence < endSequence:
responseMessage += "*" + str(maxChunkSize)
startSequence += 1
# except:
# logging.error("Unable to generate proper READ response message.")
logging.debug('RESPONSE MESSAGE == ' + str(responseMessage))
#send our message
fL.send(self.s, responseMessage)
logging.debug('SENT == ' + responseMessage)
# Visual confirmation for debugging: confirm success of read()
logging.debug('Read successfully handled')
TCP_PORT2 = 9666
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((TCP_ADDRESS, TCP_PORT1))
s.listen(1)
while 1:
conn, addr = s.accept()
data = fL.recv(conn)
print data
splitdata = data.split("|")
#s.connect((API_ADDRESS, TCP_PORT2))
if splitdata[0] == "CREATE":
fL.send(conn, "10.10.117.104|01")
elif splitdata[0] == "APPEND":
fL.send(conn, "10.10.117.104|01")
elif splitdata[0] == "READ":
fL.send(conn, "READ|10.10.117.104*01*0")
elif (splitdata[1] == 01):
fL.send(conn, "poop")
elif splitdata[0] == "CHUNKSPACE?":
fL.send(conn, "10")
elif splitdata[0] == "CREATECHUNK":
fL.send(conn, "10.10.117.104|01")
elif splitdata[0] == "DELETE":
fL.send(conn, "MARKED")
elif splitdata[0] == "UNDELETE":
fL.send(conn, "MARKED")
def interrogateChunkServer(self, IP, retry):
logging.debug('Initialize interrogateChunkServer()')
# Create an instance of a TCP socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Define the variable that will hold the the data received from the chunkservers
data = " "
try:
s.connect((IP, chunkPort))
logging.debug('Connection Established: ' + str(IP) + ' on port ' + str(chunkPort))
# Request the chunk contents of the specified chunkserver
fL.send(s, 'CONTENTS?')
logging.debug('Sent chunkserver a CONTENTS? message')
# Received the chunk contents of the speicified chunkserver
data = fL.recv(s)
# Close the socket connection
s.close()
# If the database is unable to connect to the chunkservers, retry the connection
# If the retry fails, remove it from the list of activehosts and move on.
except:
# Make sure the socket connection is closed before doing anything, so if
# a retry occurs, the socket will not already be in use.
s.close()
# heartBeat the chunkserver. If that indicates it is alive, try to
# interrogate the chunkserver again. If not, remove if from the active
# hosts list and move on.
if retry < 3:
# It the heartBeat indicated the chunkserver is still alive, try again.
if hB.heartBeat(IP) == 1:
self.intterogateChunkServer(IP, retry + 1)
logging.debug('Retry connect to chunkserver for interrogation')
else:
self.remFromAhosts(IP)
logging.warning('Heartbeat indicates chunkserver dead. Not interrogating, moving on.')
return -1
else:
self.remFromAhosts(IP)
# Log the fact that we were unable to connect to a chunkserver
logging.error("interrogateChunkServer failed to connect to " + IP)
return -1
logging.debug('Received response from chunkserver')
# If the IP is not already in the location lookup, add it!
if IP not in self.locDict.keys():
self.locDict[IP] = []
# If the chunkserver has nothing on it, it should return whitespace. If this is the case,
# then nothing in the database can be updated, so it will continue onto the next IP.
# If the chunkserver returns something other than whitespace (a message formatted chunk1|chunk2|... )
# then the data can be processed.
if data != " ":
# Convert the pipe separated string into a list
chunkData = data.split('|')
# In the event that data is formatted poorly with additional | characters, we want to get rid of null elements
chunkData = filter(None, chunkData)
# For every chunk handle in that list, update that chunk objects locations list
for chunk in chunkData:
# If the IP key is not already in the location lookup, add it!
if IP not in self.locDict.keys():
self.locDict[IP] = []
# Add the chunk to the list of values for the IP key
self.locDict[IP].append(chunk)
# If the location does already exist, append the current chunk to its list of chunk values
else:
assChnks = self.locDict[IP]
# But first, make sure that chunk isn't already in the values, so you don't get
# multiple copies of the same chunk in the lookup.
if chunk not in assChnks:
# Add the chunk to the list of values for the IP key
self.locDict[IP].append(chunk)
try:
# Find which file the chunk is associated with in the lookup dictionary
fileName = self.lookup[chunk]
# From the file name we found the chunk to be associated with in the
# lookup, we can append the current IP to the list of chunk locations
# in the chunk object within the file object dictionary.
if IP not in self.data[fileName].chunks[chunk].locations:
self.data[fileName].chunks[chunk].locations.append(IP)
logging.debug('Appended location to chunk ' + str(chunk) + ' location list')
# If the chunk is not recognized in the master's database, the chunk is an orphan (does not
# belong to a file). In this case, the chunk should be removed.
except KeyError:
# Create an instance of a TCP socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.connect((IP, chunkPort))
logging.debug('Connection Established: ' + str(IP) + ' on port ' + str(chunkPort))
# Tell the chunkserver to sanitize the chunk
fL.send(s, 'SANITIZE|' + str(chunk))
logging.debug('Sent chunkserver a SANITIZE message')
# Received the chunk contents of the speicified chunkserver
data = fL.recv(s)
if data == "SUCCESS":
logging.debug("Orphan removal successful")
self.locDict[IP].remove(chunk)
elif data == "FAILED":
logging.debug("Orphan removal failed.")
# Close the socket connection
s.close()
logging.debug('interrogateChunkServer() complete')
#try:
# Create a TCP socket instance
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Set the timeout of the socket
s.settimeout(3)
# Allow the socket to re-use address
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Connect to the chunkserver over the specified port
s.connect((config.masterip, config.port))
#except:
# print "CONNECTION ERROR"
# errorList.append('SCRUBBER - Unable to connect to master')
print "connected"
fL.send(s, "GETDELETEDATA|*")
data = fL.recv(s)
s.close()
print "recvd to delete list"
toDelete = []
for item in data.split("|"):
if item != "":
toDelete.append(item)
# Create an instance of the Scrubber object, and initiate it.
scrub = scrubber.Scrubber(toDelete)
scrubStart = time.time()
# Listen for client connections
s.listen(1)
print "Listening..."
# Accept the incoming connection
conn, addr = s.accept()
print "accepted connection"
# Receive the data
data = fL.recv(conn)
# If data was received
if data:
print data
# If the option to send back an eot was yes, send the message
# with the eot
if ending == "y":
fL.send(conn, "Received successfully")
# If the option to send back an eot was no, send the message
# without the eot
elif ending == "n":
conn.send("Received successfully")
# If no data was received
if not data:
print "No data received!"
# If the option to send back an eot was yes, send the message
# with the eot
if ending == "y":
fL.send(conn, "Received unsuccessfully")
# If the option to send back an eot was no, send the message
# without the eot
elif ending == "n":
def append(self, filename, newData,flag):
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#send APPEND request to master
try:
fL.send(m, "APPEND|" + filename)
except:
print "COULD NOT SEND APPEND REQUEST TO MASTER"
return 0
#receive data back from master
self.data = fL.recv(m)
#close connection to master
m.close()
#some error handling
if (self.data == "FAILED"):
print "ERROR: MASTER SENT FAIL MESSAGE exiting..."
return 0
elif (self.data == "OPEN"):
print "ERROR: FILE " +filename+" ALREADY OPEN"
return 0
#parse the data into useful parts
self.splitdata = self.data.split("|")
dataLength = len(self.splitdata)
cH = self.splitdata[-1]
#get length of the requested new data to use for append across chunks
if flag == 1:
with open(newData,"rb") as da:
newData = da.read()
dataSize = len(newData)
lenNewData = int(dataSize)
for n in range(0, dataLength-1):
location = self.splitdata[n]
#create socket to connect to chunk server at location
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#attempt to connect to chunk server at location
try:
self.s.connect((location,self.TCP_PORT))
except:
print "ERROR: COULD NOT CONNECT TO CHUNK SERVER AT ", location
#ask chunk server how much room is left on latest chunk
fL.send(self.s, "CHUNKSPACE?|" + cH)
#the response is stored in remainingSpace
remainingSpace = fL.recv(self.s)
self.s.close()
#some error handling
if remainingSpace == "FAILED":
print "CHUNKSPACE REQUEST FAILED. exiting..."
return 0
#make remainingSpace an integer
remainingSpace = int(remainingSpace)
#if the length of the new data is greater than the room left in the chunk...
if (lenNewData > remainingSpace):
#...split the data into two parts, the first part being equal to the
#amount of room left in the current chunk. the second part being the
#rest of the data.
cut = remainingSpace
newData1 = newData[0:cut]
print "Sending data of length:" + str(len(newData1))
newData2 = newData[cut:]
#tell the chunk server to append the first part of the new data that
#will fill up the rest of the remaining space on a chunk
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
s.connect((location, self.TCP_PORT))
except:
print "ERROR: COULD NOT REOPEN SOCKET"
fL.send(s, "APPEND|" + cH + "|" + newData1)
print "first append"
SoF = fL.recv(s)
#close connection to chunk server
s.close()
#error handling
if SoF == "FAILED":
print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..."
return 0
elif (lenNewData <= remainingSpace):
t = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
t.connect((location, self.TCP_PORT))
try:
fL.send(t, "APPEND|" + cH + "|" + newData)
except:
print "ERROR: COULD NOT SEND APPEND TO CHUNK SERVER"
SoF = fL.recv(t)
t.close()
#error handling/acks
if SoF == "FAILED":
print "ERROR WITH APPEND ON CHUNK SERVER SIDE. exiting..."
return 0
###################
if lenNewData > remainingSpace:
# Create socket connection to the master
m = self.createSocket()
# if there was an error making the socket, exit the function
if not m:
return 0
#tell the master to create a new chunk for the remaining data
try:
fL.send(m, "CREATECHUNK|" + filename + "|" + cH)
except:
print "ERROR: COULD NOT CREATE NEW CHUNK TO APPEND TO"
#receive data back from master
cData = fL.recv(m)
#parse this data and handle it very similarly as the in the create function
if self.data == "FAIL2":
print "NO SUCH FILE EXISTS FOR CHUNK CREATION"
exit(0)
splitcData = cData.split("|")
cDataLength = len(splitcData)
cH = splitcData[-1]
#close the connection to the master so we can connect to the chunk servers
m.close()
#iterate through each IP address received from the master
for n in range(0, cDataLength-1):
#create a socket to be used to connect to chunk server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#designate the IP for this iteration
location = splitcData[n]
print location
#attempt to connect to the chunk server at the current location
try:
s.connect((location,self.TCP_PORT))
except:
print "ERROR: COULD NOT CONNECT TO CHUNKSERVER AT ", location
continue
#send CREATE request to the chunk server at the current location
fL.send(s, "CREATE|" + cH)
global ack
ack = fL.recv(s)
################
#close connection to current chunk server.
s.close()
#do some acks
if ack == "FAILED":
print "ERROR: CHUNK CREATION FAILED"
#fL.send(m, "FAILED")
elif ack == "CREATED":
print "Chunk creation successful!"
#fL.send(m, "CREATED")
#m.close()
#now that the new chunk has been created on all of the servers...
#...run append again with the second part of the new data
#self.s.close()
try:
self.append(filename, newData2,False)
except UnboundLocalError:
pass
return 1
def run(self):
c = fL.recv(self.connection) # listens for a command on
# the connection handed
# down from the main
# thread
com = c.split('|') # if c has multiple parts, they will be seperated by
# pipes. This will put each part into a list
command = com[0] # the command should be the first part of the message,
# thus the first part of the list
logging.debug("Recieved command " + command)
# next the distributor hands the connection to the appropriate
# worker based on the command given, invalid commands simply fall
# through
if command == "<3?":
# heartbeat to see if each chunkserver is running. If it is, it will
# send back a confirmation of <3!
try:
# in this and each other if/elif statement the correct
# worker thread is started for a given command
fL.send(self.connection, "<3!")
logging.debug("Send heart beat back")
self.connection.close()
logging.debug("Closed connection.")
except socket.error as e:
logging.error(e)
elif command == "CHUNKSPACE?":
try:
# fL.send(self.connection, "CONTINUE") # after receiving the connection
# # the thread confirms that it is
# # ready to receive arguments
# logging.debug("send a continue")
# chunkHandle = fL.recv(self.connection) # it listens on its
# # connection for a chunkhandle
chunkHandle = com[1] # name of the chunkhandle
logging.debug("recieved name of the chunkhandle: " + chunkHandle)
emptySpace = str(mg64 - os.stat(chunkPath + "/" + chunkHandle).st_size) # then checks the
# difference
# between the
# file's size and
# 64mg (the max
# chunk size)
fL.send(self.connection, str(emptySpace)) # and returns the amount of space
# left to the API
logging.debug("Send the space remaining")
# self.connection.close() # closes the connection
# logging.debug("Closed the connection")
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection,"FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection,"FAILED")
logging.error(e)
elif command == "READ":
# read data from a chunk
try:
# fL.send(self.connection, "CONTINUE") # confirms readiness for data
# logging.debug("sent continue #1")
# chunkHandle = fL.recv(self.connection) # listens for chunkHandle
chunkHandle = com[1]
# logging.debug("recieved name of the chunkhandle: " + chunkHandle)
# fL.send(self.connection, "CONTINUE") # confirms ready state
# logging.debug("sent continue #2")
byteOffSet = int(com[2]) # listens for a byte
# offset to read from
# (relative to the
# beginning of the
# given chunk)
# byteOffSet
# logging.debug("recieved the byte offset number.")
# fL.send(self.connection, "CONTINUE") # confirms the desire for EVEN MORE data
# logging.debug("sent continue #3")
bytesToRead = int(com[3]) #int(fL.recv(self.connection)) # listens for the
# number of bytes to read
logging.debug("recieved the number of bytes to read")
chunk = open(chunkPath+"/"+chunkHandle) # opens the designated chunk to read from
chunk.seek(int(byteOffSet)) # goes to the specified byte offset
fileContent = chunk.read(bytesToRead) # stuffs all the stuff to be
# read into a variable
fL.send(self.connection, fileContent)
logging.debug("send the file content")
chunk.close() # closes the chunk
logging.debug("closed the connection")
self.connection.close() # closes the connection
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection,"FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection,"FAILED")
logging.error(e)
elif command == "CONTENTS?":
try:
output = ""
for files in os.walk(chunkPath): # read every file
output = str('|'.join(item for item in files[-1])) # turn the list into a string
print output
if output == "": # if there is nothing in the dir
print "output is empty"
fL.send(self.connection, " ") # send an empty string
logging.debug("Sent an empty string which should be the output")
else: # otherwise
print "output is not empty"
fL.send(self.connection, output) # send everything as a string
logging.debug("sent the output")
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection,"FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection,"FAILED")
logging.error(e)
elif command == "CREATE":
# create a new chunk
try:
# fL.send(self.connection, "CONTINUE")
# logging.debug("Sent continue")
chunkHandle = com[1] #fL.recv(self.connection) # get the name of the chunk
logging.debug("recieved name of the chunk")
open(chunkPath + "/" + chunkHandle, 'w').close() # create the file
except IOError as e:
logging.error(e)
fL.send(self.connection, "FAILED")
except Exception as e:
logging.error(e)
fL.send(self.connection, "FAILED")
else:
fL.send(self.connection, "CREATED")
elif command == "APPEND":
# append new data to a chunk
try:
# fL.send(self.connection, "CONTINUE")
# logging.debug("sent continue #1")
# chunkHandle = fL.recv(self.connection) # name of the chunk
chunkHandle = com[1]
# logging.debug("Recieved name of the chunk")
# fL.send(self.connection, "CONTINUE")
# logging.debug("Sent continue #2")
# data = fL.recv(self.connection) # data being added
data = "|".join(com[2:])
length = str(len(data))
logging.error(length)
logging.debug("Recieved the data to be added")
with open(chunkPath+"/"+chunkHandle, 'a') as a: # open the chunk
#for item in data:
a.write(data) # add the data to it
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection,"FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection,"FAILED")
logging.error(e)
else:
fL.send(self.connection,"SUCCESS")
elif command == "SANITIZE":
# recieves SANITIZE from the scrubber which tells the chunkserver to delete a chunk
try:
chunkHandle = com[1] # name of the chunk
logging.debug("Recieved name of the chunk")
try:
os.remove(chunkPath + '/' + chunkHandle) # remove file from directory
logging.debug("Removed chunk handle.")
# If there is an error thrown that the chunk does not exist, we return success
# because sanitize is called to remove a chunk from a location. If it does not
# exist at that location, then removal is technially achieved.
except OSError:
fL.send(self.connection, "SUCCESS")
logging.debug("chunk already did not exist. sending success")
fL.send(self.connection, "SUCCESS") # send a success
logging.debug("removal successfull!")
except socket.error as e:
logging.error(e)
except IOError as e:
fL.send(self.connection,"FAILED")
logging.error(e)
except Exception as e:
fL.send(self.connection,"FAILED")
logging.error(e)
else:
error = "Received invalid command: " + command
logging.error(error)
def fileList(self):
# call the database object's returnData method
list = str(database.getFiles())
fL.send(self.s, list)
port = int(raw_input('What port to connect over? : '))
# Define a socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Allow socket address reuse
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Connect to the server over the specified IP and port
s.connect(('', port))
message = str(raw_input('What message would you like to send? : '))
ending = str(raw_input('Include proper EOT character? (y/n): ')).lower()
# Send the message with a proper EOT termination
if ending == "y":
fL.send(s, message)
# Send the message without the proper EOT termination
elif ending == "n":
s.send(message)
# Receive data back from the server
data = fL.recv(s)
# Print the data to console
print "DATA == ", data
s.close()
def getAllChunks(self):
chunks = database.allChunks(self.fileName)
fL.send(self.s, chunks)
def clean(self):
logging.debug("SCRUBBER: Commencing Clean")
# For all of the files in the toDelete list
for fileName in self.data:
# Create a TCP connection to the master
self.connect(config.masterip, 0)
try:
# Request all the chunks associated with a given file
fL.send(self.s, "GETALLCHUNKS|" + fileName)
# Receive all the chunks associated with a file
data = fL.recv(self.s)
except (socket.timeout, socket.error) as e:
logging.error("SCRUBBER: Connection to master failed with error: " + str(e) + ". Unable to continue for file: " + str(fileName))
# Set data to be empty so the deletion process will not continue to
# try and delete a file it received no information about.
data = ""
# Be sure the socket is closed.
self.s.close()
# Convert the pipe-separated string of chunk handles into a list
chunkHandles = data.split("|")
# Ensure there are no "" elements in the list
chunkHandles = filter(None, chunkHandles)
# Create a counter for successful chunk deletions
successfulChunkDelete = 0
# For each of those chunk handles
for handle in chunkHandles:
# Create a counter for successful deletions from a chunkserver
successDeleteFromCS = 0
# Create a TCP connection to the master
self.connect(config.masterip, 0)
try:
# Request all the locations associated with a given chunk
fL.send(self.s, "GETLOCATIONS|" + handle)
# Receive all the locations associated with a chunk
data = fL.recv(self.s)
except (socket.timeout, socket.error) as e:
logging.error("SCRUBBER: Connection to master failed with error: " + str(e) + ". Unable to continue for chunk: " + str(handle))
# Set data to be empty so the deletion process will not continue to
# try and delete a file it received no information about.
data = ""
# Be sure the socket is closed
self.s.close()
# Convert the pipe-separated string of locations into a list
locations = data.split("|")
# Ensure there are no "" elements in the list
locations = filter(None, locations)
# For each location the chunk is stored on
for location in locations:
# Send a SANITIZE message to a specified location
data = self.cleanLocation(location, handle)
# If the chunk server responds with a success message, increment the success counter
if data == "SUCCESS":
logging.debug("SCRUBBER: Chunk successfully removed from chunkserver")
successDeleteFromCS += 1
# If the chunk server responds with a failure message, DO SOMETHING ELSE!
elif data == "FAILED":
retryAck = self.cleanLocation(location, handle)
if retryAck == "SUCCESS":
logging.debug("SCRUBBER: Chunk successfully removed from chunkserver")
successDeleteFromCS += 1
elif retryAck == "FAILED":
logging.error("SCRUBBER: Received failure message on chunk delete. Chunkhandle : " + str(handle))
else:
logging.error("SCRUBBER: Unexpected Receive: " + str(data) + " from chunkserver " + str(location))
# If the chunk server responds with something other than SUCCESS or FAILED, something went wrong.
else:
logging.error("SCRUBBER: Unexpected Receive: " + str(data) + " from chunkserver " + str(location))
# If the success counter is equal to the amount of all the IPs, then
# all the IPs successfully deleted that chunk, so increment the
# successfulChunkDelete counter
if len(locations) == successDeleteFromCS:
successfulChunkDelete += 1
else:
# Improve error handling to maybe automatically retry
logging.error("SCRUBBER: Not all chunk location deletes were successful")
# If the number of successful chunk deletes is equal to the number of chunks
# associated with the file, then all the chunks for that file have been deleted,
# so the file entry can be deleted
if len(chunkHandles) == successfulChunkDelete:
# Call the database sanitize function, which removes the key/value pair
# from the database.
self.connect(config.masterip, 0)
fL.send(self.s, "SANITIZE|" + fileName)
#data = fL.recv(self.s)
self.s.close()
logging.debug("SCRUBBER: " + str(fileName) + 'successfully sanitized')
else:
# Improve error handling to automatically resolve problem
logging.error("SCRUBBER: Not all chunk deletes were successful")
def read(self):
# Get the byte offset and bytes to read from the received message
try:
byteOffset = int(self.msg[2])
bytesToRead = int(self.msg[3])
# If there is an index error (the values do not exist in the message)
# alert the client and end the read function.
except IndexError:
fL.send(self.s, "READ command not given proper parameters")
return
logging.debug("parsed byte offset and bytes to read")
# Get the size of a chunk from the config file
maxChunkSize = config.chunkSize
# Find the sequence of the chunk in the given file by using integer division
# (divide and take the floor)
startSequence = byteOffset // maxChunkSize
if startSequence > len(database.data[self.fileName].chunks.keys()):
logging.debug("No such byte offset exists for the given file")
fL.send(self.s, "FAILED, NO SUCH BYTE OFFSET EXISTS FOR THIS FILE")
return
# Get the offset of the read-start within its given chunk
chunkByteOffset = byteOffset % maxChunkSize
logging.debug("start sequence # == " + str(startSequence))
logging.debug("chunk byte offset == " + str(chunkByteOffset))
# If the user inputs a bytes to read of -1, the read will go until the end
# of the file.
if bytesToRead == -1:
# The ending offset will be the max file size
endOffset = maxChunkSize
# The end sequence can be found be knowing how many chunks a file has, and
# subtracting by 1 because the sequence numbers start at 0, not 1.
endSequence = len(database.data[self.fileName].chunks.keys()) - 1
else:
# To find where the user wants to read ending, add the number of bytes they want
# to read to their starting point, the byteOffest. This will give the byte offset
# of the end of the read
endReadOffset = byteOffset + bytesToRead
# Find the sequence of the chunk in which the end of the read will terminate
endSequence = endReadOffset // maxChunkSize
# Get the offset of the read-end within its given chunk
endOffset = endReadOffset % maxChunkSize
logging.debug("end sequence # == " + str(endSequence))
logging.debug("end read offset == " + str(endOffset))
# Create an empty string to hold the message that will be sent to the client
responseMessage = "READFROM"
# For each sequence number that exists between (and including) the read-start chunk
# and the read-end chunk, get the file's chunk with the appropriate sequence number,
# and append to the response message, a location it is stored at, its chunk handle,
# the byte offset from within that chunk to begin reading from, and the byte offset
# to end reading from
for sequence in range(startSequence, (endSequence + 1)):
# try:
# Get the chunkHandles associated with the given file, and sort the chunkHandles from
# least to greatest in the list. This will put them in their sequence order where the
# 0th element is now the 0th sequence, 1st element the 1st sequence, etc.
logging.debug(sorted(database.data[self.fileName].chunks.keys()))
associatedChunkHandles = sorted(database.data[self.fileName].chunks.keys())
# Append a location of where the start-sequence chunk is stored to the message
logging.debug(database.data[self.fileName].chunks[associatedChunkHandles[sequence]].locations)
responseMessage += "|" + str(
database.data[self.fileName].chunks[associatedChunkHandles[sequence]].locations[0]
)
# Append the chunk handle to the message
responseMessage += "*" + str(associatedChunkHandles[sequence])
# Append the byte offset to start reading from to the message
responseMessage += "*" + str(chunkByteOffset)
# If there are multiple chunks that will be read over, the next chunk will start
# the read from the beginning
chunkByteOffset = 0
# Check to see if the READ will take place in the same chunk. If it does, append the
# endOffset to the message so the client will know where to end reading
if startSequence == endSequence:
responseMessage += "*" + str(endOffset)
# If the READ takes place over multiple chunks, write the end of read for the current
# chunk to be the end of the chunk, and then increase the start sequence number so when the
# metadata for the last chunk is processed, it will be caught by the if statement above
# and send the appropriate ending offset.
elif startSequence < endSequence:
responseMessage += "*" + str(maxChunkSize)
startSequence += 1
# except:
# logging.error("Unable to generate proper READ response message.")
logging.debug("RESPONSE MESSAGE == " + str(responseMessage))
# send our message
fL.send(self.s, responseMessage)
logging.debug("SENT == " + responseMessage)
# Visual confirmation for debugging: confirm success of read()
logging.debug("Read successfully handled")