def sanitizeFile(self, fileName):
try:
# Before we delete the file from the database, we want to make sure we know which chunks
# are associated with it, so they can be removed from the lookup.
associatedChunks = self.data[fileName].chunks.keys()
# For each key, remove it from the lookup
for chunk in associatedChunks:
del self.lookup[str(chunk)]
# We also want to make sure that it is no longer in the toDelete list, since it has been deleted
if fileName in self.toDelete:
self.toDelete.remove(fileName)
# We want to remove it from the location-->chunk lookup as well.
# For all the keyed locations, we want to remove any chunk value that belongs to
# a file being deleted.
for key in self.locDict:
for chunk in associatedChunks:
if chunk in self.locDict[key]:
self.locDict[key].remove(chunk)
# Delete the specified key/value pair from the database.
del self.data[fileName]
# Update the opLog that a new file was created
fL.appendToOpLog("SANITIZED|-1|" + fileName)
logging.debug('sanitizeFile() success')
except KeyError:
logging.error("The file to be deleted does not exist.")
def sanitizeFile(self, fileName):
try:
# Before we delete the file from the database, we want to make sure we know which chunks
# are associated with it, so they can be removed from the lookup.
associatedChunks = self.data[fileName].chunks.keys()
# For each key, remove it from the lookup
for chunk in associatedChunks:
del self.lookup[str(chunk)]
# We also want to make sure that it is no longer in the toDelete list, since it has been deleted
if fileName in self.toDelete:
self.toDelete.remove(fileName)
# We want to remove it from the location-->chunk lookup as well.
# For all the keyed locations, we want to remove any chunk value that belongs to
# a file being deleted.
for key in self.locDict:
for chunk in associatedChunks:
if chunk in self.locDict[key]:
self.locDict[key].remove(chunk)
# Delete the specified key/value pair from the database.
del self.data[fileName]
# Update the opLog that a new file was created
fL.appendToOpLog("SANITIZED|-1|" + fileName)
logging.debug('sanitizeFile() success')
except KeyError:
logging.error("The file to be deleted does not exist.")
def flagUndelete(self, fileName):
logging.debug('Initialize flagUndelete()')
# Unflag the given file for deletion
self.data[fileName].delete = False
# Remove the file name from the list of files to be deleted
self.toDelete.remove(fileName)
# Update the opLog that a new file was created
fL.appendToOpLog("UNDELETE|-1|" + fileName)
logging.debug('Delete flag updated')
def flagDelete(self, fileName):
logging.debug('Initialize flagDelete()')
# Flag the given file for deletion
self.data[fileName].delete = True
# Add the file name to the list of files to be deleted
self.toDelete.append(fileName)
# Update the opLog that a new file was created
fL.appendToOpLog("DELETE|-1|" + fileName)
logging.debug('Delete flag updated')
def flagUndelete(self, fileName):
logging.debug('Initialize flagUndelete()')
# Unflag the given file for deletion
self.data[fileName].delete = False
# Remove the file name from the list of files to be deleted
self.toDelete.remove(fileName)
# Update the opLog that a new file was created
fL.appendToOpLog("UNDELETE|-1|" + fileName)
logging.debug('Delete flag updated')
def flagDelete(self, fileName):
logging.debug('Initialize flagDelete()')
# Flag the given file for deletion
self.data[fileName].delete = True
# Add the file name to the list of files to be deleted
self.toDelete.append(fileName)
# Update the opLog that a new file was created
fL.appendToOpLog("DELETE|-1|" + fileName)
logging.debug('Delete flag updated')
def createNewFile(self, fileName, cH):
# Check to see if the fileName already exists
if fileName in self.data:
logging.error('file name already exists in database')
# Return -1 to the master, signifying that the fileName already exists,
# so the master can alert the client.
return -1
else:
# Create a new instance of the File object
file = File(fileName)
# Add the file object, keyed to the file name, to the database
self.data[fileName] = file
# Update the opLog that a new file was created
fL.appendToOpLog("CREATEFILE|-1|" + fileName)
# Create a new chunk to be associated with the file just created
self.createNewChunk(fileName, -1, cH)
logging.debug('createNewFile() ==> new file successfully added to database')
def createNewFile(self, fileName, cH):
# Check to see if the fileName already exists
if fileName in self.data:
logging.error('file name already exists in database')
# Return -1 to the master, signifying that the fileName already exists,
# so the master can alert the client.
return -1
else:
# Create a new instance of the File object
file = File(fileName)
# Add the file object, keyed to the file name, to the database
self.data[fileName] = file
# Update the opLog that a new file was created
fL.appendToOpLog("CREATEFILE|-1|" + fileName)
# Create a new chunk to be associated with the file just created
self.createNewChunk(fileName, -1, cH)
logging.debug(
'createNewFile() ==> new file successfully added to database')
def createNewChunk(self, fileName, handleOfFullChunk, cH):
# Check to see if the specified filename exists in the database
if fileName not in self.data:
logging.error('createNewChunk() ==> file for new chunk does not exist')
# Return an error flag to be parsed by the master, so it can alert the client
# that the file name does not exist
return -2
else:
# Define the latest chunk to be less than the possible chunkhandles so it will
# fail the handleOfFullChunk == latestChunk below. This way, if the message for
# handleOfFullChunk that we get is not -1 (the flag for a file created), then
# it will not erroneously make a chunk.
latestChunk = -2
# If the handleOfFullChunk is not the flag for a file create, -1, (where no latest
# chunk should exist as the file was Just created), then find the chunk with the
# highest chunk handle (the chunk with the highest sequence number)
if handleOfFullChunk != -1:
latestChunk = self.findLatestChunk(fileName)
# Check to see if the triggering chunk is in the list of chunk handles associated
# with that file. If it is, that means a new chunk must be created
if int(handleOfFullChunk) == int(latestChunk) or int(handleOfFullChunk) == -1:
# Create a new instance of the Chunk object
chunk = Chunk()
# Get a new chunkHandle
chunkHandle = cH
logging.debug('Got new chunk handle')
# Add the chunk to the file object, keyed to its new chunkHandle
self.data[fileName].chunks[chunkHandle] = chunk
# Get the three locations where the chunk will be stored
locations = fL.chooseHosts().split("|")
locations = filter(None, locations)
logging.debug(locations)
#string to be returned
string = ''
# Append the chunkserver locations to the chunk's location list and update
# the location-->chunk lookup
for location in locations:
self.locDict[location].append(chunkHandle)
logging.debug('adding locations to new chunk')
self.data[fileName].chunks[chunkHandle].locations.append(location)
logging.debug('Appending locations to chunk ' + str(chunkHandle) + ' locations list')
string += location+"|"
logging.debug('file: ' + fileName + ' chunk: ' + str(self.data[fileName].chunks[chunkHandle]))
# Add the chunk to the chunk/file lookup
self.lookup[chunkHandle] = fileName
logging.debug(self.data[fileName].chunks[chunkHandle].locations)
# Update the opLog that a new chunk was created
fL.appendToOpLog("CREATECHUNK|" + chunkHandle + "|" + fileName)
string += chunkHandle
#If this completed successfully, return the string to send.
return string
# The full chunk is not the latest chunk, so a new chunk has already been created for
# the file. We dont want to branch a file into multiple chunks, so we let the master know
# that a new chunk already exists to append to.
else:
return -3
def createNewChunk(self, fileName, handleOfFullChunk, cH):
# Check to see if the specified filename exists in the database
if fileName not in self.data:
logging.error(
'createNewChunk() ==> file for new chunk does not exist')
# Return an error flag to be parsed by the master, so it can alert the client
# that the file name does not exist
return -2
else:
# Define the latest chunk to be less than the possible chunkhandles so it will
# fail the handleOfFullChunk == latestChunk below. This way, if the message for
# handleOfFullChunk that we get is not -1 (the flag for a file created), then
# it will not erroneously make a chunk.
latestChunk = -2
# If the handleOfFullChunk is not the flag for a file create, -1, (where no latest
# chunk should exist as the file was Just created), then find the chunk with the
# highest chunk handle (the chunk with the highest sequence number)
if handleOfFullChunk != -1:
latestChunk = self.findLatestChunk(fileName)
# Check to see if the triggering chunk is in the list of chunk handles associated
# with that file. If it is, that means a new chunk must be created
if int(handleOfFullChunk) == int(latestChunk) or int(
handleOfFullChunk) == -1:
# Create a new instance of the Chunk object
chunk = Chunk()
# Get a new chunkHandle
chunkHandle = cH
logging.debug('Got new chunk handle')
# Add the chunk to the file object, keyed to its new chunkHandle
self.data[fileName].chunks[chunkHandle] = chunk
# Get the three locations where the chunk will be stored
locations = fL.chooseHosts().split("|")
locations = filter(None, locations)
logging.debug(locations)
#string to be returned
string = ''
# Append the chunkserver locations to the chunk's location list and update
# the location-->chunk lookup
for location in locations:
self.locDict[location].append(chunkHandle)
logging.debug('adding locations to new chunk')
self.data[fileName].chunks[chunkHandle].locations.append(
location)
logging.debug('Appending locations to chunk ' +
str(chunkHandle) + ' locations list')
string += location + "|"
logging.debug('file: ' + fileName + ' chunk: ' +
str(self.data[fileName].chunks[chunkHandle]))
# Add the chunk to the chunk/file lookup
self.lookup[chunkHandle] = fileName
logging.debug(
self.data[fileName].chunks[chunkHandle].locations)
# Update the opLog that a new chunk was created
fL.appendToOpLog("CREATECHUNK|" + chunkHandle + "|" + fileName)
string += chunkHandle
#If this completed successfully, return the string to send.
return string
# The full chunk is not the latest chunk, so a new chunk has already been created for
# the file. We dont want to branch a file into multiple chunks, so we let the master know
# that a new chunk already exists to append to.
else:
return -3