def async_file_upload(self, file_name):
# Used asynchronously for uploading a file to InfiniDrive after it has been uploaded and cached on the FTP server.
print('Starting asynchronous upload of ' + str(file_name) + '.')
# Get Drive service.
drive_connect = drive_api.get_service()
# Get directory ID.
dir_id = drive_api.get_file_id_from_name(drive_connect, file_name)
# Get a list of the fragments that currently make up the file. If this is a new upload, it should come back empty.
orig_fragments = drive_api.get_files_list_from_folder(drive_connect, dir_id)
# Set chunk size for reading files to 9.750365257263184MB (10223999 bytes)
read_chunk_sizes = 10223999
# Doc number
doc_num = 1
# Used to keep track of the numbers for fragments that have failed uploads.
failed_fragments = set()
# Iterate through file in chunks.
infile = open('ftp_upload_cache/' + str(file_name), 'rb')
# Read an initial chunk from the file.
file_bytes = infile.read(read_chunk_sizes)
# Keep looping until no more data is read.
while file_bytes:
if doc_num <= len(orig_fragments):
# A remote fragment is present, so update it.
upload_handler.handle_update_fragment(drive_api, orig_fragments[doc_num-1], file_bytes, drive_connect, doc_num, self.debug_log)
else:
# Process the fragment and upload it to Google Drive.
upload_handler.handle_upload_fragment(drive_api, file_bytes, drive_connect, dir_id, doc_num, failed_fragments, self.debug_log)
# Increment docNum for next Word document and read next chunk of data.
doc_num = doc_num + 1
file_bytes = infile.read(read_chunk_sizes)
# Run garbage collection. Hopefully, this will prevent process terminations by the operating system on memory-limited devices such as the Raspberry Pi.
gc.collect()
infile.close()
# If an update took place and the new file had fewer fragments than the previous file, delete any leftover fragments from the previous upload.
doc_num = doc_num - 1
while doc_num < len(orig_fragments):
drive_api.delete_file_by_id(drive_connect, orig_fragments[doc_num]['id'])
doc_num = doc_num + 1
# Process fragment upload failures
upload_handler.process_failed_fragments(drive_api, failed_fragments, dir_id, self.debug_log)
# Delete the local cache of the file.
remove('ftp_upload_cache/' + str(file_name))
# Report completion of the asynchronous upload.
print('Asynchronous upload of ' + str(file_name) + ' complete.')
def update(self, file_name=None, file_path=None):
# If no file name or file path is set, use the command line arguments.
if file_name == None and file_path == None:
file_name = sys.argv[2]
file_path = sys.argv[3]
# Get Drive service.
driveConnect = drive_api.get_service()
# Check if a remote file with the given name exists. If one does not, print an error message and return.
if not drive_api.file_with_name_exists(driveConnect, file_name):
print('Remote file with name ' + file_name + ' does not exist.')
return
# Get directory ID.
dirId = drive_api.get_file_id_from_name(driveConnect, file_name)
# Get a list of the fragments that currently make up the file. If this is a new upload, it should come back empty.
orig_fragments = drive_api.get_files_list_from_folder(
driveConnect, dirId)
# Determine if upload is taking place from an HTTP or HTTPS URL.
urlUpload = False
if file_path[0:4].lower() == 'http':
urlUpload = True
urlUploadHandle = requests.get(file_path,
stream=True,
allow_redirects=True)
fileSize = -1 # If file is being uploaded from web server and size cannot be retrieved this will stay at -1.
if urlUpload:
try:
fileSize = int(urlUploadHandle.headers.get('content-length'))
except TypeError:
pass
if fileSize == -1:
# If fileSize is set to -1, set totalFrags to "an unknown number of"
totalFrags = 'an unknown number of'
else:
fileSize = os.stat(file_path).st_size
if fileSize != -1:
totalFrags = math.ceil(fileSize / 10223999)
print('Upload started. Upload will be composed of ' + str(totalFrags) +
' fragments.\n')
# Set chunk size for reading files to 9.750365257263184MB (10223999 bytes)
readChunkSizes = 10223999
# Doc number
docNum = 1
# Used to keep track of the numbers for fragments that have failed uploads.
failedFragmentsSet = set()
# Progress bar
if fileSize == -1:
# The file size is unknown
upBar = Spinner('Uploading... ')
else:
# The file size is known
upBar = ShadyBar('Uploading...',
max=max(math.ceil(fileSize / 10223999),
len(orig_fragments)))
if urlUpload:
# If the upload is taking place from a URL...
# Iterate through remote file until no more data is read.
for fileBytes in urlUploadHandle.iter_content(
chunk_size=readChunkSizes):
# Advance progress bar
upBar.next()
if docNum <= len(orig_fragments):
# A remote fragment is present, so update it.
upload_handler.handle_update_fragment(
drive_api, orig_fragments[docNum - 1], fileBytes,
driveConnect, docNum, self.debug_log)
else:
# Process the fragment and upload it to Google Drive.
upload_handler.handle_upload_fragment(
drive_api, fileBytes, driveConnect, dirId, docNum,
failedFragmentsSet, self.debug_log)
# Increment docNum for next Word document.
docNum = docNum + 1
# Run garbage collection. Hopefully, this will prevent process terminations by the operating system on memory-limited devices such as the Raspberry Pi.
gc.collect()
else:
# If the upload is taking place from a file path...
# Get file byte size
fileSize = os.path.getsize(file_path)
# Iterate through file in chunks.
infile = open(str(file_path), 'rb')
# Read an initial chunk from the file.
fileBytes = infile.read(readChunkSizes)
# Keep looping until no more data is read.
while fileBytes:
# Advance progress bar
upBar.next()
if docNum <= len(orig_fragments):
# A remote fragment is present, so update it.
upload_handler.handle_update_fragment(
drive_api, orig_fragments[docNum - 1], fileBytes,
driveConnect, docNum, self.debug_log)
else:
# Process the fragment and upload it to Google Drive.
upload_handler.handle_upload_fragment(
drive_api, fileBytes, driveConnect, dirId, docNum,
failedFragmentsSet, self.debug_log)
# Increment docNum for next Word document and read next chunk of data.
docNum = docNum + 1
fileBytes = infile.read(readChunkSizes)
# Run garbage collection. Hopefully, this will prevent process terminations by the operating system on memory-limited devices such as the Raspberry Pi.
gc.collect()
infile.close()
# If an update took place and the new file had fewer fragments than the previous file, delete any leftover fragments from the previous upload.
docNum = docNum - 1
while docNum < len(orig_fragments):
upBar.next()
drive_api.delete_file_by_id(drive_api.get_service(),
orig_fragments[docNum]['id'])
docNum = docNum + 1
# Process fragment upload failures
upload_handler.process_failed_fragments(drive_api, failedFragmentsSet,
dirId, self.debug_log)
upBar.finish()
# If the number of fragments to expect from a file upload is known, verify that the upload is not corrupted.
if totalFrags != 'an unknown number of':
print('Verifying upload.')
foundFrags = len(
drive_api.get_files_list_from_folder(drive_api.get_service(),
dirId))
if (totalFrags != foundFrags):
self.debug_log.write(
"----------------------------------------\n")
self.debug_log.write(
"InfiniDrive detected upload corruption.\n")
self.debug_log.write("Expected Fragments: " + str(totalFrags) +
"\n")
self.debug_log.write("Actual Fragments : " + str(foundFrags) +
"\n")
print(
'InfiniDrive has detected that your upload was corrupted. Please report this issue on the InfiniDrive GitHub issue tracker and upload your "log.txt" file.'
)
print('Upload complete!')
def RETR(self, cmd):
# Downloads an InfiniDrive file
# Extract the name of the file to download from the command
filename = cmd[5:-2].lstrip('/')
# Open the data socket.
print('Downloading', filename)
self.conn.send(b'150 Opening data connection.\r\n')
self.start_datasock()
if not drive_api.file_with_name_exists(self.drive_service, filename):
# Check if the file exists. If it does not, close the socket and send an error.
self.stop_datasock()
self.conn.send(b'551 File does not exist.\r\n')
return
# Get a count of the number of fragments that make up the file.
fragment_count = drive_api.get_fragment_count(self.drive_service, filename)
# For indexing fragments.
fragment_index = 1
# Get the InfiniDrive file ID from its name
file_id = drive_api.get_file_id_from_name(self.drive_service, filename)
# If the client has requested a custom starting position, slice off irrelevant fragments and calculate the fragment byte offset.
if self.rest:
fragment_index = self.pos // 10223999 + 1
self.frag_byte_offset = self.pos % 10223999
# Asynchronously retrieve a list of all files. We do this so that we can reduce Drive API calls, but if we wait for the list,
# the FTP client will likely time out before we can finish, so we will retrieve one fragment at a time at first while the
# entire list is retrieved in the background here.
files = list()
threading.Thread(target=drive_api.get_files_list_from_folder_async, args=(drive_api.get_service(), file_id, files)).start()
# For all fragments...
while fragment_index <= fragment_count:
# Get the fragment with the given index
file = None
if files == []:
# The fragment list is not available yet, so retrieve one fragment.
file = drive_api.get_files_with_name_from_folder(self.drive_service, file_id, str(fragment_index))[0]
else:
# The fragment list is available, so use it to locate the fragment.
file = files[0][fragment_index - 1]
# Get the RGB pixel values from the image as a list of tuples that we will break up and then convert to a bytestring.
while True:
try:
pixelVals = list(Image.open(drive_api.get_image_bytes_from_doc(self.drive_service, file)).convert('RGB').getdata())
except Exception as e:
self.debug_log.write("----------------------------------------\n")
self.debug_log.write("Fragment download failure\n")
self.debug_log.write("Error:\n")
self.debug_log.write(str(e) + "\n")
continue
pixelVals = [j for i in pixelVals for j in i]
if len(pixelVals) == 10224000:
break
# If the downloaded values do not match the fragment hash, terminate download and report corruption.
if hash_handler.is_download_invalid(file, bytearray(pixelVals)):
self.stop_datasock()
self.conn.send(b'551 File is corrupted.\r\n')
return
# Strip the null byte padding and "spacer byte" from pixelVals.
pixelVals = array.array('B', pixelVals).tobytes().rstrip(b'\x00')[:-1]
# If the client requested a custom starting position, slice off the start of the byte array using the calculated frag_byte_offset value.
if self.rest:
pixelVals = pixelVals[self.frag_byte_offset:]
self.rest = False
# Send the byte array to the client.
self.datasock.send(pixelVals)
# Increment fragment_index
fragment_index += 1
# File transfer is complete. Close the data socket and report completion.
self.stop_datasock()
self.conn.send(b'226 Transfer complete.\r\n')
def download(self):
# Save file name from command line arguments.
file_name = str(sys.argv[2])
# Check if the file exists. If it does not, print an error message and return.
if not drive_api.file_with_name_exists(drive_api.get_service(),
file_name):
print('File with name "' + file_name + '" does not exist.')
return
# Get Drive service.
drive_service = drive_api.get_service()
# Get a count of the number of fragments that make up the file.
fragment_count = drive_api.get_fragment_count(drive_service, file_name)
# For indexing fragments.
fragment_index = 1
# Get the InfiniDrive file ID from its name
file_id = drive_api.get_file_id_from_name(drive_service, file_name)
# Asynchronously retrieve a list of all files. We do this so that we can reduce Drive API calls, but if we wait for the list,
# the FTP client will likely time out before we can finish, so we will retrieve one fragment at a time at first while the
# entire list is retrieved in the background here.
files = list()
threading.Thread(target=drive_api.get_files_list_from_folder_async,
args=(drive_api.get_service(), file_id,
files)).start()
# Open a file at the user-specified path to write the data to
result = open(str(sys.argv[3]), "wb")
# Download complete print flag
showDownloadComplete = True
# For all fragments...
downBar = ShadyBar('Downloading...',
max=fragment_count) # Progress bar
while fragment_index <= fragment_count:
downBar.next()
# Get the fragment with the given index
file = None
if files == []:
# The fragment list is not available yet, so retrieve one fragment.
file = drive_api.get_files_with_name_from_folder(
drive_service, file_id, str(fragment_index))[0]
else:
# The fragment list is available, so use it to locate the fragment.
file = files[0][fragment_index - 1]
# Get the RGB pixel values from the image as a list of tuples that we will break up and then convert to a bytestring.
while True:
try:
pixelVals = list(
Image.open(
drive_api.get_image_bytes_from_doc(
drive_api.get_service(),
file)).convert('RGB').getdata())
except Exception as e:
self.debug_log.write(
"----------------------------------------\n")
self.debug_log.write("Fragment download failure\n")
self.debug_log.write("Error:\n")
self.debug_log.write(str(e) + "\n")
continue
pixelVals = [j for i in pixelVals for j in i]
if len(pixelVals) == 10224000:
break
# If the downloaded values do not match the fragment hash, terminate download and report corruption.
if hash_handler.is_download_invalid(file, bytearray(pixelVals)):
downBar.finish()
print(
"\nError: InfiniDrive has detected that the file upload on Google Drive is corrupted and the download cannot complete.",
end="")
showDownloadComplete = False
break
# Strip the null byte padding and "spacer byte" from pixelVals.
pixelVals = array.array('B',
pixelVals).tobytes().rstrip(b'\x00')[:-1]
# Write the data stored in "pixelVals" to the output file.
result.write(pixelVals)
fragment_index += 1
# Run garbage collection. Hopefully, this will prevent process terminations by the operating system on memory-limited devices such as the Raspberry Pi.
gc.collect()
result.close()
downBar.finish()
if showDownloadComplete:
print('\nDownload complete!')