def download_jfsfile(remote_object, tofolder=None, checksum=False):
'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
if tofolder is None:
tofolder = '.' # with no arguments, store in current dir
total_size = remote_object.size
if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT, JFS.ProtoFile.STATE_INCOMPLETE):
puts(colored.red('%s was NOT downloaded successfully - Incomplete file' % remote_file.name))
return False
topath = os.path.join(tofolder, remote_object.name)
with open(topath, 'wb') as fh:
bytes_read = 0
puts(colored.white('Downloading: %s, size: %s \t' % (remote_object.name,
print_size(total_size, humanize=True))))
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_object.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if checksum:
md5_lf = JFS.calculate_md5(open(topath, 'rb'))
md5_jf = remote_object.md5
logging.info('%s - Checksum for downloaded file' % md5_lf)
logging.info('%s - Checksum for server file' % md5_jf)
if md5_lf != md5_jf:
puts(colored.blue('%s - Checksum for downloaded file' % md5_lf))
puts(colored.blue('%s - Checksum for server file' % md5_jf))
puts(colored.red('%s was NOT downloaded successfully - cheksum mismatch' % remote_object.name))
return False
puts(colored.green('%s was downloaded successfully - checksum matched' % remote_object.name))
return True
def download_jfsfile(remote_object, tofolder=None, checksum=False):
'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
if tofolder is None:
tofolder = '.' # with no arguments, store in current dir
total_size = remote_object.size
if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete file' %
remote_file.name))
return False
topath = os.path.join(tofolder, remote_object.name)
with open(topath, 'wb') as fh:
bytes_read = 0
puts(
colored.white('Downloading: %s, size: %s \t' %
(remote_object.name,
print_size(total_size, humanize=True))))
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_object.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if checksum:
md5_lf = JFS.calculate_md5(open(topath, 'rb'))
md5_jf = remote_object.md5
logging.info('%s - Checksum for downloaded file' % md5_lf)
logging.info('%s - Checksum for server file' % md5_jf)
if md5_lf != md5_jf:
puts(colored.blue('%s - Checksum for downloaded file' %
md5_lf))
puts(colored.blue('%s - Checksum for server file' % md5_jf))
puts(
colored.red(
'%s was NOT downloaded successfully - cheksum mismatch'
% remote_object.name))
return False
puts(
colored.green(
'%s was downloaded successfully - checksum matched' %
remote_object.name))
return True
def download(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description='Download a file or folder from Jottacloud.')
parser.add_argument('remoteobject', help='The path to the file or folder that you want to download')
parser.add_argument('-m', '--mode',
help='Mode of operation(decides where the default root should be): DEVICE - Root is above device level (need to specify both Device and Mountpoint in front of the folder i.e. DEVICENAME/mountpoint/folder), ARCHIVE - root is in the archive mountpoint, SYNC - root is in sync mountpoint, Default: %(default)s.',
choices=( 'device', 'archive', 'sync'), default='sync')
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
parser.add_argument('-c', '--checksum', help='Verify checksum of file after download', action='store_true' )
parser.add_argument('-r', '--resume', help='Will not download the files again if it exist in path', action='store_true' )
parser.add_argument('-v', '--verbose', help='Increase output verbosity', action='store_true' )
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
if args.mode == 'sync': #Device is Jotta and mountpoint is Sync
device = 'Jotta'
mountpoint = 'Sync'
print('Device is: %s' % device)
print('Mountpoint is: %s' % mountpoint)
root_folder = get_root_dir(jfs,device,mountpoint)
elif args.mode == 'archive': #Device is Jotta and mountpoint is Archive
device = 'Jotta'
mountpoint = 'Archive'
print('Device is: %s' % device)
print('Mountpoint is: %s' % mountpoint)
root_folder = get_root_dir(jfs,device,mountpoint)
elif args.mode == 'device': #Need to get the Device from first part of path and mountpoint from second part
path_in_parts = os.path.normpath(args.remoteobject).split(os.sep)
device = path_in_parts[0]
mountpoint = path_in_parts[1]
print('Device is: %s' % device)
print('Mountpoint is: %s' % mountpoint)
if len(path_in_parts)<3:
print('You need to specify at least one folder of file in addition to the device and mountpoint (i.e. 3 levels)')
exit(1)
else:
root_folder = get_root_dir(jfs,device,mountpoint)
del path_in_parts[0:2] #Removing device and mountpoint from path
args.remoteobject = "/".join(path_in_parts)
else:
exit(1)#This shouldn't really be necessary
path_to_object = posixpath.join(root_folder.path, args.remoteobject)
if args.verbose:
print('Root folder path: %s' % root_folder.path)
print('Relative path to object: %s' % args.remoteobject)
print('Absolute path to object: %s' % path_to_object)
remote_object = jfs.getObject(path_to_object)
if hasattr(remote_object, 'size'): #Check if it's a file that is downloaded by checking if the attribute 'size' exist
remote_file = remote_object
total_size = remote_file.size
if total_size == -1: # Indicates an incomplete file
print('%s was NOT downloaded successfully - Incomplete file' % remote_file.name)
exit(1)
with open(remote_file.name, 'wb') as fh:
bytes_read = 0
with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
for chunk_num, chunk in enumerate(remote_file.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if args.checksum:
md5_lf = JFS.calculate_md5(open(remote_file.name, 'rb'))
md5_jf = remote_file.md5
if md5_lf != md5_jf:
print('%s - Checksum for downloaded file' % md5_lf)
print('%s - Checksum for server file' % md5_jf)
print('%s was NOT downloaded successfully - cheksum mismatch' % remote_file.name)
exit(1)
if args.verbose:
print('%s - Checksum for downloaded file' % md5_lf)
print('%s - Checksum for server file' % md5_jf)
print('%s was downloaded successfully - checksum matched' % remote_file.name)
exit(1)
print('%s downloaded successfully - checksum not checked' % remote_file.name)
else: #if it's not a file it has to be a folder
incomplete_files = [] #Create an list where we can store incomplete files
checksum_error_files = [] #Create an list where we can store checksum error files
zero_files = [] #Create an list where we can store zero files
long_path = [] #Create an list where we can store skipped files and folders because of long path
if args.verbose:
print "It's a folder that is downloaded - getting the folder and file structure. This might take a while if the tree is big..."
fileTree = remote_object.filedirlist().tree #Download the folder tree
if args.verbose:
print('Total number of folders to download: %d' % len(fileTree))
#Iterate through each folder
for folder in fileTree:
#We need to strip the path to the folder path from account,device and mountpoint details
rel_path_to_object = folder[len(jfs.username)+len(device)+len(mountpoint)+4:]
if len(rel_path_to_object) > 250: #Windows has a limit of 250 characters in path
print('%s was NOT downloaded successfully - path to long' % rel_path_to_object)
long_path.append(rel_path_to_object)
else:
if args.verbose:
print('Entering a new folder: %s' % rel_path_to_object)
if not os.path.exists(rel_path_to_object): #Create the folder locally if it doesn't exist
os.makedirs(rel_path_to_object)
for _file in fileTree[folder]: #Enter the folder and download the files within
abs_path_to_object = posixpath.join(root_folder.path, posixpath.join(rel_path_to_object, _file[0])) #This is the absolute path to the file that is going to be downloaded
if args.verbose:
print('Downloading the file from: %s' % abs_path_to_object)
if _file[1]==-1: #Corrupt and incomplete files will be skipped
print('%s was NOT downloaded successfully - Incomplete or corrupt file' % _file[0])
incomplete_files.append(posixpath.join(rel_path_to_object,_file[0]))
else:
remote_object = jfs.getObject(abs_path_to_object)
remote_file = remote_object
total_size = remote_file.size
if total_size == 0: # Indicates an zero file
print('%s was NOT downloaded successfully - zero file' % remote_file.name)
zero_files.append(posixpath.join(rel_path_to_object,remote_file.name))
else:
if len(posixpath.join(rel_path_to_object,remote_file.name)) > 250: #Windows has a limit of 250 characters in path
print('%s was NOT downloaded successfully - path to long' % remote_file.name)
long_path.append(posixpath.join(rel_path_to_object,remote_file.name))
else:
if args.verbose:
print('Downloading the file to: %s' % posixpath.join(rel_path_to_object,remote_file.name))
if args.resume: #Check if file exist in path
if os.path.isfile(posixpath.join(rel_path_to_object,remote_file.name)):
print('File exist - skipping downloading: %s' % posixpath.join(rel_path_to_object,remote_file.name))
else:
with open(posixpath.join(rel_path_to_object,remote_file.name), 'wb') as fh:
bytes_read = 0
with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
for chunk_num, chunk in enumerate(remote_file.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
else:
with open(posixpath.join(rel_path_to_object,remote_file.name), 'wb') as fh:
bytes_read = 0
with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
for chunk_num, chunk in enumerate(remote_file.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if args.checksum:
md5_lf = JFS.calculate_md5(open(posixpath.join(rel_path_to_object,remote_file.name), 'rb'))
md5_jf = remote_file.md5
if md5_lf != md5_jf:
print('%s - Checksum for downloaded file' % md5_lf)
print('%s - Checksum for server file' % md5_jf)
print('%s was NOT downloaded successfully - cheksum mismatch' % remote_file.name)
checksum_error_files.append(posixpath.join(rel_path_to_object,remote_file.name))
else:
if args.verbose:
print('%s - Checksum for downloaded file' % md5_lf)
print('%s - Checksum for server file' % md5_jf)
print('%s was downloaded successfully - checksum matched' % remote_file.name)
else:
print('%s downloaded successfully - checksum not checked' % remote_file.name)
#Incomplete files
if len(incomplete_files)> 0:
with codecs.open("incomplete_files.txt", "w", "utf-8") as text_file:
for item in incomplete_files:
text_file.write("%s\n" % item)
print('Incomplete files (not downloaded): %d' % len(incomplete_files))
if args.verbose:
for _files in incomplete_files:
print _files
#Checksum error files
if len(checksum_error_files)> 0:
with codecs.open("checksum_error_files.txt", "w", "utf-8") as text_file:
for item in checksum_error_files:
text_file.write("%s\n" % item)
print('Files with checksum error (not downloaded): %d' % len(checksum_error_files))
if args.verbose:
for _files in checksum_error_files:
print _files
#zero files
if len(zero_files)> 0:
with codecs.open("zero_files.txt", "w", "utf-8") as text_file:
for item in zero_files:
text_file.write("%s\n" % item)
print('Files with zero size (not downloaded): %d' % len(zero_files))
if args.verbose:
for _files in zero_files:
print _files
#long path
if len(long_path)> 0:
with codecs.open("long_path.txt", "w", "utf-8") as text_file:
for item in long_path:
text_file.write("%s\n" % item)
print('Folder and files not downloaded because of path to long: %d' % len(long_path))
if args.verbose:
for _files in long_path:
print _files
