def download_jfsfile(remote_object, tofolder=None, checksum=False): 'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean' if tofolder is None: tofolder = '.' # with no arguments, store in current dir total_size = remote_object.size if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT, JFS.ProtoFile.STATE_INCOMPLETE): puts(colored.red('%s was NOT downloaded successfully - Incomplete file' % remote_file.name)) return False topath = os.path.join(tofolder, remote_object.name) with open(topath, 'wb') as fh: bytes_read = 0 puts(colored.white('Downloading: %s, size: %s \t' % (remote_object.name, print_size(total_size, humanize=True)))) with ProgressBar(expected_size=total_size) as bar: for chunk_num, chunk in enumerate(remote_object.stream()): fh.write(chunk) bytes_read += len(chunk) bar.show(bytes_read) if checksum: md5_lf = JFS.calculate_md5(open(topath, 'rb')) md5_jf = remote_object.md5 logging.info('%s - Checksum for downloaded file' % md5_lf) logging.info('%s - Checksum for server file' % md5_jf) if md5_lf != md5_jf: puts(colored.blue('%s - Checksum for downloaded file' % md5_lf)) puts(colored.blue('%s - Checksum for server file' % md5_jf)) puts(colored.red('%s was NOT downloaded successfully - cheksum mismatch' % remote_object.name)) return False puts(colored.green('%s was downloaded successfully - checksum matched' % remote_object.name)) return True
def download_jfsfile(remote_object, tofolder=None, checksum=False): 'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean' if tofolder is None: tofolder = '.' # with no arguments, store in current dir total_size = remote_object.size if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT, JFS.ProtoFile.STATE_INCOMPLETE): puts( colored.red( '%s was NOT downloaded successfully - Incomplete file' % remote_file.name)) return False topath = os.path.join(tofolder, remote_object.name) with open(topath, 'wb') as fh: bytes_read = 0 puts( colored.white('Downloading: %s, size: %s \t' % (remote_object.name, print_size(total_size, humanize=True)))) with ProgressBar(expected_size=total_size) as bar: for chunk_num, chunk in enumerate(remote_object.stream()): fh.write(chunk) bytes_read += len(chunk) bar.show(bytes_read) if checksum: md5_lf = JFS.calculate_md5(open(topath, 'rb')) md5_jf = remote_object.md5 logging.info('%s - Checksum for downloaded file' % md5_lf) logging.info('%s - Checksum for server file' % md5_jf) if md5_lf != md5_jf: puts(colored.blue('%s - Checksum for downloaded file' % md5_lf)) puts(colored.blue('%s - Checksum for server file' % md5_jf)) puts( colored.red( '%s was NOT downloaded successfully - cheksum mismatch' % remote_object.name)) return False puts( colored.green( '%s was downloaded successfully - checksum matched' % remote_object.name)) return True
def download(argv=None): if argv is None: argv = sys.argv[1:] parser = argparse.ArgumentParser(description='Download a file or folder from Jottacloud.') parser.add_argument('remoteobject', help='The path to the file or folder that you want to download') parser.add_argument('-m', '--mode', help='Mode of operation(decides where the default root should be): DEVICE - Root is above device level (need to specify both Device and Mountpoint in front of the folder i.e. DEVICENAME/mountpoint/folder), ARCHIVE - root is in the archive mountpoint, SYNC - root is in sync mountpoint, Default: %(default)s.', choices=( 'device', 'archive', 'sync'), default='sync') parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.', choices=('debug', 'info', 'warning', 'error'), default='warning') parser.add_argument('-c', '--checksum', help='Verify checksum of file after download', action='store_true' ) parser.add_argument('-r', '--resume', help='Will not download the files again if it exist in path', action='store_true' ) parser.add_argument('-v', '--verbose', help='Increase output verbosity', action='store_true' ) args = parse_args_and_apply_logging_level(parser, argv) jfs = JFS.JFS() if args.mode == 'sync': #Device is Jotta and mountpoint is Sync device = 'Jotta' mountpoint = 'Sync' print('Device is: %s' % device) print('Mountpoint is: %s' % mountpoint) root_folder = get_root_dir(jfs,device,mountpoint) elif args.mode == 'archive': #Device is Jotta and mountpoint is Archive device = 'Jotta' mountpoint = 'Archive' print('Device is: %s' % device) print('Mountpoint is: %s' % mountpoint) root_folder = get_root_dir(jfs,device,mountpoint) elif args.mode == 'device': #Need to get the Device from first part of path and mountpoint from second part path_in_parts = os.path.normpath(args.remoteobject).split(os.sep) device = path_in_parts[0] mountpoint = path_in_parts[1] print('Device is: %s' % device) print('Mountpoint is: %s' % mountpoint) if len(path_in_parts)<3: print('You need to specify at least one folder of file in addition to the device and mountpoint (i.e. 3 levels)') exit(1) else: root_folder = get_root_dir(jfs,device,mountpoint) del path_in_parts[0:2] #Removing device and mountpoint from path args.remoteobject = "/".join(path_in_parts) else: exit(1)#This shouldn't really be necessary path_to_object = posixpath.join(root_folder.path, args.remoteobject) if args.verbose: print('Root folder path: %s' % root_folder.path) print('Relative path to object: %s' % args.remoteobject) print('Absolute path to object: %s' % path_to_object) remote_object = jfs.getObject(path_to_object) if hasattr(remote_object, 'size'): #Check if it's a file that is downloaded by checking if the attribute 'size' exist remote_file = remote_object total_size = remote_file.size if total_size == -1: # Indicates an incomplete file print('%s was NOT downloaded successfully - Incomplete file' % remote_file.name) exit(1) with open(remote_file.name, 'wb') as fh: bytes_read = 0 with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar: for chunk_num, chunk in enumerate(remote_file.stream()): fh.write(chunk) bytes_read += len(chunk) bar.show(bytes_read) if args.checksum: md5_lf = JFS.calculate_md5(open(remote_file.name, 'rb')) md5_jf = remote_file.md5 if md5_lf != md5_jf: print('%s - Checksum for downloaded file' % md5_lf) print('%s - Checksum for server file' % md5_jf) print('%s was NOT downloaded successfully - cheksum mismatch' % remote_file.name) exit(1) if args.verbose: print('%s - Checksum for downloaded file' % md5_lf) print('%s - Checksum for server file' % md5_jf) print('%s was downloaded successfully - checksum matched' % remote_file.name) exit(1) print('%s downloaded successfully - checksum not checked' % remote_file.name) else: #if it's not a file it has to be a folder incomplete_files = [] #Create an list where we can store incomplete files checksum_error_files = [] #Create an list where we can store checksum error files zero_files = [] #Create an list where we can store zero files long_path = [] #Create an list where we can store skipped files and folders because of long path if args.verbose: print "It's a folder that is downloaded - getting the folder and file structure. This might take a while if the tree is big..." fileTree = remote_object.filedirlist().tree #Download the folder tree if args.verbose: print('Total number of folders to download: %d' % len(fileTree)) #Iterate through each folder for folder in fileTree: #We need to strip the path to the folder path from account,device and mountpoint details rel_path_to_object = folder[len(jfs.username)+len(device)+len(mountpoint)+4:] if len(rel_path_to_object) > 250: #Windows has a limit of 250 characters in path print('%s was NOT downloaded successfully - path to long' % rel_path_to_object) long_path.append(rel_path_to_object) else: if args.verbose: print('Entering a new folder: %s' % rel_path_to_object) if not os.path.exists(rel_path_to_object): #Create the folder locally if it doesn't exist os.makedirs(rel_path_to_object) for _file in fileTree[folder]: #Enter the folder and download the files within abs_path_to_object = posixpath.join(root_folder.path, posixpath.join(rel_path_to_object, _file[0])) #This is the absolute path to the file that is going to be downloaded if args.verbose: print('Downloading the file from: %s' % abs_path_to_object) if _file[1]==-1: #Corrupt and incomplete files will be skipped print('%s was NOT downloaded successfully - Incomplete or corrupt file' % _file[0]) incomplete_files.append(posixpath.join(rel_path_to_object,_file[0])) else: remote_object = jfs.getObject(abs_path_to_object) remote_file = remote_object total_size = remote_file.size if total_size == 0: # Indicates an zero file print('%s was NOT downloaded successfully - zero file' % remote_file.name) zero_files.append(posixpath.join(rel_path_to_object,remote_file.name)) else: if len(posixpath.join(rel_path_to_object,remote_file.name)) > 250: #Windows has a limit of 250 characters in path print('%s was NOT downloaded successfully - path to long' % remote_file.name) long_path.append(posixpath.join(rel_path_to_object,remote_file.name)) else: if args.verbose: print('Downloading the file to: %s' % posixpath.join(rel_path_to_object,remote_file.name)) if args.resume: #Check if file exist in path if os.path.isfile(posixpath.join(rel_path_to_object,remote_file.name)): print('File exist - skipping downloading: %s' % posixpath.join(rel_path_to_object,remote_file.name)) else: with open(posixpath.join(rel_path_to_object,remote_file.name), 'wb') as fh: bytes_read = 0 with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar: for chunk_num, chunk in enumerate(remote_file.stream()): fh.write(chunk) bytes_read += len(chunk) bar.show(bytes_read) else: with open(posixpath.join(rel_path_to_object,remote_file.name), 'wb') as fh: bytes_read = 0 with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar: for chunk_num, chunk in enumerate(remote_file.stream()): fh.write(chunk) bytes_read += len(chunk) bar.show(bytes_read) if args.checksum: md5_lf = JFS.calculate_md5(open(posixpath.join(rel_path_to_object,remote_file.name), 'rb')) md5_jf = remote_file.md5 if md5_lf != md5_jf: print('%s - Checksum for downloaded file' % md5_lf) print('%s - Checksum for server file' % md5_jf) print('%s was NOT downloaded successfully - cheksum mismatch' % remote_file.name) checksum_error_files.append(posixpath.join(rel_path_to_object,remote_file.name)) else: if args.verbose: print('%s - Checksum for downloaded file' % md5_lf) print('%s - Checksum for server file' % md5_jf) print('%s was downloaded successfully - checksum matched' % remote_file.name) else: print('%s downloaded successfully - checksum not checked' % remote_file.name) #Incomplete files if len(incomplete_files)> 0: with codecs.open("incomplete_files.txt", "w", "utf-8") as text_file: for item in incomplete_files: text_file.write("%s\n" % item) print('Incomplete files (not downloaded): %d' % len(incomplete_files)) if args.verbose: for _files in incomplete_files: print _files #Checksum error files if len(checksum_error_files)> 0: with codecs.open("checksum_error_files.txt", "w", "utf-8") as text_file: for item in checksum_error_files: text_file.write("%s\n" % item) print('Files with checksum error (not downloaded): %d' % len(checksum_error_files)) if args.verbose: for _files in checksum_error_files: print _files #zero files if len(zero_files)> 0: with codecs.open("zero_files.txt", "w", "utf-8") as text_file: for item in zero_files: text_file.write("%s\n" % item) print('Files with zero size (not downloaded): %d' % len(zero_files)) if args.verbose: for _files in zero_files: print _files #long path if len(long_path)> 0: with codecs.open("long_path.txt", "w", "utf-8") as text_file: for item in long_path: text_file.write("%s\n" % item) print('Folder and files not downloaded because of path to long: %d' % len(long_path)) if args.verbose: for _files in long_path: print _files