def share(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Share a file on JottaCloud and get the public URI.',
epilog='Note: This utility needs to find JOTTACLOUD_USERNAME'
' and JOTTACLOUD_PASSWORD in the running environment.')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('localfile',
help='The local file that you want to share',
type=argparse.FileType('r'))
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
jottadev = get_jfs_device(jfs)
jottashare = jottadev.mountPoints['Shared']
upload = jottashare.up(args.localfile) # upload file
public = upload.share() # share file
logging.debug('Shared %r and got: %r (%s)', args.localfile, public,
dir(public))
for (filename, uuid, publicURI) in public.sharedFiles():
print('%s is now available to the world at %s' % (filename, publicURI))
return True # TODO: check return value of command
def rm(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Delete an item from Jottacloud')
parser.add_argument('file',
type=commandline_text,
help='The path to the item that you want to delete')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('-f',
'--force',
help='Completely deleted, no restore possiblity',
action='store_true')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_dir = get_root_dir(jfs)
item_path = posixpath.join(root_dir.path, args.file)
item = jfs.getObject(item_path)
if args.force:
item.hard_delete()
else:
item.delete()
print('%s deleted' % args.file)
return True # TODO: check return value of command
def upload(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Upload a file to JottaCloud.')
parser.add_argument('localfile',
help='The local file that you want to upload',
type=argparse.FileType('r'))
parser.add_argument('remote_dir',
help='The remote directory to upload the file to',
nargs='?',
type=commandline_text)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
jfs = JFS.JFS()
args = parse_args_and_apply_logging_level(parser, argv)
decoded_filename = commandline_text(args.localfile.name)
progress_bar = ProgressBar()
callback = lambda monitor, size: progress_bar.show(monitor.bytes_read, size
)
root_folder = get_root_dir(jfs)
if args.remote_dir:
target_dir_path = posixpath.join(root_folder.path, args.remote_dir)
target_dir = jfs.getObject(target_dir_path)
else:
target_dir = root_folder
upload = target_dir.up(args.localfile,
os.path.basename(decoded_filename),
upload_callback=callback)
print('%s uploaded successfully' % decoded_filename)
print(upload)
return True # TODO: check return value
def test_xml(self):
xml = b"""<?xml version="1.0" encoding="UTF-8"?>
<device time="2015-09-12-T23:14:25Z" host="dn-093.site-000.jotta.no">
<name xml:space="preserve">Jotta</name>
<type>JOTTA</type>
<sid>ee93a510-907a-4d7c-bbb9-59df7894xxxx</sid>
<size>58428516774</size>
<modified>2015-09-12-T23:10:51Z</modified>
<user>havardgulldahl</user>
<mountPoints>
<mountPoint>
<name xml:space="preserve">Archive</name>
<size>18577011381</size>
<modified>2015-09-12-T23:10:51Z</modified>
</mountPoint>
<mountPoint>
<name xml:space="preserve">Photos</name>
<size>18577011381</size>
<modified>2015-09-12-T23:10:51Z</modified>
</mountPoint>
<mountPoint>
<name xml:space="preserve">Shared</name>
<size>43777</size>
<modified>2015-09-03-T21:12:55Z</modified>
</mountPoint>
<mountPoint>
<name xml:space="preserve">Sync</name>
<size>39851461616</size>
<modified>2015-07-26-T22:26:54Z</modified>
</mountPoint>
</mountPoints>
<metadata first="" max="" total="3" num_mountpoints="3"/>
</device>"""
o = lxml.objectify.fromstring(xml)
dev = JFS.JFSDevice(o, jfs, parentpath=jfs.rootpath)
assert isinstance(o, lxml.objectify.ObjectifiedElement)
# Test that mountpoints are populated correctly"
assert sorted(dev.mountPoints.keys()) == ['Archive', 'Shared', 'Sync']
assert all(isinstance(item, JFS.JFSMountPoint) for item in dev.mountPoints.values())
# test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
assert all(isinstance(item, JFS.JFSFile) for item in dev.files('Archive'))
# test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
mps = dev.mountpointobjects()
assert all(isinstance(item, JFS.JFSFile) for item in dev.files(mps[2]))
#test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
assert all(isinstance(item, JFS.JFSFolder) for item in dev.folders('Archive'))
#test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
mps = dev.mountpointobjects()
assert all(isinstance(item, JFS.JFSFolder) for item in dev.folders(mps[0]))
# test_properties
assert isinstance(dev.modified, datetime.datetime)
assert dev.path == '%s/%s' % (jfs.rootpath, 'Jotta')
assert dev.name == 'Jotta'
assert dev.type == 'JOTTA'
assert dev.size == 58428516774
assert dev.sid == 'ee93a510-907a-4d7c-bbb9-59df7894xxxx'
def fuse(argv=None):
if argv is None:
argv = sys.argv[1:]
if not HAS_FUSE:
message = [
'jotta-fuse requires fusepy (pip install fusepy), install that and try again.'
]
if os.name == 'nt':
message.append(
'Note: jotta-fuse is not supported on Windows, but Cygwin might work.'
)
print(' '.join(message))
sys.exit(1)
from .jottafuse import JottaFuse
parser = argparse.ArgumentParser(
description=__doc__,
epilog=
"""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project."""
)
parser.add_argument(
'--debug',
action='store_true',
help=
'Run fuse in the foreground and add a lot of messages to help debug')
parser.add_argument('--debug-fuse',
action='store_true',
help='Show all low-level filesystem operations')
parser.add_argument('--debug-http',
action='store_true',
help='Show all HTTP traffic')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument(
'mountpoint',
type=is_dir,
help=
'A path to an existing directory where you want your JottaCloud tree mounted'
)
args = parser.parse_args(argv)
if args.debug_http:
http_client.HTTPConnection.debuglevel = 1
if args.debug:
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
logging.basicConfig(level=logging.DEBUG)
auth = JFS.get_auth_info()
fuse = FUSE(JottaFuse(auth),
args.mountpoint,
debug=args.debug_fuse,
sync_read=True,
foreground=args.debug,
raw_fi=False,
fsname="JottaCloudFS",
subtype="fuse")
def test_errors(self):
with pytest.raises(JFS.JFSCredentialsError): # HTTP 401
JFS.JFS(auth=('PYTEST','PYTEST'))
with pytest.raises(JFS.JFSNotFoundError): # HTTP 404
jfs.get('/Jotta/Archive/FileNot.found')
with pytest.raises(JFS.JFSRangeError): # HTTP 416
p = "//Jotta/Archive/testfile_up_and_readpartial.txt"
t = jfs.up(p, six.BytesIO(TESTFILEDATA))
f = jfs.getObject(p)
f.readpartial(10, 3) # Range start index larger than end index;
f.delete()
def monitor(argv=None):
if argv is None:
argv = sys.argv[1:]
if not HAS_WATCHDOG:
message = [
'jotta-monitor requires watchdog (pip install watchdog), install that and try again.'
]
print(' '.join(message))
sys.exit(1)
# Has watchdog, can safely import filemonitor
from .monitor import filemonitor
parser = argparse.ArgumentParser(
description=__doc__,
epilog=
"""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project."""
)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('--errorfile',
help='A file to write errors to',
default='./jottacloudclient.log')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument(
'--dry-run',
action='store_true',
help=
"don't actually do any uploads or deletes, just show what would be done"
)
parser.add_argument('topdir',
type=is_dir,
help='Path to local dir that needs syncing')
parser.add_argument(
'mode',
type=commandline_text,
help='Mode of operation: ARCHIVE, SYNC or SHARE. See README.md',
choices=('archive', 'sync', 'share'))
args = parse_args_and_apply_logging_level(parser, argv)
fh = logging.FileHandler(args.errorfile)
fh.setLevel(logging.ERROR)
logging.getLogger('').addHandler(fh)
jfs = JFS.JFS()
filemonitor(args.topdir, args.mode, jfs)
def cat(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Display contents of a file from Jottacloud')
parser.add_argument('file',
type=commandline_text,
help='The path to the file that you want to show')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
if args.file.startswith('//'):
# break out of root_folder
mountpoint = str.join(str('/'), args.file.split("/")[3:4])
path = str.join(str('/'), args.file.split("/")[4:])
logging.debug('mountpoint = ' + mountpoint)
logging.debug('path = ' + path)
root_folder = get_root_dir(jfs, mountpoint=mountpoint)
item_path = posixpath.join(root_folder.path, path)
# item_path = posixpath.join(jfs.rootpath, args.file[1:])
else:
root_dir = get_root_dir(jfs)
item_path = posixpath.join(root_dir.path, args.file)
item = jfs.getObject(item_path)
if not isinstance(item, JFS.JFSFile):
print("%r is not a file (it's a %s), so we can't show it" %
(args.file, type(item)))
sys.exit(1)
s = ''
for chunk in item.stream():
if isinstance(chunk, six.text_type):
print(chunk.encode(sys.getdefaultencoding()))
s = s + chunk
else:
print(chunk)
s = s + chunk.decode(sys.getdefaultencoding())
return s
def mkdir(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Create a new folder in Jottacloud.')
parser.add_argument('newdir',
type=commandline_text,
help='The path to the folder that you want to create')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
root_folder.mkdir(args.newdir)
return True # TODO: check return value of mkdir
def download_jfsfile(remote_object, tofolder=None, checksum=False):
'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
if tofolder is None:
tofolder = '.' # with no arguments, store in current dir
total_size = remote_object.size
if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete file' %
remote_file.name))
return False
topath = os.path.join(tofolder, remote_object.name)
with open(topath, 'wb') as fh:
bytes_read = 0
puts(
colored.white('Downloading: %s, size: %s \t' %
(remote_object.name,
print_size(total_size, humanize=True))))
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_object.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if checksum:
md5_lf = JFS.calculate_md5(open(topath, 'rb'))
md5_jf = remote_object.md5
logging.info('%s - Checksum for downloaded file' % md5_lf)
logging.info('%s - Checksum for server file' % md5_jf)
if md5_lf != md5_jf:
puts(colored.blue('%s - Checksum for downloaded file' %
md5_lf))
puts(colored.blue('%s - Checksum for server file' % md5_jf))
puts(
colored.red(
'%s was NOT downloaded successfully - cheksum mismatch'
% remote_object.name))
return False
puts(
colored.green(
'%s was downloaded successfully - checksum matched' %
remote_object.name))
return True
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Import JottaCloud libraries.
try:
from jottalib_ng import JFS
except ImportError:
raise BackendException(
'JottaCloud backend requires jottalib_ng'
' (see https://pypi.python.org/pypi/jottalib_ng).')
# Set jottalib_ng loggers to the same verbosity as duplicity
duplicity_log_level = get_duplicity_log_level()
set_jottalib_ng_logging_level(duplicity_log_level)
# Ensure jottalib_ng and duplicity log to the same handlers
set_jottalib_ng_log_handlers(log._logger.handlers)
# Will fetch jottacloud auth from environment or .netrc
self.client = JFS.JFS()
self.folder = self.get_or_create_directory(parsed_url.path.lstrip('/'))
log.Debug("Jottacloud folder for duplicity: %r" % self.folder.path)
def scanner(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description=__doc__,
epilog=
"""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project."""
)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('--errorfile',
type=commandline_text,
help='A file to write errors to',
default='./jottacloudclient.log')
parser.add_argument(
'--exclude',
type=re.compile,
action='append',
help='Exclude paths matched by this pattern (can be repeated)')
parser.add_argument('--prune-files',
dest='prune_files',
help='Delete files that does not exist locally',
action='store_true')
parser.add_argument('--prune-folders',
dest='prune_folders',
help='Delete folders that does not exist locally',
action='store_true')
parser.add_argument('--prune-all',
dest='prune_all',
help='Combines --prune-files and --prune-folders',
action='store_true')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument(
'--dry-run',
action='store_true',
help=
"don't actually do any uploads or deletes, just show what would be done"
)
parser.add_argument('topdir',
type=is_dir,
help='Path to local dir that needs syncing')
parser.add_argument(
'jottapath',
type=commandline_text,
help=
'The path at JottaCloud where the tree shall be synced (must exist)')
args = parse_args_and_apply_logging_level(parser, argv)
if args.prune_all:
args.prune_files = True
args.prune_folders = True
fh = logging.FileHandler(args.errorfile)
fh.setLevel(logging.ERROR)
logging.getLogger('').addHandler(fh)
jfs = JFS.JFS()
logging.info('args: topdir %r, jottapath %r', args.topdir, args.jottapath)
filescanner(args.topdir, args.jottapath, jfs, args.errorfile, args.exclude,
args.dry_run, args.prune_files, args.prune_folders)
import six from six import StringIO # import dependencies import lxml import dateutil import requests # import py.test import pytest # pip install pytest # import jotta from jottalib_ng import JFS, __version__ jfs = JFS.JFS() # get username and password from environment or .netrc TESTFILEDATA=b""" Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla est dolor, convallis fermentum sapien in, fringilla congue ligula. Fusce at justo ac felis vulputate laoreet vel at metus. Aenean justo lacus, porttitor dignissim imperdiet a, elementum cursus ligula. Vivamus eu est viverra, pretium arcu eget, imperdiet eros. Curabitur in bibendum.""" TESTFILETEXT=u""" Lørem ipsum dolor sit amet, consectetur adipiscing elit. Nulla est dolor, convallis fermentum sapien in, fringilla congue ligula. Fusce at justo ac felis vulputate laoreet vel at metus. Aenean justo lacus, porttitor dignissim imperdiet a, elementum cursus ligula.
def ls(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description='List files in Jotta folder.',
add_help=False)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument(
'-h',
'--humanize', # this matches ls(1)
help='Print human-readable file sizes.',
action='store_true')
parser.add_argument(
'-a',
'--all',
action='store_true',
help='Include deleted and incomplete files (otherwise ignored)')
parser.add_argument(
'item',
nargs='?',
help='The file or directory to list. Defaults to the root dir',
type=commandline_text)
parser.add_argument(
'-H', # because -h means --humanize
'--help',
help='Print this help',
action='help')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
if args.item:
if args.item.startswith('//'):
# break out of root_folder
mountpoint = str.join(str('/'), args.item.split("/")[3:4])
path = str.join(str('/'), args.item.split("/")[4:])
logging.debug('len: ' + str(len(args.item.split("/"))))
if len(args.item.split("/")) == 4:
path += u'/'
logging.debug('mountpoint = ' + mountpoint)
logging.debug('path = ' + path)
root_folder = get_root_dir(jfs, mountpoint=mountpoint)
item_path = posixpath.join(root_folder.path, path)
else:
item_path = posixpath.join(root_folder.path, args.item)
logging.debug('item_path = ' + item_path)
item = jfs.getObject(item_path)
else:
item = root_folder
timestamp_width = 25
logging.debug('about to ls %r', item)
logging.debug('item.instance: ' + str(item))
if isinstance(item, JFS.JFSFolder) or isinstance(item, JFS.JFSMountPoint):
logging.debug('JFSMountPoint')
logging.debug('item.files: ' + str(item.files()))
files = []
for f in item.files():
logging.debug('f: ' + str(f))
if not f is None:
files.append([
f.created,
print_size(f.size, humanize=args.humanize)
if f.size else u'', u'D' if f.deleted else
u'I' if f.state == 'INCOMPLETE' else u' ', f.name
])
# files = [(
# f.created,
# print_size(f.size, humanize=args.humanize) if f.size else u'',
# u'D' if f.deleted else u'I' if f.state == 'INCOMPLETE' else u' ',
# f.name) for f in item.files() if not f is None or (f.deleted and f.state != 'INCOMPLETE' or args.all)]
folders = [(u' ' * timestamp_width, u'', u'D' if f.deleted else u' ',
unicode(f.name)) for f in item.folders()
if not f is None or (not f.deleted or args.all)]
widest_size = 0
for f in files:
logging.debug('f in files: ' + str(f))
if len(f[1]) > widest_size:
widest_size = len(f[1])
for item in sorted(files + folders, key=lambda t: t[3]):
if args.all:
print(u'%s %s %s %s' %
(item[0], item[1].rjust(widest_size), item[2], item[3]))
else:
print(u'%s %s %s' %
(item[0], item[1].rjust(widest_size), item[3]))
# elif isinstance(item, JFS.JFSFolder):
# files = [(
# f.created,
# print_size(f.size, humanize=args.humanize) if f.size else u'',
# u'D' if f.deleted else u'I' if f.state == 'INCOMPLETE' else u' ',
# f.name) for f in item.files() if not f.deleted and f.state != 'INCOMPLETE' or args.all]
#
#
# folders = [(u' '*timestamp_width, u'', u'D' if f.deleted else u' ', unicode(f.name))
# for f in item.folders() if not f.deleted or args.all]
# widest_size = 0
# for f in files:
# if len(f[1]) > widest_size:
# widest_size = len(f[1])
# for item in sorted(files + folders, key=lambda t: t[3]):
# if args.all:
# print(u'%s %s %s %s' % (item[0], item[1].rjust(widest_size), item[2], item[3]))
# else:
# print(u'%s %s %s' % (item[0], item[1].rjust(widest_size), item[3]))
else:
print(' '.join([
str(item.created),
print_size(item.size, humanize=args.humanize), item.name
]))
return True # TODO: check return value of command
def download(argv=None):
def download_jfsfile(remote_object, tofolder=None, checksum=False):
'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
if tofolder is None:
tofolder = '.' # with no arguments, store in current dir
total_size = remote_object.size
if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete file' %
remote_file.name))
return False
topath = os.path.join(tofolder, remote_object.name)
with open(topath, 'wb') as fh:
bytes_read = 0
puts(
colored.white('Downloading: %s, size: %s \t' %
(remote_object.name,
print_size(total_size, humanize=True))))
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_object.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if checksum:
md5_lf = JFS.calculate_md5(open(topath, 'rb'))
md5_jf = remote_object.md5
logging.info('%s - Checksum for downloaded file' % md5_lf)
logging.info('%s - Checksum for server file' % md5_jf)
if md5_lf != md5_jf:
puts(colored.blue('%s - Checksum for downloaded file' %
md5_lf))
puts(colored.blue('%s - Checksum for server file' % md5_jf))
puts(
colored.red(
'%s was NOT downloaded successfully - cheksum mismatch'
% remote_object.name))
return False
puts(
colored.green(
'%s was downloaded successfully - checksum matched' %
remote_object.name))
return True
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Download a file or folder from Jottacloud.')
parser.add_argument(
'remoteobject',
help='The path to the file or folder that you want to download',
type=commandline_text)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('-c',
'--checksum',
help='Verify checksum of file after download',
action='store_true')
#parser.add_argument('-r', '--resume',
# help='Will not download the files again if it exist in path',
# action='store_true' )
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
if args.remoteobject.startswith('//'):
# break out of root_folder
root_folder = jfs.rootpath
item_path = posixpath.join(root_folder, args.remoteobject[2:])
else:
root_folder = get_root_dir(jfs).path
item_path = posixpath.join(root_folder, args.remoteobject)
logging.info('Root folder path: %s' % root_folder)
logging.info('Command line path to object: %s' % args.remoteobject)
logging.info('Jotta path to object: %s' % item_path)
remote_object = jfs.getObject(item_path)
if isinstance(remote_object, JFS.JFSFile):
if download_jfsfile(remote_object, checksum=args.checksum):
logging.info('%r downloaded successfully', remote_object.path)
return True
else:
puts(colored.red('%r download failed' % remote_object.path))
return False
else: #if it's not a file it has to be a folder
incomplete_files = [
] #Create an list where we can store incomplete files
checksum_error_files = [
] #Create an list where we can store checksum error files
zero_files = [] #Create an list where we can store zero files
long_path = [
] #Create an list where we can store skipped files and folders because of long path
puts(colored.blue("Getting index for folder: %s" % remote_object.name))
fileTree = remote_object.filedirlist().tree #Download the folder tree
puts(
colored.blue('Total number of folders to download: %d' %
len(fileTree)))
topdir = os.path.dirname(item_path)
logging.info("topdir: %r", topdir)
#Iterate through each folder
for folder in fileTree:
#We need to strip the path to the folder path from account,device and mountpoint details
logging.debug("folder: %r", folder)
_abs_folder_path = posixpath.join(JFS.JFS_ROOT, folder[1:])
logging.debug("absolute folder path : %r", _abs_folder_path)
_rel_folder_path = str.join(str('/'), folder.split('/')[4:])
logging.info('relative folder path: %r', _rel_folder_path)
if len(_rel_folder_path
) > 250: #Windows has a limit of 250 characters in path
puts(
colored.red(
'%s was NOT downloaded successfully - path too long' %
_rel_folder_path))
long_path.append(_rel_folder_path)
else:
logging.info('Entering a new folder: %s' % _rel_folder_path)
if not os.path.exists(
_rel_folder_path
): #Create the folder locally if it doesn't exist
os.makedirs(_rel_folder_path)
for _file in fileTree[
folder]: #Enter the folder and download the files within
logging.info("file: %r", _file)
#This is the absolute path to the file that is going to be downloaded
abs_path_to_object = posixpath.join(
topdir, _rel_folder_path, _file.name)
logging.info('Downloading the file from: %s' %
abs_path_to_object)
if _file.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
#Corrupt and incomplete files will be skipped
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete or corrupt file'
% _file.name))
incomplete_files.append(
posixpath.join(_rel_folder_path, _file.name))
continue
remote_object = jfs.getObject(abs_path_to_object)
remote_file = remote_object
total_size = remote_file.size
if total_size == 0: # Indicates an zero file
puts(
colored.red(
'%s was NOT downloaded successfully - zero file'
% remote_file.name))
zero_files.append(
posixpath.join(_rel_folder_path, remote_file.name))
continue
if len(
posixpath.join(_rel_folder_path, remote_file.name)
) > 250: #Windows has a limit of 250 characters in path
puts(
colored.red(
'%s was NOT downloaded successfully - path too long'
% remote_file.name))
long_path.append(
posixpath.join(_rel_folder_path, remote_file.name))
continue
#TODO: implement args.resume:
if not download_jfsfile(remote_file,
tofolder=_rel_folder_path,
checksum=args.checksum):
# download failed
puts(
colored.red("Download failed: %r" %
remote_file.path))
#Incomplete files
if len(incomplete_files) > 0:
with codecs.open("incomplete_files.txt", "w",
"utf-8") as text_file:
for item in incomplete_files:
text_file.write("%s\n" % item)
print('Incomplete files (not downloaded): %d' % len(incomplete_files))
for _files in incomplete_files:
logging.info("Incomplete: %r", _files)
#Checksum error files
if len(checksum_error_files) > 0:
with codecs.open("checksum_error_files.txt", "w",
"utf-8") as text_file:
for item in checksum_error_files:
text_file.write("%s\n" % item)
print('Files with checksum error (not downloaded): %d' %
len(checksum_error_files))
for _files in checksum_error_files:
logging.info("Checksum error: %r", _files)
#zero files
if len(zero_files) > 0:
with codecs.open("zero_files.txt", "w", "utf-8") as text_file:
for item in zero_files:
text_file.write("%s\n" % item)
print('Files with zero size (not downloaded): %d' % len(zero_files))
for _files in zero_files:
logging.info("Zero sized files: %r", _files)
#long path
if len(long_path) > 0:
with codecs.open("long_path.txt", "w", "utf-8") as text_file:
for item in long_path:
text_file.write("%s\n" % item)
print('Folder and files not downloaded because of path too long: %d' %
len(long_path))
for _files in long_path:
logging.info("Path too long: %r", _files)
return True
def test_xml(self):
xml = b"""<?xml version="1.0" encoding="UTF-8"?>
<mountPoint time="2015-09-13-T00:16:31Z" host="dn-097.site-000.jotta.no">
<name xml:space="preserve">Sync</name>
<path xml:space="preserve">/havardgulldahl//Jotta</path>
<abspath xml:space="preserve">/havardgulldahl//Jotta</abspath>
<size>39851461616</size>
<modified>2015-07-26-T22:26:54Z</modified>
<device>Jotta</device>
<user>havardgulldahl</user>
<folders>
<folder name="folder1"/>
<folder name="folder2"/>
<folder name="folder3"/>
<folder name="folder4"/>
<folder name="folder5"/>
<folder name="folder6"/>
<folder name="folder7"/>
</folders>
<files>
<file name="bigfile" uuid="7a36a217-88d5-4804-99df-bbc42eb4a9f2">
<latestRevision>
<number>1</number>
<state>INCOMPLETE</state>
<created>2015-05-29-T09:02:53Z</created>
<modified>2015-05-29-T09:02:53Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<md5>4d710d3a12699730976216836a5217a8</md5>
<updated>2015-05-29-T09:02:53Z</updated>
</latestRevision>
</file>
<file name="test.pdf" uuid="1caec763-2ed0-4e88-9d3c-650f3babecc4">
<currentRevision>
<number>3</number>
<state>COMPLETED</state>
<created>2015-07-26-T22:26:54Z</created>
<modified>2015-07-26-T22:26:54Z</modified>
<mime>application/pdf</mime>
<mstyle>APPLICATION_PDF</mstyle>
<size>116153</size>
<md5>138396327a51ea6bf20caa72cf6d6667</md5>
<updated>2015-07-26-T22:26:54Z</updated>
</currentRevision>
</file>
</files>
<metadata first="" max="" total="9" num_folders="7" num_files="2"/>
</mountPoint>"""
o = lxml.objectify.fromstring(xml)
dev = JFS.JFSMountPoint(o, jfs, parentpath=jfs.rootpath + '/Jotta')
#test native properties
assert dev.name == 'Sync'
assert dev.size == 39851461616
assert isinstance(dev.modified, datetime.datetime)
assert dev.modified == datetime.datetime(2015, 7, 26, 22, 26, 54).replace(tzinfo=dateutil.tz.tzutc())
with pytest.raises(JFS.JFSError):
dev.delete()
dev.rename('sillywalkministry')
# test JFSFolder inheritance
assert dev.path == jfs.rootpath + '/Jotta/Sync'
assert dev.deleted == None
assert dev.is_deleted() == False
assert all(isinstance(item, (JFS.JFSFile, JFS.JFSIncompleteFile)) for item in dev.files())
assert all(isinstance(item, JFS.JFSFolder) for item in dev.folders())
newf = dev.mkdir('testdir')
assert isinstance(newf, JFS.JFSFolder)
newf.delete()
_f = tempfile.NamedTemporaryFile()
_f.write(b'123test')
newfile = dev.up(_f)
assert isinstance(newfile, JFS.JFSFile)
newfile.delete()
newfile = dev.up(_f, filename='heyhei123.txt')
assert isinstance(newfile, JFS.JFSFile)
assert newfile.name == 'heyhei123.txt'
newfile.delete()
assert isinstance(dev.filedirlist(), JFS.JFSFileDirList)
def __init__(self, auth, path='.'):
self.client = JFS.JFS(auth)
self.__newfiles = {} # a dict of stringio objects
self.__newfolders = []
self.ino = 0
def test_xml(self):
xml = b"""<?xml version="1.0" encoding="UTF-8"?>
<file name="testimage.jpg" uuid="9ebcfe1a-98b1-4e38-a73e-f498555da865" time="2015-09-13-T21:22:46Z" host="dn-094.site-000.jotta.no">
<path xml:space="preserve">/havardgulldahl/Jotta/Archive</path>
<abspath xml:space="preserve">/havardgulldahl/Jotta/Archive</abspath>
<currentRevision>
<number>5</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:18:49Z</created>
<modified>2015-07-25-T21:18:49Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:18:50Z</updated>
</currentRevision>
<revisions>
<revision>
<number>4</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:18:05Z</created>
<modified>2015-07-25-T21:18:05Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:18:07Z</updated>
</revision>
<revision>
<number>3</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:17:49Z</created>
<modified>2015-07-25-T21:17:49Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:17:50Z</updated>
</revision>
<revision>
<number>2</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:01:45Z</created>
<modified>2015-07-25-T21:01:45Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:01:46Z</updated>
</revision>
<revision>
<number>1</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:00:02Z</created>
<modified>2015-07-25-T21:00:02Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:00:03Z</updated>
</revision>
</revisions>
</file>"""
o = lxml.objectify.fromstring(xml)
dev = JFS.JFSFile(o, jfs, parentpath=jfs.rootpath + '/Jotta/Archive')
#test ProtoFile properties
assert dev.path == jfs.rootpath + '/Jotta/Archive/testimage.jpg'
assert dev.name == 'testimage.jpg'
assert dev.uuid == '9ebcfe1a-98b1-4e38-a73e-f498555da865'
assert dev.deleted == None
assert dev.is_deleted() == False
#test native properties
assert dev.revisionNumber == 5
assert dev.created == datetime.datetime(2015, 7, 25, 21, 18, 49).replace(tzinfo=dateutil.tz.tzutc())
assert dev.modified == datetime.datetime(2015, 7, 25, 21, 18, 49).replace(tzinfo=dateutil.tz.tzutc())
assert dev.updated == datetime.datetime(2015, 7, 25, 21, 18, 50).replace(tzinfo=dateutil.tz.tzutc())
assert dev.size == 1816221
assert dev.md5 == '125073533339a616b99bc53efc509561'
assert dev.mime == 'image/jpeg'
assert dev.state == 'COMPLETED'
units = ['B','KiB','MiB','GiB','TiB','PiB','EiB','ZiB','YiB']
p = math.floor(math.log(size, 2)/10)
return "%.3f%s" % (size/math.pow(1024,p),units[int(p)])
if __name__ == '__main__':
# we need an active login to test
import netrc
try:
n = netrc.netrc()
username, account, password = n.authenticators('jottacloud.com') # read .netrc entry for 'machine jottacloud'
except Exception as e:
log.exception(e)
username = os.environ['JOTTACLOUD_USERNAME']
password = os.environ['JOTTACLOUD_PASSWORD']
jfs = JFS.JFS(auth=(username, password))
lite = LiteJFS(username, password)
filesize = 1024*10*10
data = os.urandom(filesize)
testfile = tempfile.NamedTemporaryFile()
puts(colored.blue('Creating test file.'))
for i in progress.bar(range(0, 1000)):
testfile.write(data)
filesize = os.path.getsize(testfile.name)
p = '/Jotta/Archive/test/%s.data' % os.path.basename(testfile.name)
# UPLOAD TEST
puts(colored.green('Test 1 | requests | Upload | File size: %s' % humanizeFileSize(filesize)))
def test_xml(self):
xml = b"""<?xml version="1.0" encoding="UTF-8"?>
<folder name="test2" time="2015-09-28-T13:49:05Z" host="dn-091.site-000.jotta.no">
<path xml:space="preserve">/havardgulldahl/Jotta/Archive</path>
<abspath xml:space="preserve">/havardgulldahl/Jotta/Archive</abspath>
<folders>
<folder name="Documents"/>
</folders>
<files>
<file name="boink.txt~" uuid="c6684726-a842-4536-95b9-140584515dd1">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>40</size>
<md5>b924ebbc79ad414ded4af442ac7080d3</md5>
<updated>2014-11-23-T21:12:47Z</updated>
</currentRevision>
</file>
<file name="boink.txx~" uuid="1b243d8e-d6df-412c-a2ce-926ebaa73f47">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>40</size>
<md5>0c7652132733ead903dbdb942577fed7</md5>
<updated>2014-11-23-T21:27:21Z</updated>
</currentRevision>
</file>
<file name="boink.txy~" uuid="ba1ec941-8901-4eec-b797-bde9b6b1958c">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>40</size>
<md5>d32f37bf39041d9bb92ad45012e32cb9</md5>
<updated>2014-11-23-T21:05:11Z</updated>
</currentRevision>
</file>
<file name="boink.txz~" uuid="6724af96-e462-4862-b82a-00b7836a0681">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>9</size>
<md5>e3a2cba90ec7630bdf1d0566c8abb93e</md5>
<updated>2014-11-23-T21:03:25Z</updated>
</currentRevision>
</file>
<file name="dingdong" uuid="95c4bbcc-9a59-4669-b4cb-ee49c186ae1b">
<currentRevision>
<number>127</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>20480</size>
<md5>daa100df6e6711906b61c9ab5aa16032</md5>
<updated>2014-11-23-T22:56:30Z</updated>
</currentRevision>
</file>
<file name="fisk.txt" uuid="8b6db048-c44b-4656-8024-737a0c38c7ad">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>text/plain</mime>
<mstyle>TEXT_PLAIN</mstyle>
<size>18</size>
<md5>308285a59ae0a4a5ede4f7a0c08d2390</md5>
<updated>2014-11-23-T19:56:51Z</updated>
</currentRevision>
</file>
<file name="nyboink.txt" uuid="3d0a0a28-4c68-4e6a-8414-fa5c37ec45cb">
<currentRevision>
<number>26</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>text/plain</mime>
<mstyle>TEXT_PLAIN</mstyle>
<size>6</size>
<md5>6882bf1ef7f4d938a4cf2931d6953fa1</md5>
<updated>2014-11-23-T21:28:55Z</updated>
</currentRevision>
</file>
<file name="oo.txt" uuid="691aa9b6-eec1-4564-803e-6288dc57aa37">
<currentRevision>
<number>3</number>
<state>COMPLETED</state>
<created>2014-12-18-T21:23:48Z</created>
<modified>2014-12-18-T21:23:48Z</modified>
<mime>text/plain</mime>
<mstyle>TEXT_PLAIN</mstyle>
<size>4</size>
<md5>aa3f5bb8c988fa9b75a1cdb1dc4d93fc</md5>
<updated>2014-12-18-T21:23:48Z</updated>
</currentRevision>
</file>
<file name="pingpong.data" uuid="f02f1ca1-c6a4-403d-b344-f4e3417d92fd">
<currentRevision>
<number>165</number>
<state>COMPLETED</state>
<created>2014-12-18-T21:20:32Z</created>
<modified>2014-12-18-T21:20:32Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>9216</size>
<md5>ec041186ebff92a26ab3ef2dd34dd0e7</md5>
<updated>2014-12-18-T21:20:32Z</updated>
</currentRevision>
</file>
<file name="testfs" uuid="a00034dd-971c-4699-8ee0-e7514045770e">
<currentRevision>
<number>2</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>0</size>
<md5>d41d8cd98f00b204e9800998ecf8427e</md5>
<updated>2014-11-23-T19:41:03Z</updated>
</currentRevision>
</file>
</files>
<metadata first="" max="" total="11" num_folders="1" num_files="10"/>
</folder>
"""
o = lxml.objectify.fromstring(xml)
dev = JFS.JFSFolder(o, jfs, parentpath=jfs.rootpath + '/Jotta/Archive')
dev.synced = True # make sure our tests are based on the xml above, and not live
#test native properties
assert dev.path == jfs.rootpath + '/Jotta/Archive/test2'
assert dev.name == 'test2'
assert dev.deleted == None
assert dev.is_deleted() == False
#test convenience methods
assert len(list(dev.folders())) == 1
assert len(list(dev.files())) == 10
assert all(isinstance(item, JFS.JFSFile) for item in dev.files())
assert all(isinstance(item, JFS.JFSFolder) for item in dev.folders())