def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Import JottaCloud libraries.
try:
from jottalib import JFS
except ImportError:
raise
raise BackendException('JottaCloud backend requires jottalib'
' (see https://pypi.python.org/pypi/jottalib).')
# Setup client instance.
_pass = os.environ.get('JOTTACLOUD_PASSWORD', None)
if _pass is None:
_pass = self.get_password()
username = parsed_url.username or os.environ.get('JOTTACLOUD_USERNAME')
self.client = JFS.JFS(auth=(username, _pass))
#self.client.http_client.debug = False
root_dir = get_root_dir(self.client)
# Fetch destination folder entry (and create hierarchy if required).
path = posixpath.join([root_dir.path, parsed_url.path.lstrip('/')])
try:
# self.folder = root_dir#self.client.getObject('%s/duplicity' % parsed_url.path.lstrip('//'))
self.folder = self.client.getObject(path)
except JFS.JFSNotFoundError:
try:
self.folder = root_dir.mkdir(parsed_url.path.lstrip('/'))
except:
raise
raise BackendException("Error while creating destination folder 'Backup')")
except:
raise
def monitor():
if not HAS_WATCHDOG:
message = ['jotta-monitor requires watchdog (pip install watchdog), install that and try again.']
print(' '.join(message))
sys.exit(1)
# Has watchdog, can safely import filemonitor
from .monitor import filemonitor
def is_dir(path):
if not os.path.isdir(path):
raise argparse.ArgumentTypeError('%s is not a valid directory' % path)
return path
parser = argparse.ArgumentParser(description=__doc__,
epilog="""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project.""")
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
parser.add_argument('--errorfile', help='A file to write errors to', default='./jottacloudclient.log')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument('--dry-run', action='store_true',
help="don't actually do any uploads or deletes, just show what would be done")
parser.add_argument('topdir', type=is_dir, help='Path to local dir that needs syncing')
parser.add_argument('mode', help='Mode of operation: ARCHIVE, SYNC or SHARE. See README.md',
choices=( 'archive', 'sync', 'share') )
args = parse_args_and_apply_logging_level(parser)
fh = logging.FileHandler(args.errorfile)
fh.setLevel(logging.ERROR)
logging.getLogger('').addHandler(fh)
jfs = JFS()
filemonitor(args.topdir, args.mode, jfs)
def download(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Download a file from Jottacloud.')
parser.add_argument('remotefile',
help='The path to the file that you want to download')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
path_to_object = posixpath.join(root_folder.path, args.remotefile)
remote_file = jfs.getObject(path_to_object)
total_size = remote_file.size
with open(remote_file.name, 'wb') as fh:
bytes_read = 0
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_file.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
print('%s downloaded successfully' % args.remotefile)
def download_jfsfile(remote_object, tofolder=None, checksum=False):
'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
if tofolder is None:
tofolder = '.' # with no arguments, store in current dir
total_size = remote_object.size
if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT, JFS.ProtoFile.STATE_INCOMPLETE):
puts(colored.red('%s was NOT downloaded successfully - Incomplete file' % remote_file.name))
return False
topath = os.path.join(tofolder, remote_object.name)
with open(topath, 'wb') as fh:
bytes_read = 0
puts(colored.white('Downloading: %s, size: %s \t' % (remote_object.name,
print_size(total_size, humanize=True))))
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_object.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if checksum:
md5_lf = JFS.calculate_md5(open(topath, 'rb'))
md5_jf = remote_object.md5
logging.info('%s - Checksum for downloaded file' % md5_lf)
logging.info('%s - Checksum for server file' % md5_jf)
if md5_lf != md5_jf:
puts(colored.blue('%s - Checksum for downloaded file' % md5_lf))
puts(colored.blue('%s - Checksum for server file' % md5_jf))
puts(colored.red('%s was NOT downloaded successfully - cheksum mismatch' % remote_object.name))
return False
puts(colored.green('%s was downloaded successfully - checksum matched' % remote_object.name))
return True
def cat(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Display contents of a file from Jottacloud')
parser.add_argument('file',
type=commandline_text,
help='The path to the file that you want to show')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
if args.file.startswith('//'):
# break out of root_folder
item_path = posixpath.join(jfs.rootpath, args.file[1:])
else:
root_dir = get_root_dir(jfs)
item_path = posixpath.join(root_dir.path, args.file)
item = jfs.getObject(item_path)
if not isinstance(item, JFS.JFSFile):
print("%r is not a file (it's a %s), so we can't show it" %
(args.file, type(item)))
sys.exit(1)
s = ''
for chunk in item.stream():
print(chunk.encode(sys.getdefaultencoding()))
s = s + chunk
return s
def share(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Share a file on JottaCloud and get the public URI.',
epilog='Note: This utility needs to find JOTTACLOUD_USERNAME'
' and JOTTACLOUD_PASSWORD in the running environment.')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('localfile',
help='The local file that you want to share',
type=argparse.FileType('r'))
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
jottadev = get_jfs_device(jfs)
jottashare = jottadev.mountPoints['Shared']
upload = jottashare.up(args.localfile) # upload file
public = upload.share() # share file
logging.debug('Shared %r and got: %r (%s)', args.localfile, public,
dir(public))
for (filename, uuid, publicURI) in public.sharedFiles():
print('%s is now available to the world at %s' % (filename, publicURI))
return True # TODO: check return value of command
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Import JottaCloud libraries.
try:
from jottalib import JFS
from jottalib.JFS import JFSNotFoundError, JFSIncompleteFile
except ImportError:
raise BackendException(
u'JottaCloud backend requires jottalib'
u' (see https://pypi.python.org/pypi/jottalib).')
# Set jottalib loggers to the same verbosity as duplicity
duplicity_log_level = get_duplicity_log_level()
set_jottalib_logging_level(duplicity_log_level)
# Ensure jottalib and duplicity log to the same handlers
set_jottalib_log_handlers(log._logger.handlers)
# Will fetch jottacloud auth from environment or .netrc
self.client = JFS.JFS()
self.folder = self.get_or_create_directory(
parsed_url.path.lstrip(u'/'))
log.Debug(u"Jottacloud folder for duplicity: %r" % self.folder.path)
def rm(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Delete an item from Jottacloud')
parser.add_argument('file',
type=commandline_text,
help='The path to the item that you want to delete')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('-f',
'--force',
help='Completely deleted, no restore possiblity',
action='store_true')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_dir = get_root_dir(jfs)
item_path = posixpath.join(root_dir.path, args.file)
item = jfs.getObject(item_path)
if args.force:
item.hard_delete()
else:
item.delete()
print('%s deleted' % args.file)
return True # TODO: check return value of command
def upload(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Upload a file to JottaCloud.')
parser.add_argument('localfile',
help='The local file that you want to upload',
type=argparse.FileType('r'))
parser.add_argument('remote_dir',
help='The remote directory to upload the file to',
nargs='?',
type=commandline_text)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
jfs = JFS.JFS()
args = parse_args_and_apply_logging_level(parser, argv)
decoded_filename = commandline_text(args.localfile.name)
progress_bar = ProgressBar()
callback = lambda monitor, size: progress_bar.show(monitor.bytes_read, size
)
root_folder = get_root_dir(jfs)
if args.remote_dir:
target_dir_path = posixpath.join(root_folder.path, args.remote_dir)
target_dir = jfs.getObject(target_dir_path)
else:
target_dir = root_folder
upload = target_dir.up(args.localfile,
os.path.basename(decoded_filename),
upload_callback=callback)
print('%s uploaded successfully' % decoded_filename)
return True # TODO: check return value
def __init__(self, username, password, path='.'):
self.client = JFS.JFS(username, password)
self.root = path
self.dirty = False # True if some method has changed/added something and we need to get fresh data from JottaCloud
# TODO: make self.dirty more smart, to know what path, to get from cache and not
self.__newfiles = []
self.__newfolders = []
def scanner():
def is_dir(path):
if not os.path.isdir(path):
raise argparse.ArgumentTypeError('%s is not a valid directory' % path)
return path
parser = argparse.ArgumentParser(description=__doc__,
epilog="""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project.""")
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
parser.add_argument('--errorfile', help='A file to write errors to', default='./jottacloudclient.log')
parser.add_argument('--exclude', type=re.compile, action='append', help='Exclude paths matched by this pattern (can be repeated)')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument('--dry-run', action='store_true',
help="don't actually do any uploads or deletes, just show what would be done")
parser.add_argument('topdir', type=is_dir, help='Path to local dir that needs syncing')
parser.add_argument('jottapath', help='The path at JottaCloud where the tree shall be synced (must exist)')
args = parse_args_and_apply_logging_level(parser)
fh = logging.FileHandler(args.errorfile)
fh.setLevel(logging.ERROR)
logging.getLogger('').addHandler(fh)
jfs = JFS()
filescanner(args.topdir, args.jottapath, jfs, args.exclude, args.dry_run)
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Import JottaCloud libraries.
try:
from jottalib import JFS
except ImportError:
raise
raise BackendException(
'JottaCloud backend requires jottalib'
' (see https://pypi.python.org/pypi/jottalib).')
# Setup client instance.
_pass = os.environ.get('JOTTACLOUD_PASSWORD', None)
if _pass is None:
_pass = self.get_password()
self.client = JFS.JFS(parsed_url.username, _pass)
#self.client.http_client.debug = False
# Fetch destination folder entry (and create hierarchy if required).
try:
self.folder = self.client.getObject('%s/backup' % parsed_url.path)
except JFS.JFSNotFoundError:
parentfolder = self.client.getObject(parsed_url.path)
try:
self.folder = parentfolder.mkdir('backup')
except:
raise
raise BackendException(
"Error while creating destination folder 'Backup')")
except:
raise
def test_xml(self):
xml = """<?xml version="1.0" encoding="UTF-8"?>
<device time="2015-09-12-T23:14:25Z" host="dn-093.site-000.jotta.no">
<name xml:space="preserve">Jotta</name>
<type>JOTTA</type>
<sid>ee93a510-907a-4d7c-bbb9-59df7894xxxx</sid>
<size>58428516774</size>
<modified>2015-09-12-T23:10:51Z</modified>
<user>havardgulldahl</user>
<mountPoints>
<mountPoint>
<name xml:space="preserve">Archive</name>
<size>18577011381</size>
<modified>2015-09-12-T23:10:51Z</modified>
</mountPoint>
<mountPoint>
<name xml:space="preserve">Shared</name>
<size>43777</size>
<modified>2015-09-03-T21:12:55Z</modified>
</mountPoint>
<mountPoint>
<name xml:space="preserve">Sync</name>
<size>39851461616</size>
<modified>2015-07-26-T22:26:54Z</modified>
</mountPoint>
</mountPoints>
<metadata first="" max="" total="3" num_mountpoints="3"/>
</device>"""
o = lxml.objectify.fromstring(xml)
dev = JFS.JFSDevice(o, jfs, parentpath=jfs.rootpath)
assert isinstance(o, lxml.objectify.ObjectifiedElement)
# Test that mountpoints are populated correctly"
assert sorted(dev.mountPoints.keys()) == ['Archive', 'Shared', 'Sync']
assert all(
isinstance(item, JFS.JFSMountPoint)
for item in dev.mountPoints.values())
# test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
assert all(
isinstance(item, JFS.JFSFile) for item in dev.files('Archive'))
# test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
mps = dev.mountpointobjects()
assert all(isinstance(item, JFS.JFSFile) for item in dev.files(mps[2]))
#test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
assert all(
isinstance(item, JFS.JFSFolder) for item in dev.folders('Archive'))
#test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
mps = dev.mountpointobjects()
assert all(
isinstance(item, JFS.JFSFolder) for item in dev.folders(mps[0]))
# test_properties
assert isinstance(dev.modified, datetime.datetime)
assert dev.path == '%s/%s' % (jfs.rootpath, 'Jotta')
assert dev.name == 'Jotta'
assert dev.type == 'JOTTA'
assert dev.size == 58428516774
assert dev.sid == 'ee93a510-907a-4d7c-bbb9-59df7894xxxx'
def fuse(argv=None):
if argv is None:
argv = sys.argv[1:]
if not HAS_FUSE:
message = [
'jotta-fuse requires fusepy (pip install fusepy), install that and try again.'
]
if os.name == 'nt':
message.append(
'Note: jotta-fuse is not supported on Windows, but Cygwin might work.'
)
print(' '.join(message))
sys.exit(1)
from .jottafuse import JottaFuse
parser = argparse.ArgumentParser(
description=__doc__,
epilog=
"""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project."""
)
parser.add_argument(
'--debug',
action='store_true',
help=
'Run fuse in the foreground and add a lot of messages to help debug')
parser.add_argument('--debug-fuse',
action='store_true',
help='Show all low-level filesystem operations')
parser.add_argument('--debug-http',
action='store_true',
help='Show all HTTP traffic')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument(
'mountpoint',
type=is_dir,
help=
'A path to an existing directory where you want your JottaCloud tree mounted'
)
args = parser.parse_args(argv)
if args.debug_http:
http_client.HTTPConnection.debuglevel = 1
if args.debug:
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
logging.basicConfig(level=logging.DEBUG)
auth = JFS.get_auth_info()
fuse = FUSE(JottaFuse(auth),
args.mountpoint,
debug=args.debug_fuse,
sync_read=True,
foreground=args.debug,
raw_fi=False,
fsname="JottaCloudFS",
subtype="fuse")
def mkdir():
parser = argparse.ArgumentParser(description='Create a new folder in Jottacloud.')
parser.add_argument('newdir', help='The path to the folder that you want to create')
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
args = parse_args_and_apply_logging_level(parser)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
root_folder.mkdir(args.newdir)
def test_errors(self):
with pytest.raises(JFS.JFSCredentialsError): # HTTP 401
JFS.JFS(auth=('PYTEST', 'PYTEST'))
with pytest.raises(JFS.JFSNotFoundError): # HTTP 404
jfs.get('/Jotta/Archive/FileNot.found')
with pytest.raises(JFS.JFSRangeError): # HTTP 416
p = "/Jotta/Archive/testfile_up_and_readpartial.txt"
t = jfs.up(p, StringIO.StringIO(TESTFILEDATA))
f = jfs.getObject(p)
f.readpartial(10, 3) # Range start index larger than end index;
f.delete()
def restore():
parser = argparse.ArgumentParser(description='Restore a deleted item from Jottacloud')
parser.add_argument('file', help='The path to the item that you want to restore')
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
args = parse_args_and_apply_logging_level(parser)
jfs = JFS.JFS()
root_dir = get_root_dir(jfs)
item_path = posixpath.join(root_dir.path, args.file)
item = jfs.getObject(item_path)
item.restore()
print '%s restored' % args.file
def ls(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description='List files in Jotta folder.',
add_help=False)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('-h',
'--humanize',
help='Print human-readable file sizes.',
action='store_true')
parser.add_argument('-a',
'--all',
help='Show all files, even deleted ones',
action='store_true')
parser.add_argument('item',
nargs='?',
help='The file or directory to list. Defaults to the '
'root dir')
parser.add_argument('-H', '--help', help='Print this help', action='help')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
if args.item:
item_path = posixpath.join(root_folder.path, args.item)
item = jfs.getObject(item_path)
else:
item = root_folder
if isinstance(item, JFS.JFSFolder):
files = [(f.created, print_size(f.size, humanize=args.humanize),
u'D' if f.deleted else u' ', f.name) for f in item.files()
if not f.deleted or args.all]
folders = [(u' ' * 25, u'', u'D' if f.deleted else u' ', f.name)
for f in item.folders()]
widest_size = 0
for f in files:
if len(f[1]) > widest_size:
widest_size = len(f[1])
for item in sorted(files + folders, key=lambda t: t[3]):
if args.all:
print(u'%s %s %s %s' %
(item[0], item[1].rjust(widest_size), item[2], item[3]))
else:
print(u'%s %s %s' %
(item[0], item[1].rjust(widest_size), item[3]))
else:
print(' '.join([
str(item.created),
print_size(item.size, humanize=args.humanize), item.name
]))
def share():
parser = argparse.ArgumentParser(description='Share a file on JottaCloud and get the public URI.',
epilog='Note: This utility needs to find JOTTACLOUD_USERNAME'
' and JOTTACLOUD_PASSWORD in the running environment.')
parser.add_argument('localfile', help='The local file that you want to share',
type=argparse.FileType('r'))
args = parser.parse_args()
jfs = JFS.JFS()
jottadev = get_jotta_device(jfs)
jottashare = jottadev.mountPoints['Shared']
upload = jottashare.up(args.localfile) # upload file
public = upload.share() # share file
for (filename, uuid, publicURI) in public.sharedFiles():
print '%s is now available to the world at %s' % (filename, publicURI)
def mkdir(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Create a new folder in Jottacloud.')
parser.add_argument('newdir',
type=commandline_text,
help='The path to the folder that you want to create')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
root_folder.mkdir(args.newdir)
return True # TODO: check return value of mkdir
def fuse(argv=None):
if argv is None:
argv = sys.argv[1:]
if not HAS_FUSE:
message = ['jotta-fuse requires fusepy (pip install fusepy), install that and try again.']
if os.name == 'nt':
message.append('Note: jotta-fuse is not supported on Windows, but Cygwin might work.')
print(' '.join(message))
sys.exit(1)
from .jottafuse import JottaFuse
parser = argparse.ArgumentParser(description=__doc__,
epilog="""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project.""")
parser.add_argument('--debug',
action='store_true',
help='Run fuse in the foreground and add a lot of messages to help debug')
parser.add_argument('--debug-fuse',
action='store_true',
help='Show all low-level filesystem operations')
parser.add_argument('--debug-http',
action='store_true',
help='Show all HTTP traffic')
parser.add_argument('--version',
action='version', version=__version__)
parser.add_argument('mountpoint',
type=is_dir,
help='A path to an existing directory where you want your JottaCloud tree mounted')
args = parser.parse_args(argv)
if args.debug_http:
http_client.HTTPConnection.debuglevel = 1
if args.debug:
requests_log = logging.getLogger("requests.packages.urllib3")
requests_log.setLevel(logging.DEBUG)
requests_log.propagate = True
logging.basicConfig(level=logging.DEBUG)
auth = JFS.get_auth_info()
fuse = FUSE(JottaFuse(auth), args.mountpoint, debug=args.debug_fuse,
sync_read=True, foreground=args.debug, raw_fi=False,
fsname="JottaCloudFS", subtype="fuse")
def download_jfsfile(remote_object, tofolder=None, checksum=False):
'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
if tofolder is None:
tofolder = '.' # with no arguments, store in current dir
total_size = remote_object.size
if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete file' %
remote_file.name))
return False
topath = os.path.join(tofolder, remote_object.name)
with open(topath, 'wb') as fh:
bytes_read = 0
puts(
colored.white('Downloading: %s, size: %s \t' %
(remote_object.name,
print_size(total_size, humanize=True))))
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_object.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if checksum:
md5_lf = JFS.calculate_md5(open(topath, 'rb'))
md5_jf = remote_object.md5
logging.info('%s - Checksum for downloaded file' % md5_lf)
logging.info('%s - Checksum for server file' % md5_jf)
if md5_lf != md5_jf:
puts(colored.blue('%s - Checksum for downloaded file' %
md5_lf))
puts(colored.blue('%s - Checksum for server file' % md5_jf))
puts(
colored.red(
'%s was NOT downloaded successfully - cheksum mismatch'
% remote_object.name))
return False
puts(
colored.green(
'%s was downloaded successfully - checksum matched' %
remote_object.name))
return True
def test_xml(self):
xml = """<?xml version="1.0" encoding="UTF-8"?>
<file name="testimage.jpg" uuid="9ebcfe1a-98b1-4e38-a73e-f498555da865" time="2015-09-13-T21:22:46Z" host="dn-094.site-000.jotta.no">
<path xml:space="preserve">/havardgulldahl/Jotta/Archive</path>
<abspath xml:space="preserve">/havardgulldahl/Jotta/Archive</abspath>
<currentRevision>
<number>5</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:18:49Z</created>
<modified>2015-07-25-T21:18:49Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:18:50Z</updated>
</currentRevision>
<revisions>
<revision>
<number>4</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:18:05Z</created>
<modified>2015-07-25-T21:18:05Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:18:07Z</updated>
</revision>
<revision>
<number>3</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:17:49Z</created>
<modified>2015-07-25-T21:17:49Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:17:50Z</updated>
</revision>
<revision>
<number>2</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:01:45Z</created>
<modified>2015-07-25-T21:01:45Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:01:46Z</updated>
</revision>
<revision>
<number>1</number>
<state>COMPLETED</state>
<created>2015-07-25-T21:00:02Z</created>
<modified>2015-07-25-T21:00:02Z</modified>
<mime>image/jpeg</mime>
<mstyle>IMAGE_JPEG</mstyle>
<size>1816221</size>
<md5>125073533339a616b99bc53efc509561</md5>
<updated>2015-07-25-T21:00:03Z</updated>
</revision>
</revisions>
</file>"""
o = lxml.objectify.fromstring(xml)
dev = JFS.JFSFile(o, jfs, parentpath=jfs.rootpath + '/Jotta/Archive')
#test ProtoFile properties
assert dev.path == jfs.rootpath + '/Jotta/Archive/testimage.jpg'
assert dev.name == 'testimage.jpg'
assert dev.uuid == '9ebcfe1a-98b1-4e38-a73e-f498555da865'
assert dev.deleted == None
assert dev.is_deleted() == False
#test native properties
assert dev.revisionNumber == 5
assert dev.created == datetime.datetime(
2015, 7, 25, 21, 18, 49).replace(tzinfo=dateutil.tz.tzutc())
assert dev.modified == datetime.datetime(
2015, 7, 25, 21, 18, 49).replace(tzinfo=dateutil.tz.tzutc())
assert dev.updated == datetime.datetime(
2015, 7, 25, 21, 18, 50).replace(tzinfo=dateutil.tz.tzutc())
assert dev.size == 1816221
assert dev.md5 == '125073533339a616b99bc53efc509561'
assert dev.mime == 'image/jpeg'
assert dev.state == 'COMPLETED'
def test_xml(self):
xml = """<?xml version="1.0" encoding="UTF-8"?>
<mountPoint time="2015-09-13-T00:16:31Z" host="dn-097.site-000.jotta.no">
<name xml:space="preserve">Sync</name>
<path xml:space="preserve">/havardgulldahl/Jotta</path>
<abspath xml:space="preserve">/havardgulldahl/Jotta</abspath>
<size>39851461616</size>
<modified>2015-07-26-T22:26:54Z</modified>
<device>Jotta</device>
<user>havardgulldahl</user>
<folders>
<folder name="folder1"/>
<folder name="folder2"/>
<folder name="folder3"/>
<folder name="folder4"/>
<folder name="folder5"/>
<folder name="folder6"/>
<folder name="folder7"/>
</folders>
<files>
<file name="bigfile" uuid="7a36a217-88d5-4804-99df-bbc42eb4a9f2">
<latestRevision>
<number>1</number>
<state>INCOMPLETE</state>
<created>2015-05-29-T09:02:53Z</created>
<modified>2015-05-29-T09:02:53Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<md5>4d710d3a12699730976216836a5217a8</md5>
<updated>2015-05-29-T09:02:53Z</updated>
</latestRevision>
</file>
<file name="test.pdf" uuid="1caec763-2ed0-4e88-9d3c-650f3babecc4">
<currentRevision>
<number>3</number>
<state>COMPLETED</state>
<created>2015-07-26-T22:26:54Z</created>
<modified>2015-07-26-T22:26:54Z</modified>
<mime>application/pdf</mime>
<mstyle>APPLICATION_PDF</mstyle>
<size>116153</size>
<md5>138396327a51ea6bf20caa72cf6d6667</md5>
<updated>2015-07-26-T22:26:54Z</updated>
</currentRevision>
</file>
</files>
<metadata first="" max="" total="9" num_folders="7" num_files="2"/>
</mountPoint>"""
o = lxml.objectify.fromstring(xml)
dev = JFS.JFSMountPoint(o, jfs, parentpath=jfs.rootpath + '/Jotta')
#test native properties
assert dev.name == 'Sync'
assert dev.size == 39851461616
assert isinstance(dev.modified, datetime.datetime)
assert dev.modified == datetime.datetime(
2015, 7, 26, 22, 26, 54).replace(tzinfo=dateutil.tz.tzutc())
with pytest.raises(JFS.JFSError):
dev.delete()
dev.rename('sillywalkministry')
# test JFSFolder inheritance
assert dev.path == jfs.rootpath + '/Jotta/Sync'
assert dev.deleted == None
assert dev.is_deleted() == False
assert all(isinstance(item, JFS.JFSFile) for item in dev.files())
assert all(isinstance(item, JFS.JFSFolder) for item in dev.folders())
newf = dev.mkdir('testdir')
assert isinstance(newf, JFS.JFSFolder)
newf.delete()
_f = tempfile.NamedTemporaryFile()
_f.write('123test')
newfile = dev.up(_f)
assert isinstance(newfile, JFS.JFSFile)
newfile.delete()
newfile = dev.up(_f, filename='heyhei123.txt')
assert isinstance(newfile, JFS.JFSFile)
assert newfile.name == 'heyhei123.txt'
newfile.delete()
assert isinstance(dev.filedirlist(), JFS.JFSFileDirList)
def download(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description='Download a file or folder from Jottacloud.')
parser.add_argument('remoteobject', help='The path to the file or folder that you want to download')
parser.add_argument('-m', '--mode',
help='Mode of operation(decides where the default root should be): DEVICE - Root is above device level (need to specify both Device and Mountpoint in front of the folder i.e. DEVICENAME/mountpoint/folder), ARCHIVE - root is in the archive mountpoint, SYNC - root is in sync mountpoint, Default: %(default)s.',
choices=( 'device', 'archive', 'sync'), default='sync')
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
parser.add_argument('-c', '--checksum', help='Verify checksum of file after download', action='store_true' )
parser.add_argument('-r', '--resume', help='Will not download the files again if it exist in path', action='store_true' )
parser.add_argument('-v', '--verbose', help='Increase output verbosity', action='store_true' )
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
if args.mode == 'sync': #Device is Jotta and mountpoint is Sync
device = 'Jotta'
mountpoint = 'Sync'
print('Device is: %s' % device)
print('Mountpoint is: %s' % mountpoint)
root_folder = get_root_dir(jfs,device,mountpoint)
elif args.mode == 'archive': #Device is Jotta and mountpoint is Archive
device = 'Jotta'
mountpoint = 'Archive'
print('Device is: %s' % device)
print('Mountpoint is: %s' % mountpoint)
root_folder = get_root_dir(jfs,device,mountpoint)
elif args.mode == 'device': #Need to get the Device from first part of path and mountpoint from second part
path_in_parts = os.path.normpath(args.remoteobject).split(os.sep)
device = path_in_parts[0]
mountpoint = path_in_parts[1]
print('Device is: %s' % device)
print('Mountpoint is: %s' % mountpoint)
if len(path_in_parts)<3:
print('You need to specify at least one folder of file in addition to the device and mountpoint (i.e. 3 levels)')
exit(1)
else:
root_folder = get_root_dir(jfs,device,mountpoint)
del path_in_parts[0:2] #Removing device and mountpoint from path
args.remoteobject = "/".join(path_in_parts)
else:
exit(1)#This shouldn't really be necessary
path_to_object = posixpath.join(root_folder.path, args.remoteobject)
if args.verbose:
print('Root folder path: %s' % root_folder.path)
print('Relative path to object: %s' % args.remoteobject)
print('Absolute path to object: %s' % path_to_object)
remote_object = jfs.getObject(path_to_object)
if hasattr(remote_object, 'size'): #Check if it's a file that is downloaded by checking if the attribute 'size' exist
remote_file = remote_object
total_size = remote_file.size
if total_size == -1: # Indicates an incomplete file
print('%s was NOT downloaded successfully - Incomplete file' % remote_file.name)
exit(1)
with open(remote_file.name, 'wb') as fh:
bytes_read = 0
with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
for chunk_num, chunk in enumerate(remote_file.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if args.checksum:
md5_lf = JFS.calculate_md5(open(remote_file.name, 'rb'))
md5_jf = remote_file.md5
if md5_lf != md5_jf:
print('%s - Checksum for downloaded file' % md5_lf)
print('%s - Checksum for server file' % md5_jf)
print('%s was NOT downloaded successfully - cheksum mismatch' % remote_file.name)
exit(1)
if args.verbose:
print('%s - Checksum for downloaded file' % md5_lf)
print('%s - Checksum for server file' % md5_jf)
print('%s was downloaded successfully - checksum matched' % remote_file.name)
exit(1)
print('%s downloaded successfully - checksum not checked' % remote_file.name)
else: #if it's not a file it has to be a folder
incomplete_files = [] #Create an list where we can store incomplete files
checksum_error_files = [] #Create an list where we can store checksum error files
zero_files = [] #Create an list where we can store zero files
long_path = [] #Create an list where we can store skipped files and folders because of long path
if args.verbose:
print "It's a folder that is downloaded - getting the folder and file structure. This might take a while if the tree is big..."
fileTree = remote_object.filedirlist().tree #Download the folder tree
if args.verbose:
print('Total number of folders to download: %d' % len(fileTree))
#Iterate through each folder
for folder in fileTree:
#We need to strip the path to the folder path from account,device and mountpoint details
rel_path_to_object = folder[len(jfs.username)+len(device)+len(mountpoint)+4:]
if len(rel_path_to_object) > 250: #Windows has a limit of 250 characters in path
print('%s was NOT downloaded successfully - path to long' % rel_path_to_object)
long_path.append(rel_path_to_object)
else:
if args.verbose:
print('Entering a new folder: %s' % rel_path_to_object)
if not os.path.exists(rel_path_to_object): #Create the folder locally if it doesn't exist
os.makedirs(rel_path_to_object)
for _file in fileTree[folder]: #Enter the folder and download the files within
abs_path_to_object = posixpath.join(root_folder.path, posixpath.join(rel_path_to_object, _file[0])) #This is the absolute path to the file that is going to be downloaded
if args.verbose:
print('Downloading the file from: %s' % abs_path_to_object)
if _file[1]==-1: #Corrupt and incomplete files will be skipped
print('%s was NOT downloaded successfully - Incomplete or corrupt file' % _file[0])
incomplete_files.append(posixpath.join(rel_path_to_object,_file[0]))
else:
remote_object = jfs.getObject(abs_path_to_object)
remote_file = remote_object
total_size = remote_file.size
if total_size == 0: # Indicates an zero file
print('%s was NOT downloaded successfully - zero file' % remote_file.name)
zero_files.append(posixpath.join(rel_path_to_object,remote_file.name))
else:
if len(posixpath.join(rel_path_to_object,remote_file.name)) > 250: #Windows has a limit of 250 characters in path
print('%s was NOT downloaded successfully - path to long' % remote_file.name)
long_path.append(posixpath.join(rel_path_to_object,remote_file.name))
else:
if args.verbose:
print('Downloading the file to: %s' % posixpath.join(rel_path_to_object,remote_file.name))
if args.resume: #Check if file exist in path
if os.path.isfile(posixpath.join(rel_path_to_object,remote_file.name)):
print('File exist - skipping downloading: %s' % posixpath.join(rel_path_to_object,remote_file.name))
else:
with open(posixpath.join(rel_path_to_object,remote_file.name), 'wb') as fh:
bytes_read = 0
with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
for chunk_num, chunk in enumerate(remote_file.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
else:
with open(posixpath.join(rel_path_to_object,remote_file.name), 'wb') as fh:
bytes_read = 0
with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
for chunk_num, chunk in enumerate(remote_file.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if args.checksum:
md5_lf = JFS.calculate_md5(open(posixpath.join(rel_path_to_object,remote_file.name), 'rb'))
md5_jf = remote_file.md5
if md5_lf != md5_jf:
print('%s - Checksum for downloaded file' % md5_lf)
print('%s - Checksum for server file' % md5_jf)
print('%s was NOT downloaded successfully - cheksum mismatch' % remote_file.name)
checksum_error_files.append(posixpath.join(rel_path_to_object,remote_file.name))
else:
if args.verbose:
print('%s - Checksum for downloaded file' % md5_lf)
print('%s - Checksum for server file' % md5_jf)
print('%s was downloaded successfully - checksum matched' % remote_file.name)
else:
print('%s downloaded successfully - checksum not checked' % remote_file.name)
#Incomplete files
if len(incomplete_files)> 0:
with codecs.open("incomplete_files.txt", "w", "utf-8") as text_file:
for item in incomplete_files:
text_file.write("%s\n" % item)
print('Incomplete files (not downloaded): %d' % len(incomplete_files))
if args.verbose:
for _files in incomplete_files:
print _files
#Checksum error files
if len(checksum_error_files)> 0:
with codecs.open("checksum_error_files.txt", "w", "utf-8") as text_file:
for item in checksum_error_files:
text_file.write("%s\n" % item)
print('Files with checksum error (not downloaded): %d' % len(checksum_error_files))
if args.verbose:
for _files in checksum_error_files:
print _files
#zero files
if len(zero_files)> 0:
with codecs.open("zero_files.txt", "w", "utf-8") as text_file:
for item in zero_files:
text_file.write("%s\n" % item)
print('Files with zero size (not downloaded): %d' % len(zero_files))
if args.verbose:
for _files in zero_files:
print _files
#long path
if len(long_path)> 0:
with codecs.open("long_path.txt", "w", "utf-8") as text_file:
for item in long_path:
text_file.write("%s\n" % item)
print('Folder and files not downloaded because of path to long: %d' % len(long_path))
if args.verbose:
for _files in long_path:
print _files
# import standardlib
import os, StringIO, logging, datetime
import tempfile, posixpath, urllib
# import dependencies
import lxml
import dateutil
import requests
# import py.test
import pytest # pip install pytest
# import jotta
from jottalib import JFS, __version__
jfs = JFS.JFS() # get username and password from environment or .netrc
TESTFILEDATA = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla est dolor, convallis fermentum sapien in, fringilla congue ligula. Fusce at justo ac felis vulputate laoreet vel at metus. Aenean justo lacus, porttitor dignissim imperdiet a, elementum cursus ligula. Vivamus eu est viverra, pretium arcu eget, imperdiet eros. Curabitur in bibendum."""
class TestJFS:
def test_xml(self):
xml = """<?xml version="1.0" encoding="UTF-8"?>
<user time="2015-09-12-T23:14:23Z" host="dn-093.site-000.jotta.no">
<username>havardgulldahl</username>
<account-type>unlimited</account-type>
<locked>false</locked>
<capacity>-1</capacity>
<max-devices>-1</max-devices>
return "%.3f%s" % (size / math.pow(1024, p), units[int(p)])
if __name__ == '__main__':
# we need an active login to test
import netrc
try:
n = netrc.netrc()
username, account, password = n.authenticators(
'jottacloud.com') # read .netrc entry for 'machine jottacloud'
except Exception as e:
log.exception(e)
username = os.environ['JOTTACLOUD_USERNAME']
password = os.environ['JOTTACLOUD_PASSWORD']
jfs = JFS.JFS(auth=(username, password))
lite = LiteJFS(username, password)
filesize = 1024 * 10 * 10
data = os.urandom(filesize)
testfile = tempfile.NamedTemporaryFile()
puts(colored.blue('Creating test file.'))
for i in progress.bar(range(0, 1000)):
testfile.write(data)
filesize = os.path.getsize(testfile.name)
p = '/Jotta/Archive/test/%s.data' % os.path.basename(testfile.name)
# UPLOAD TEST
puts(
import sys, os, os.path
import urllib, logging, datetime
import argparse # 2.7
# import jottacloud client
from jottalib import JFS
if __name__ == '__main__':
logging.basicConfig(level=logging.WARNING)
parser = argparse.ArgumentParser(
description='Share a file on JottaCloud and get the public URI.',
epilog=
'Note: This utility needs to find JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.'
)
parser.add_argument('localfile',
help='The local file that you want to share',
type=argparse.FileType('r'))
args = parser.parse_args()
#logging.debug(args)
jfs = JFS.JFS(os.environ['JOTTACLOUD_USERNAME'],
password=os.environ['JOTTACLOUD_PASSWORD'])
jottadev = None
for j in jfs.devices: # find Jotta/Shared folder
if j.name == 'Jotta':
jottadev = j
jottashare = jottadev.mountPoints['Shared']
upload = jottashare.up(args.localfile) # upload file
public = upload.share() # share file
for (filename, uuid, publicURI) in public.sharedFiles():
print '%s is now available to the world at %s' % (filename, publicURI)
def test_xml(self):
xml = b"""<?xml version="1.0" encoding="UTF-8"?>
<folder name="test2" time="2015-09-28-T13:49:05Z" host="dn-091.site-000.jotta.no">
<path xml:space="preserve">/havardgulldahl/Jotta/Archive</path>
<abspath xml:space="preserve">/havardgulldahl/Jotta/Archive</abspath>
<folders>
<folder name="Documents"/>
</folders>
<files>
<file name="boink.txt~" uuid="c6684726-a842-4536-95b9-140584515dd1">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>40</size>
<md5>b924ebbc79ad414ded4af442ac7080d3</md5>
<updated>2014-11-23-T21:12:47Z</updated>
</currentRevision>
</file>
<file name="boink.txx~" uuid="1b243d8e-d6df-412c-a2ce-926ebaa73f47">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>40</size>
<md5>0c7652132733ead903dbdb942577fed7</md5>
<updated>2014-11-23-T21:27:21Z</updated>
</currentRevision>
</file>
<file name="boink.txy~" uuid="ba1ec941-8901-4eec-b797-bde9b6b1958c">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>40</size>
<md5>d32f37bf39041d9bb92ad45012e32cb9</md5>
<updated>2014-11-23-T21:05:11Z</updated>
</currentRevision>
</file>
<file name="boink.txz~" uuid="6724af96-e462-4862-b82a-00b7836a0681">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>9</size>
<md5>e3a2cba90ec7630bdf1d0566c8abb93e</md5>
<updated>2014-11-23-T21:03:25Z</updated>
</currentRevision>
</file>
<file name="dingdong" uuid="95c4bbcc-9a59-4669-b4cb-ee49c186ae1b">
<currentRevision>
<number>127</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>20480</size>
<md5>daa100df6e6711906b61c9ab5aa16032</md5>
<updated>2014-11-23-T22:56:30Z</updated>
</currentRevision>
</file>
<file name="fisk.txt" uuid="8b6db048-c44b-4656-8024-737a0c38c7ad">
<currentRevision>
<number>1</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>text/plain</mime>
<mstyle>TEXT_PLAIN</mstyle>
<size>18</size>
<md5>308285a59ae0a4a5ede4f7a0c08d2390</md5>
<updated>2014-11-23-T19:56:51Z</updated>
</currentRevision>
</file>
<file name="nyboink.txt" uuid="3d0a0a28-4c68-4e6a-8414-fa5c37ec45cb">
<currentRevision>
<number>26</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>text/plain</mime>
<mstyle>TEXT_PLAIN</mstyle>
<size>6</size>
<md5>6882bf1ef7f4d938a4cf2931d6953fa1</md5>
<updated>2014-11-23-T21:28:55Z</updated>
</currentRevision>
</file>
<file name="oo.txt" uuid="691aa9b6-eec1-4564-803e-6288dc57aa37">
<currentRevision>
<number>3</number>
<state>COMPLETED</state>
<created>2014-12-18-T21:23:48Z</created>
<modified>2014-12-18-T21:23:48Z</modified>
<mime>text/plain</mime>
<mstyle>TEXT_PLAIN</mstyle>
<size>4</size>
<md5>aa3f5bb8c988fa9b75a1cdb1dc4d93fc</md5>
<updated>2014-12-18-T21:23:48Z</updated>
</currentRevision>
</file>
<file name="pingpong.data" uuid="f02f1ca1-c6a4-403d-b344-f4e3417d92fd">
<currentRevision>
<number>165</number>
<state>COMPLETED</state>
<created>2014-12-18-T21:20:32Z</created>
<modified>2014-12-18-T21:20:32Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>9216</size>
<md5>ec041186ebff92a26ab3ef2dd34dd0e7</md5>
<updated>2014-12-18-T21:20:32Z</updated>
</currentRevision>
</file>
<file name="testfs" uuid="a00034dd-971c-4699-8ee0-e7514045770e">
<currentRevision>
<number>2</number>
<state>COMPLETED</state>
<created>2014-10-05-T10:23:18Z</created>
<modified>2014-10-05-T10:23:18Z</modified>
<mime>application/octet-stream</mime>
<mstyle>APPLICATION_OCTET_STREAM</mstyle>
<size>0</size>
<md5>d41d8cd98f00b204e9800998ecf8427e</md5>
<updated>2014-11-23-T19:41:03Z</updated>
</currentRevision>
</file>
</files>
<metadata first="" max="" total="11" num_folders="1" num_files="10"/>
</folder>
"""
o = lxml.objectify.fromstring(xml)
dev = JFS.JFSFolder(o, jfs, parentpath=jfs.rootpath + '/Jotta/Archive')
dev.synced = True # make sure our tests are based on the xml above, and not live
#test native properties
assert dev.path == jfs.rootpath + '/Jotta/Archive/test2'
assert dev.name == 'test2'
assert dev.deleted == None
assert dev.is_deleted() == False
#test convenience methods
assert len(list(dev.folders())) == 1
assert len(list(dev.files())) == 10
assert all(isinstance(item, JFS.JFSFile) for item in dev.files())
assert all(isinstance(item, JFS.JFSFolder) for item in dev.folders())
WIN32 = (sys.platform == "win32")
TESTFILEDATA = u"""
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla est dolor, convallis fermentum sapien in, fringilla congue ligula. Fusce at justo ac felis vulputate laoreet vel at metus. Aenean justo lacus, porttitor dignissim imperdiet a, elementum cursus ligula. Vivamus eu est viverra, pretium arcu eget, imperdiet eros. Curabitur in bibendum."""
EPOCH = datetime(1970, 1, 1)
def timestamp():
"""
:return: now in unix time, eg. seconds since 1970
"""
return (datetime.utcnow() - EPOCH).total_seconds()
jfs = JFS.JFS()
dev = cli.get_jfs_device(jfs)
root = cli.get_root_dir(jfs)
def test_get_jotta_device():
assert isinstance(dev, JFS.JFSDevice)
assert dev.name == 'Jotta'
def test_get_root_dir():
assert isinstance(root, JFS.JFSMountPoint)
def test_ls():
assert cli.ls([])
def ls(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description='List files in Jotta folder.',
add_help=False)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument(
'-h',
'--humanize', # this matches ls(1)
help='Print human-readable file sizes.',
action='store_true')
parser.add_argument(
'-a',
'--all',
action='store_true',
help='Include deleted and incomplete files (otherwise ignored)')
parser.add_argument(
'item',
nargs='?',
help='The file or directory to list. Defaults to the root dir',
type=commandline_text)
parser.add_argument(
'-H', # because -h means --humanize
'--help',
help='Print this help',
action='help')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
if args.item:
if args.item.startswith('//'):
# break out of root_folder
item_path = posixpath.join(jfs.rootpath, args.item[1:])
else:
item_path = posixpath.join(root_folder.path, args.item)
item = jfs.getObject(item_path)
else:
item = root_folder
timestamp_width = 25
logging.debug('about to ls %r', item)
if isinstance(item, JFS.JFSFolder):
files = [
(f.created,
print_size(f.size, humanize=args.humanize) if f.size else u'',
u'D' if f.deleted else u'I' if f.state == 'INCOMPLETE' else u' ',
f.name) for f in item.files()
if not f.deleted and f.state != 'INCOMPLETE' or args.all
]
folders = [(u' ' * timestamp_width, u'', u'D' if f.deleted else u' ',
unicode(f.name)) for f in item.folders()
if not f.deleted or args.all]
widest_size = 0
for f in files:
if len(f[1]) > widest_size:
widest_size = len(f[1])
for item in sorted(files + folders, key=lambda t: t[3]):
if args.all:
print(u'%s %s %s %s' %
(item[0], item[1].rjust(widest_size), item[2], item[3]))
else:
print(u'%s %s %s' %
(item[0], item[1].rjust(widest_size), item[3]))
else:
print(' '.join([
str(item.created),
print_size(item.size, humanize=args.humanize), item.name
]))
return True # TODO: check return value of command
def download(argv=None):
def download_jfsfile(remote_object, tofolder=None, checksum=False):
'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
if tofolder is None:
tofolder = '.' # with no arguments, store in current dir
total_size = remote_object.size
if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete file' %
remote_file.name))
return False
topath = os.path.join(tofolder, remote_object.name)
with open(topath, 'wb') as fh:
bytes_read = 0
puts(
colored.white('Downloading: %s, size: %s \t' %
(remote_object.name,
print_size(total_size, humanize=True))))
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_object.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if checksum:
md5_lf = JFS.calculate_md5(open(topath, 'rb'))
md5_jf = remote_object.md5
logging.info('%s - Checksum for downloaded file' % md5_lf)
logging.info('%s - Checksum for server file' % md5_jf)
if md5_lf != md5_jf:
puts(colored.blue('%s - Checksum for downloaded file' %
md5_lf))
puts(colored.blue('%s - Checksum for server file' % md5_jf))
puts(
colored.red(
'%s was NOT downloaded successfully - cheksum mismatch'
% remote_object.name))
return False
puts(
colored.green(
'%s was downloaded successfully - checksum matched' %
remote_object.name))
return True
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Download a file or folder from Jottacloud.')
parser.add_argument(
'remoteobject',
help='The path to the file or folder that you want to download',
type=commandline_text)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('-c',
'--checksum',
help='Verify checksum of file after download',
action='store_true')
#parser.add_argument('-r', '--resume',
# help='Will not download the files again if it exist in path',
# action='store_true' )
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
if args.remoteobject.startswith('//'):
# break out of root_folder
root_folder = jfs.rootpath
item_path = posixpath.join(root_folder, args.remoteobject[2:])
else:
root_folder = get_root_dir(jfs).path
item_path = posixpath.join(root_folder, args.remoteobject)
logging.info('Root folder path: %s' % root_folder)
logging.info('Command line path to object: %s' % args.remoteobject)
logging.info('Jotta path to object: %s' % item_path)
remote_object = jfs.getObject(item_path)
if isinstance(remote_object, JFS.JFSFile):
if download_jfsfile(remote_object, checksum=args.checksum):
logging.info('%r downloaded successfully', remote_object.path)
return True
else:
puts(colored.red('%r download failed' % remote_object.path))
return False
else: #if it's not a file it has to be a folder
incomplete_files = [
] #Create an list where we can store incomplete files
checksum_error_files = [
] #Create an list where we can store checksum error files
zero_files = [] #Create an list where we can store zero files
long_path = [
] #Create an list where we can store skipped files and folders because of long path
puts(colored.blue("Getting index for folder: %s" % remote_object.name))
fileTree = remote_object.filedirlist().tree #Download the folder tree
puts(
colored.blue('Total number of folders to download: %d' %
len(fileTree)))
topdir = os.path.dirname(item_path)
logging.info("topdir: %r", topdir)
#Iterate through each folder
for folder in fileTree:
#We need to strip the path to the folder path from account,device and mountpoint details
logging.debug("folder: %r", folder)
_abs_folder_path = posixpath.join(JFS.JFS_ROOT, folder[1:])
logging.debug("absolute folder path : %r", _abs_folder_path)
_rel_folder_path = _abs_folder_path[len(topdir) + 1:]
logging.info('relative folder path: %r', _rel_folder_path)
if len(_rel_folder_path
) > 250: #Windows has a limit of 250 characters in path
puts(
colored.red(
'%s was NOT downloaded successfully - path too long' %
_rel_folder_path))
long_path.append(_rel_folder_path)
else:
logging.info('Entering a new folder: %s' % _rel_folder_path)
if not os.path.exists(
_rel_folder_path
): #Create the folder locally if it doesn't exist
os.makedirs(_rel_folder_path)
for _file in fileTree[
folder]: #Enter the folder and download the files within
logging.info("file: %r", _file)
#This is the absolute path to the file that is going to be downloaded
abs_path_to_object = posixpath.join(
topdir, _rel_folder_path, _file.name)
logging.info('Downloading the file from: %s' %
abs_path_to_object)
if _file.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
#Corrupt and incomplete files will be skipped
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete or corrupt file'
% _file.name))
incomplete_files.append(
posixpath.join(_rel_folder_path, _file.name))
continue
remote_object = jfs.getObject(abs_path_to_object)
remote_file = remote_object
total_size = remote_file.size
if total_size == 0: # Indicates an zero file
puts(
colored.red(
'%s was NOT downloaded successfully - zero file'
% remote_file.name))
zero_files.append(
posixpath.join(_rel_folder_path, remote_file.name))
continue
if len(
posixpath.join(_rel_folder_path, remote_file.name)
) > 250: #Windows has a limit of 250 characters in path
puts(
colored.red(
'%s was NOT downloaded successfully - path too long'
% remote_file.name))
long_path.append(
posixpath.join(_rel_folder_path, remote_file.name))
continue
#TODO: implement args.resume:
if not download_jfsfile(remote_file,
tofolder=_rel_folder_path,
checksum=args.checksum):
# download failed
puts(
colored.red("Download failed: %r" %
remote_file.path))
#Incomplete files
if len(incomplete_files) > 0:
with codecs.open("incomplete_files.txt", "w",
"utf-8") as text_file:
for item in incomplete_files:
text_file.write("%s\n" % item)
print('Incomplete files (not downloaded): %d' % len(incomplete_files))
for _files in incomplete_files:
logging.info("Incomplete: %r", _files)
#Checksum error files
if len(checksum_error_files) > 0:
with codecs.open("checksum_error_files.txt", "w",
"utf-8") as text_file:
for item in checksum_error_files:
text_file.write("%s\n" % item)
print('Files with checksum error (not downloaded): %d' %
len(checksum_error_files))
for _files in checksum_error_files:
logging.info("Checksum error: %r", _files)
#zero files
if len(zero_files) > 0:
with codecs.open("zero_files.txt", "w", "utf-8") as text_file:
for item in zero_files:
text_file.write("%s\n" % item)
print('Files with zero size (not downloaded): %d' % len(zero_files))
for _files in zero_files:
logging.info("Zero sized files: %r", _files)
#long path
if len(long_path) > 0:
with codecs.open("long_path.txt", "w", "utf-8") as text_file:
for item in long_path:
text_file.write("%s\n" % item)
print('Folder and files not downloaded because of path too long: %d' %
len(long_path))
for _files in long_path:
logging.info("Path too long: %r", _files)
return True
def scanner(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description=__doc__,
epilog=
"""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project."""
)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('--errorfile',
type=commandline_text,
help='A file to write errors to',
default='./jottacloudclient.log')
parser.add_argument(
'--exclude',
type=re.compile,
action='append',
help='Exclude paths matched by this pattern (can be repeated)')
parser.add_argument('--prune-files',
dest='prune_files',
help='Delete files that does not exist locally',
action='store_true')
parser.add_argument('--prune-folders',
dest='prune_folders',
help='Delete folders that does not exist locally',
action='store_true')
parser.add_argument('--prune-all',
dest='prune_all',
help='Combines --prune-files and --prune-folders',
action='store_true')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument(
'--dry-run',
action='store_true',
help=
"don't actually do any uploads or deletes, just show what would be done"
)
parser.add_argument('topdir',
type=is_dir,
help='Path to local dir that needs syncing')
parser.add_argument(
'jottapath',
type=commandline_text,
help=
'The path at JottaCloud where the tree shall be synced (must exist)')
args = parse_args_and_apply_logging_level(parser, argv)
if args.prune_all:
args.prune_files = True
args.prune_folders = True
fh = logging.FileHandler(args.errorfile)
fh.setLevel(logging.ERROR)
logging.getLogger('').addHandler(fh)
jfs = JFS.JFS()
logging.info('args: topdir %r, jottapath %r', args.topdir, args.jottapath)
filescanner(args.topdir, args.jottapath, jfs, args.errorfile, args.exclude,
args.dry_run, args.prune_files, args.prune_folders)