def __init__(self, username, password, path='.'):
self.client = JFS.JFS(username, password)
self.root = path
self.dirty = False # True if some method has changed/added something and we need to get fresh data from JottaCloud
# TODO: make self.dirty more smart, to know what path, to get from cache and not
self.__newfiles = []
self.__newfolders = []
def monitor():
if not HAS_WATCHDOG:
message = ['jotta-monitor requires watchdog (pip install watchdog), install that and try again.']
print(' '.join(message))
sys.exit(1)
# Has watchdog, can safely import filemonitor
from .monitor import filemonitor
def is_dir(path):
if not os.path.isdir(path):
raise argparse.ArgumentTypeError('%s is not a valid directory' % path)
return path
parser = argparse.ArgumentParser(description=__doc__,
epilog="""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project.""")
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
parser.add_argument('--errorfile', help='A file to write errors to', default='./jottacloudclient.log')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument('--dry-run', action='store_true',
help="don't actually do any uploads or deletes, just show what would be done")
parser.add_argument('topdir', type=is_dir, help='Path to local dir that needs syncing')
parser.add_argument('mode', help='Mode of operation: ARCHIVE, SYNC or SHARE. See README.md',
choices=( 'archive', 'sync', 'share') )
args = parse_args_and_apply_logging_level(parser)
fh = logging.FileHandler(args.errorfile)
fh.setLevel(logging.ERROR)
logging.getLogger('').addHandler(fh)
jfs = JFS()
filemonitor(args.topdir, args.mode, jfs)
def download(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Download a file from Jottacloud.')
parser.add_argument('remotefile',
help='The path to the file that you want to download')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
path_to_object = posixpath.join(root_folder.path, args.remotefile)
remote_file = jfs.getObject(path_to_object)
total_size = remote_file.size
with open(remote_file.name, 'wb') as fh:
bytes_read = 0
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_file.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
print('%s downloaded successfully' % args.remotefile)
def rm(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Delete an item from Jottacloud')
parser.add_argument('file',
type=commandline_text,
help='The path to the item that you want to delete')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('-f',
'--force',
help='Completely deleted, no restore possiblity',
action='store_true')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_dir = get_root_dir(jfs)
item_path = posixpath.join(root_dir.path, args.file)
item = jfs.getObject(item_path)
if args.force:
item.hard_delete()
else:
item.delete()
print('%s deleted' % args.file)
return True # TODO: check return value of command
def scanner():
def is_dir(path):
if not os.path.isdir(path):
raise argparse.ArgumentTypeError('%s is not a valid directory' % path)
return path
parser = argparse.ArgumentParser(description=__doc__,
epilog="""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project.""")
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
parser.add_argument('--errorfile', help='A file to write errors to', default='./jottacloudclient.log')
parser.add_argument('--exclude', type=re.compile, action='append', help='Exclude paths matched by this pattern (can be repeated)')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument('--dry-run', action='store_true',
help="don't actually do any uploads or deletes, just show what would be done")
parser.add_argument('topdir', type=is_dir, help='Path to local dir that needs syncing')
parser.add_argument('jottapath', help='The path at JottaCloud where the tree shall be synced (must exist)')
args = parse_args_and_apply_logging_level(parser)
fh = logging.FileHandler(args.errorfile)
fh.setLevel(logging.ERROR)
logging.getLogger('').addHandler(fh)
jfs = JFS()
filescanner(args.topdir, args.jottapath, jfs, args.exclude, args.dry_run)
def cat(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Display contents of a file from Jottacloud')
parser.add_argument('file',
type=commandline_text,
help='The path to the file that you want to show')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
if args.file.startswith('//'):
# break out of root_folder
item_path = posixpath.join(jfs.rootpath, args.file[1:])
else:
root_dir = get_root_dir(jfs)
item_path = posixpath.join(root_dir.path, args.file)
item = jfs.getObject(item_path)
if not isinstance(item, JFS.JFSFile):
print("%r is not a file (it's a %s), so we can't show it" %
(args.file, type(item)))
sys.exit(1)
s = ''
for chunk in item.stream():
print(chunk.encode(sys.getdefaultencoding()))
s = s + chunk
return s
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Import JottaCloud libraries.
try:
from jottalib import JFS
from jottalib.JFS import JFSNotFoundError, JFSIncompleteFile
except ImportError:
raise BackendException(
u'JottaCloud backend requires jottalib'
u' (see https://pypi.python.org/pypi/jottalib).')
# Set jottalib loggers to the same verbosity as duplicity
duplicity_log_level = get_duplicity_log_level()
set_jottalib_logging_level(duplicity_log_level)
# Ensure jottalib and duplicity log to the same handlers
set_jottalib_log_handlers(log._logger.handlers)
# Will fetch jottacloud auth from environment or .netrc
self.client = JFS.JFS()
self.folder = self.get_or_create_directory(
parsed_url.path.lstrip(u'/'))
log.Debug(u"Jottacloud folder for duplicity: %r" % self.folder.path)
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Import JottaCloud libraries.
try:
from jottalib import JFS
except ImportError:
raise
raise BackendException(
'JottaCloud backend requires jottalib'
' (see https://pypi.python.org/pypi/jottalib).')
# Setup client instance.
_pass = os.environ.get('JOTTACLOUD_PASSWORD', None)
if _pass is None:
_pass = self.get_password()
self.client = JFS.JFS(parsed_url.username, _pass)
#self.client.http_client.debug = False
# Fetch destination folder entry (and create hierarchy if required).
try:
self.folder = self.client.getObject('%s/backup' % parsed_url.path)
except JFS.JFSNotFoundError:
parentfolder = self.client.getObject(parsed_url.path)
try:
self.folder = parentfolder.mkdir('backup')
except:
raise
raise BackendException(
"Error while creating destination folder 'Backup')")
except:
raise
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
# Import JottaCloud libraries.
try:
from jottalib import JFS
except ImportError:
raise
raise BackendException('JottaCloud backend requires jottalib'
' (see https://pypi.python.org/pypi/jottalib).')
# Setup client instance.
_pass = os.environ.get('JOTTACLOUD_PASSWORD', None)
if _pass is None:
_pass = self.get_password()
username = parsed_url.username or os.environ.get('JOTTACLOUD_USERNAME')
self.client = JFS.JFS(auth=(username, _pass))
#self.client.http_client.debug = False
root_dir = get_root_dir(self.client)
# Fetch destination folder entry (and create hierarchy if required).
path = posixpath.join([root_dir.path, parsed_url.path.lstrip('/')])
try:
# self.folder = root_dir#self.client.getObject('%s/duplicity' % parsed_url.path.lstrip('//'))
self.folder = self.client.getObject(path)
except JFS.JFSNotFoundError:
try:
self.folder = root_dir.mkdir(parsed_url.path.lstrip('/'))
except:
raise
raise BackendException("Error while creating destination folder 'Backup')")
except:
raise
def upload(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Upload a file to JottaCloud.')
parser.add_argument('localfile',
help='The local file that you want to upload',
type=argparse.FileType('r'))
parser.add_argument('remote_dir',
help='The remote directory to upload the file to',
nargs='?',
type=commandline_text)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
jfs = JFS.JFS()
args = parse_args_and_apply_logging_level(parser, argv)
decoded_filename = commandline_text(args.localfile.name)
progress_bar = ProgressBar()
callback = lambda monitor, size: progress_bar.show(monitor.bytes_read, size
)
root_folder = get_root_dir(jfs)
if args.remote_dir:
target_dir_path = posixpath.join(root_folder.path, args.remote_dir)
target_dir = jfs.getObject(target_dir_path)
else:
target_dir = root_folder
upload = target_dir.up(args.localfile,
os.path.basename(decoded_filename),
upload_callback=callback)
print('%s uploaded successfully' % decoded_filename)
return True # TODO: check return value
def share(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Share a file on JottaCloud and get the public URI.',
epilog='Note: This utility needs to find JOTTACLOUD_USERNAME'
' and JOTTACLOUD_PASSWORD in the running environment.')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('localfile',
help='The local file that you want to share',
type=argparse.FileType('r'))
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
jottadev = get_jfs_device(jfs)
jottashare = jottadev.mountPoints['Shared']
upload = jottashare.up(args.localfile) # upload file
public = upload.share() # share file
logging.debug('Shared %r and got: %r (%s)', args.localfile, public,
dir(public))
for (filename, uuid, publicURI) in public.sharedFiles():
print('%s is now available to the world at %s' % (filename, publicURI))
return True # TODO: check return value of command
def mkdir():
parser = argparse.ArgumentParser(description='Create a new folder in Jottacloud.')
parser.add_argument('newdir', help='The path to the folder that you want to create')
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
args = parse_args_and_apply_logging_level(parser)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
root_folder.mkdir(args.newdir)
def test_errors(self):
with pytest.raises(JFS.JFSCredentialsError): # HTTP 401
JFS.JFS(auth=('PYTEST', 'PYTEST'))
with pytest.raises(JFS.JFSNotFoundError): # HTTP 404
jfs.get('/Jotta/Archive/FileNot.found')
with pytest.raises(JFS.JFSRangeError): # HTTP 416
p = "/Jotta/Archive/testfile_up_and_readpartial.txt"
t = jfs.up(p, StringIO.StringIO(TESTFILEDATA))
f = jfs.getObject(p)
f.readpartial(10, 3) # Range start index larger than end index;
f.delete()
def ls(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description='List files in Jotta folder.',
add_help=False)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('-h',
'--humanize',
help='Print human-readable file sizes.',
action='store_true')
parser.add_argument('-a',
'--all',
help='Show all files, even deleted ones',
action='store_true')
parser.add_argument('item',
nargs='?',
help='The file or directory to list. Defaults to the '
'root dir')
parser.add_argument('-H', '--help', help='Print this help', action='help')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
if args.item:
item_path = posixpath.join(root_folder.path, args.item)
item = jfs.getObject(item_path)
else:
item = root_folder
if isinstance(item, JFS.JFSFolder):
files = [(f.created, print_size(f.size, humanize=args.humanize),
u'D' if f.deleted else u' ', f.name) for f in item.files()
if not f.deleted or args.all]
folders = [(u' ' * 25, u'', u'D' if f.deleted else u' ', f.name)
for f in item.folders()]
widest_size = 0
for f in files:
if len(f[1]) > widest_size:
widest_size = len(f[1])
for item in sorted(files + folders, key=lambda t: t[3]):
if args.all:
print(u'%s %s %s %s' %
(item[0], item[1].rjust(widest_size), item[2], item[3]))
else:
print(u'%s %s %s' %
(item[0], item[1].rjust(widest_size), item[3]))
else:
print(' '.join([
str(item.created),
print_size(item.size, humanize=args.humanize), item.name
]))
def restore():
parser = argparse.ArgumentParser(description='Restore a deleted item from Jottacloud')
parser.add_argument('file', help='The path to the item that you want to restore')
parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'), default='warning')
args = parse_args_and_apply_logging_level(parser)
jfs = JFS.JFS()
root_dir = get_root_dir(jfs)
item_path = posixpath.join(root_dir.path, args.file)
item = jfs.getObject(item_path)
item.restore()
print '%s restored' % args.file
def share():
parser = argparse.ArgumentParser(description='Share a file on JottaCloud and get the public URI.',
epilog='Note: This utility needs to find JOTTACLOUD_USERNAME'
' and JOTTACLOUD_PASSWORD in the running environment.')
parser.add_argument('localfile', help='The local file that you want to share',
type=argparse.FileType('r'))
args = parser.parse_args()
jfs = JFS.JFS()
jottadev = get_jotta_device(jfs)
jottashare = jottadev.mountPoints['Shared']
upload = jottashare.up(args.localfile) # upload file
public = upload.share() # share file
for (filename, uuid, publicURI) in public.sharedFiles():
print '%s is now available to the world at %s' % (filename, publicURI)
def mkdir(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Create a new folder in Jottacloud.')
parser.add_argument('newdir',
type=commandline_text,
help='The path to the folder that you want to create')
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
root_folder.mkdir(args.newdir)
return True # TODO: check return value of mkdir
# import standardlib
import os, StringIO, logging, datetime
import tempfile, posixpath, urllib
# import dependencies
import lxml
import dateutil
import requests
# import py.test
import pytest # pip install pytest
# import jotta
from jottalib import JFS, __version__
jfs = JFS.JFS() # get username and password from environment or .netrc
TESTFILEDATA = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla est dolor, convallis fermentum sapien in, fringilla congue ligula. Fusce at justo ac felis vulputate laoreet vel at metus. Aenean justo lacus, porttitor dignissim imperdiet a, elementum cursus ligula. Vivamus eu est viverra, pretium arcu eget, imperdiet eros. Curabitur in bibendum."""
class TestJFS:
def test_xml(self):
xml = """<?xml version="1.0" encoding="UTF-8"?>
<user time="2015-09-12-T23:14:23Z" host="dn-093.site-000.jotta.no">
<username>havardgulldahl</username>
<account-type>unlimited</account-type>
<locked>false</locked>
<capacity>-1</capacity>
<max-devices>-1</max-devices>
return "%.3f%s" % (size / math.pow(1024, p), units[int(p)])
if __name__ == '__main__':
# we need an active login to test
import netrc
try:
n = netrc.netrc()
username, account, password = n.authenticators(
'jottacloud.com') # read .netrc entry for 'machine jottacloud'
except Exception as e:
log.exception(e)
username = os.environ['JOTTACLOUD_USERNAME']
password = os.environ['JOTTACLOUD_PASSWORD']
jfs = JFS.JFS(auth=(username, password))
lite = LiteJFS(username, password)
filesize = 1024 * 10 * 10
data = os.urandom(filesize)
testfile = tempfile.NamedTemporaryFile()
puts(colored.blue('Creating test file.'))
for i in progress.bar(range(0, 1000)):
testfile.write(data)
filesize = os.path.getsize(testfile.name)
p = '/Jotta/Archive/test/%s.data' % os.path.basename(testfile.name)
# UPLOAD TEST
puts(
import sys, os, os.path
import urllib, logging, datetime
import argparse # 2.7
# import jottacloud client
from jottalib import JFS
if __name__ == '__main__':
logging.basicConfig(level=logging.WARNING)
parser = argparse.ArgumentParser(
description='Share a file on JottaCloud and get the public URI.',
epilog=
'Note: This utility needs to find JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.'
)
parser.add_argument('localfile',
help='The local file that you want to share',
type=argparse.FileType('r'))
args = parser.parse_args()
#logging.debug(args)
jfs = JFS.JFS(os.environ['JOTTACLOUD_USERNAME'],
password=os.environ['JOTTACLOUD_PASSWORD'])
jottadev = None
for j in jfs.devices: # find Jotta/Shared folder
if j.name == 'Jotta':
jottadev = j
jottashare = jottadev.mountPoints['Shared']
upload = jottashare.up(args.localfile) # upload file
public = upload.share() # share file
for (filename, uuid, publicURI) in public.sharedFiles():
print '%s is now available to the world at %s' % (filename, publicURI)
def test_get_jotta_device():
jfs = JFS.JFS()
dev = cli.get_jotta_device(jfs)
assert isinstance(dev, JFS.JFSDevice)
assert dev.name == 'Jotta'
WIN32 = (sys.platform == "win32")
TESTFILEDATA = u"""
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla est dolor, convallis fermentum sapien in, fringilla congue ligula. Fusce at justo ac felis vulputate laoreet vel at metus. Aenean justo lacus, porttitor dignissim imperdiet a, elementum cursus ligula. Vivamus eu est viverra, pretium arcu eget, imperdiet eros. Curabitur in bibendum."""
EPOCH = datetime(1970, 1, 1)
def timestamp():
"""
:return: now in unix time, eg. seconds since 1970
"""
return (datetime.utcnow() - EPOCH).total_seconds()
jfs = JFS.JFS()
dev = cli.get_jfs_device(jfs)
root = cli.get_root_dir(jfs)
def test_get_jotta_device():
assert isinstance(dev, JFS.JFSDevice)
assert dev.name == 'Jotta'
def test_get_root_dir():
assert isinstance(root, JFS.JFSMountPoint)
def test_ls():
assert cli.ls([])
def test_get_root_dir():
jfs = JFS.JFS()
root = cli.get_root_dir(jfs)
assert isinstance(root, JFS.JFSMountPoint)
def scanner(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description=__doc__,
epilog=
"""The program expects to find an entry for "jottacloud.com" in your .netrc,
or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
This is not an official JottaCloud project."""
)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('--errorfile',
type=commandline_text,
help='A file to write errors to',
default='./jottacloudclient.log')
parser.add_argument(
'--exclude',
type=re.compile,
action='append',
help='Exclude paths matched by this pattern (can be repeated)')
parser.add_argument('--prune-files',
dest='prune_files',
help='Delete files that does not exist locally',
action='store_true')
parser.add_argument('--prune-folders',
dest='prune_folders',
help='Delete folders that does not exist locally',
action='store_true')
parser.add_argument('--prune-all',
dest='prune_all',
help='Combines --prune-files and --prune-folders',
action='store_true')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument(
'--dry-run',
action='store_true',
help=
"don't actually do any uploads or deletes, just show what would be done"
)
parser.add_argument('topdir',
type=is_dir,
help='Path to local dir that needs syncing')
parser.add_argument(
'jottapath',
type=commandline_text,
help=
'The path at JottaCloud where the tree shall be synced (must exist)')
args = parse_args_and_apply_logging_level(parser, argv)
if args.prune_all:
args.prune_files = True
args.prune_folders = True
fh = logging.FileHandler(args.errorfile)
fh.setLevel(logging.ERROR)
logging.getLogger('').addHandler(fh)
jfs = JFS.JFS()
logging.info('args: topdir %r, jottapath %r', args.topdir, args.jottapath)
filescanner(args.topdir, args.jottapath, jfs, args.errorfile, args.exclude,
args.dry_run, args.prune_files, args.prune_folders)
def download(argv=None):
def download_jfsfile(remote_object, tofolder=None, checksum=False):
'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
if tofolder is None:
tofolder = '.' # with no arguments, store in current dir
total_size = remote_object.size
if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete file' %
remote_file.name))
return False
topath = os.path.join(tofolder, remote_object.name)
with open(topath, 'wb') as fh:
bytes_read = 0
puts(
colored.white('Downloading: %s, size: %s \t' %
(remote_object.name,
print_size(total_size, humanize=True))))
with ProgressBar(expected_size=total_size) as bar:
for chunk_num, chunk in enumerate(remote_object.stream()):
fh.write(chunk)
bytes_read += len(chunk)
bar.show(bytes_read)
if checksum:
md5_lf = JFS.calculate_md5(open(topath, 'rb'))
md5_jf = remote_object.md5
logging.info('%s - Checksum for downloaded file' % md5_lf)
logging.info('%s - Checksum for server file' % md5_jf)
if md5_lf != md5_jf:
puts(colored.blue('%s - Checksum for downloaded file' %
md5_lf))
puts(colored.blue('%s - Checksum for server file' % md5_jf))
puts(
colored.red(
'%s was NOT downloaded successfully - cheksum mismatch'
% remote_object.name))
return False
puts(
colored.green(
'%s was downloaded successfully - checksum matched' %
remote_object.name))
return True
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(
description='Download a file or folder from Jottacloud.')
parser.add_argument(
'remoteobject',
help='The path to the file or folder that you want to download',
type=commandline_text)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument('-c',
'--checksum',
help='Verify checksum of file after download',
action='store_true')
#parser.add_argument('-r', '--resume',
# help='Will not download the files again if it exist in path',
# action='store_true' )
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
if args.remoteobject.startswith('//'):
# break out of root_folder
root_folder = jfs.rootpath
item_path = posixpath.join(root_folder, args.remoteobject[2:])
else:
root_folder = get_root_dir(jfs).path
item_path = posixpath.join(root_folder, args.remoteobject)
logging.info('Root folder path: %s' % root_folder)
logging.info('Command line path to object: %s' % args.remoteobject)
logging.info('Jotta path to object: %s' % item_path)
remote_object = jfs.getObject(item_path)
if isinstance(remote_object, JFS.JFSFile):
if download_jfsfile(remote_object, checksum=args.checksum):
logging.info('%r downloaded successfully', remote_object.path)
return True
else:
puts(colored.red('%r download failed' % remote_object.path))
return False
else: #if it's not a file it has to be a folder
incomplete_files = [
] #Create an list where we can store incomplete files
checksum_error_files = [
] #Create an list where we can store checksum error files
zero_files = [] #Create an list where we can store zero files
long_path = [
] #Create an list where we can store skipped files and folders because of long path
puts(colored.blue("Getting index for folder: %s" % remote_object.name))
fileTree = remote_object.filedirlist().tree #Download the folder tree
puts(
colored.blue('Total number of folders to download: %d' %
len(fileTree)))
topdir = os.path.dirname(item_path)
logging.info("topdir: %r", topdir)
#Iterate through each folder
for folder in fileTree:
#We need to strip the path to the folder path from account,device and mountpoint details
logging.debug("folder: %r", folder)
_abs_folder_path = posixpath.join(JFS.JFS_ROOT, folder[1:])
logging.debug("absolute folder path : %r", _abs_folder_path)
_rel_folder_path = _abs_folder_path[len(topdir) + 1:]
logging.info('relative folder path: %r', _rel_folder_path)
if len(_rel_folder_path
) > 250: #Windows has a limit of 250 characters in path
puts(
colored.red(
'%s was NOT downloaded successfully - path too long' %
_rel_folder_path))
long_path.append(_rel_folder_path)
else:
logging.info('Entering a new folder: %s' % _rel_folder_path)
if not os.path.exists(
_rel_folder_path
): #Create the folder locally if it doesn't exist
os.makedirs(_rel_folder_path)
for _file in fileTree[
folder]: #Enter the folder and download the files within
logging.info("file: %r", _file)
#This is the absolute path to the file that is going to be downloaded
abs_path_to_object = posixpath.join(
topdir, _rel_folder_path, _file.name)
logging.info('Downloading the file from: %s' %
abs_path_to_object)
if _file.state in (JFS.ProtoFile.STATE_CORRUPT,
JFS.ProtoFile.STATE_INCOMPLETE):
#Corrupt and incomplete files will be skipped
puts(
colored.red(
'%s was NOT downloaded successfully - Incomplete or corrupt file'
% _file.name))
incomplete_files.append(
posixpath.join(_rel_folder_path, _file.name))
continue
remote_object = jfs.getObject(abs_path_to_object)
remote_file = remote_object
total_size = remote_file.size
if total_size == 0: # Indicates an zero file
puts(
colored.red(
'%s was NOT downloaded successfully - zero file'
% remote_file.name))
zero_files.append(
posixpath.join(_rel_folder_path, remote_file.name))
continue
if len(
posixpath.join(_rel_folder_path, remote_file.name)
) > 250: #Windows has a limit of 250 characters in path
puts(
colored.red(
'%s was NOT downloaded successfully - path too long'
% remote_file.name))
long_path.append(
posixpath.join(_rel_folder_path, remote_file.name))
continue
#TODO: implement args.resume:
if not download_jfsfile(remote_file,
tofolder=_rel_folder_path,
checksum=args.checksum):
# download failed
puts(
colored.red("Download failed: %r" %
remote_file.path))
#Incomplete files
if len(incomplete_files) > 0:
with codecs.open("incomplete_files.txt", "w",
"utf-8") as text_file:
for item in incomplete_files:
text_file.write("%s\n" % item)
print('Incomplete files (not downloaded): %d' % len(incomplete_files))
for _files in incomplete_files:
logging.info("Incomplete: %r", _files)
#Checksum error files
if len(checksum_error_files) > 0:
with codecs.open("checksum_error_files.txt", "w",
"utf-8") as text_file:
for item in checksum_error_files:
text_file.write("%s\n" % item)
print('Files with checksum error (not downloaded): %d' %
len(checksum_error_files))
for _files in checksum_error_files:
logging.info("Checksum error: %r", _files)
#zero files
if len(zero_files) > 0:
with codecs.open("zero_files.txt", "w", "utf-8") as text_file:
for item in zero_files:
text_file.write("%s\n" % item)
print('Files with zero size (not downloaded): %d' % len(zero_files))
for _files in zero_files:
logging.info("Zero sized files: %r", _files)
#long path
if len(long_path) > 0:
with codecs.open("long_path.txt", "w", "utf-8") as text_file:
for item in long_path:
text_file.write("%s\n" % item)
print('Folder and files not downloaded because of path too long: %d' %
len(long_path))
for _files in long_path:
logging.info("Path too long: %r", _files)
return True
def ls(argv=None):
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser(description='List files in Jotta folder.',
add_help=False)
parser.add_argument('-l',
'--loglevel',
help='Logging level. Default: %(default)s.',
choices=('debug', 'info', 'warning', 'error'),
default='warning')
parser.add_argument(
'-h',
'--humanize', # this matches ls(1)
help='Print human-readable file sizes.',
action='store_true')
parser.add_argument(
'-a',
'--all',
action='store_true',
help='Include deleted and incomplete files (otherwise ignored)')
parser.add_argument(
'item',
nargs='?',
help='The file or directory to list. Defaults to the root dir',
type=commandline_text)
parser.add_argument(
'-H', # because -h means --humanize
'--help',
help='Print this help',
action='help')
args = parse_args_and_apply_logging_level(parser, argv)
jfs = JFS.JFS()
root_folder = get_root_dir(jfs)
if args.item:
if args.item.startswith('//'):
# break out of root_folder
item_path = posixpath.join(jfs.rootpath, args.item[1:])
else:
item_path = posixpath.join(root_folder.path, args.item)
item = jfs.getObject(item_path)
else:
item = root_folder
timestamp_width = 25
logging.debug('about to ls %r', item)
if isinstance(item, JFS.JFSFolder):
files = [
(f.created,
print_size(f.size, humanize=args.humanize) if f.size else u'',
u'D' if f.deleted else u'I' if f.state == 'INCOMPLETE' else u' ',
f.name) for f in item.files()
if not f.deleted and f.state != 'INCOMPLETE' or args.all
]
folders = [(u' ' * timestamp_width, u'', u'D' if f.deleted else u' ',
unicode(f.name)) for f in item.folders()
if not f.deleted or args.all]
widest_size = 0
for f in files:
if len(f[1]) > widest_size:
widest_size = len(f[1])
for item in sorted(files + folders, key=lambda t: t[3]):
if args.all:
print(u'%s %s %s %s' %
(item[0], item[1].rjust(widest_size), item[2], item[3]))
else:
print(u'%s %s %s' %
(item[0], item[1].rjust(widest_size), item[3]))
else:
print(' '.join([
str(item.created),
print_size(item.size, humanize=args.humanize), item.name
]))
return True # TODO: check return value of command
def __init__(self, auth, path='.'):
self.client = JFS.JFS(auth)
self.__newfiles = {} # a dict of stringio objects
self.__newfolders = []
self.ino = 0
