示例#1
0
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        # Import JottaCloud libraries.
        try:
            from jottalib import JFS
        except ImportError:
            raise
            raise BackendException('JottaCloud backend requires jottalib'
                                   ' (see https://pypi.python.org/pypi/jottalib).')

        # Setup client instance.
        _pass = os.environ.get('JOTTACLOUD_PASSWORD', None)
        if _pass is None:
            _pass = self.get_password()
        username = parsed_url.username or os.environ.get('JOTTACLOUD_USERNAME')
        self.client = JFS.JFS(auth=(username, _pass))
        #self.client.http_client.debug = False
        root_dir = get_root_dir(self.client)

        # Fetch destination folder entry (and create hierarchy if required).
        path = posixpath.join([root_dir.path, parsed_url.path.lstrip('/')])
        try:
        #    self.folder = root_dir#self.client.getObject('%s/duplicity' % parsed_url.path.lstrip('//'))
            self.folder = self.client.getObject(path)
        except JFS.JFSNotFoundError:
            try:
                self.folder = root_dir.mkdir(parsed_url.path.lstrip('/'))
            except:
                raise
                raise BackendException("Error while creating destination folder 'Backup')")
        except:
            raise
示例#2
0
文件: cli.py 项目: forsberg/jottalib
def monitor():
    if not HAS_WATCHDOG:
        message = ['jotta-monitor requires watchdog (pip install watchdog), install that and try again.']
        print(' '.join(message))
        sys.exit(1)

    # Has watchdog, can safely import filemonitor
    from .monitor import filemonitor

    def is_dir(path):
        if not os.path.isdir(path):
            raise argparse.ArgumentTypeError('%s is not a valid directory' % path)
        return path
    parser = argparse.ArgumentParser(description=__doc__,
                                    epilog="""The program expects to find an entry for "jottacloud.com" in your .netrc,
                                    or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
                                    This is not an official JottaCloud project.""")
    parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'), default='warning')
    parser.add_argument('--errorfile', help='A file to write errors to', default='./jottacloudclient.log')
    parser.add_argument('--version', action='version', version=__version__)
    parser.add_argument('--dry-run', action='store_true',
                        help="don't actually do any uploads or deletes, just show what would be done")
    parser.add_argument('topdir', type=is_dir, help='Path to local dir that needs syncing')
    parser.add_argument('mode', help='Mode of operation: ARCHIVE, SYNC or SHARE. See README.md',
                        choices=( 'archive', 'sync', 'share') )
    args = parse_args_and_apply_logging_level(parser)
    fh = logging.FileHandler(args.errorfile)
    fh.setLevel(logging.ERROR)
    logging.getLogger('').addHandler(fh)

    jfs = JFS()

    filemonitor(args.topdir, args.mode, jfs)
示例#3
0
def download(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(
        description='Download a file from Jottacloud.')
    parser.add_argument('remotefile',
                        help='The path to the file that you want to download')
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()
    root_folder = get_root_dir(jfs)
    path_to_object = posixpath.join(root_folder.path, args.remotefile)
    remote_file = jfs.getObject(path_to_object)
    total_size = remote_file.size
    with open(remote_file.name, 'wb') as fh:
        bytes_read = 0
        with ProgressBar(expected_size=total_size) as bar:
            for chunk_num, chunk in enumerate(remote_file.stream()):
                fh.write(chunk)
                bytes_read += len(chunk)
                bar.show(bytes_read)
    print('%s downloaded successfully' % args.remotefile)
示例#4
0
 def download_jfsfile(remote_object, tofolder=None, checksum=False):
     'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
     if tofolder is None:
         tofolder = '.' # with no arguments, store in current dir
     total_size = remote_object.size
     if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT, JFS.ProtoFile.STATE_INCOMPLETE):
         puts(colored.red('%s was NOT downloaded successfully - Incomplete file' % remote_file.name))
         return False
     topath = os.path.join(tofolder, remote_object.name)
     with open(topath, 'wb') as fh:
         bytes_read = 0
         puts(colored.white('Downloading: %s, size: %s \t' % (remote_object.name, 
                                                              print_size(total_size, humanize=True))))   
         with ProgressBar(expected_size=total_size) as bar:
             for chunk_num, chunk in enumerate(remote_object.stream()):
                 fh.write(chunk)
                 bytes_read += len(chunk)
                 bar.show(bytes_read)
     if checksum:
         md5_lf = JFS.calculate_md5(open(topath, 'rb'))
         md5_jf = remote_object.md5
         logging.info('%s - Checksum for downloaded file' % md5_lf)
         logging.info('%s - Checksum for server file' % md5_jf)
         if md5_lf != md5_jf:
             puts(colored.blue('%s - Checksum for downloaded file' % md5_lf))
             puts(colored.blue('%s - Checksum for server file' % md5_jf))
             puts(colored.red('%s was NOT downloaded successfully - cheksum mismatch' % remote_object.name))
             return False
         puts(colored.green('%s was downloaded successfully - checksum  matched' % remote_object.name))
     return True
示例#5
0
def cat(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(
        description='Display contents of a file from Jottacloud')
    parser.add_argument('file',
                        type=commandline_text,
                        help='The path to the file that you want to show')
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()
    if args.file.startswith('//'):
        # break out of root_folder
        item_path = posixpath.join(jfs.rootpath, args.file[1:])
    else:
        root_dir = get_root_dir(jfs)
        item_path = posixpath.join(root_dir.path, args.file)
    item = jfs.getObject(item_path)
    if not isinstance(item, JFS.JFSFile):
        print("%r is not a file (it's a %s), so we can't show it" %
              (args.file, type(item)))
        sys.exit(1)
    s = ''
    for chunk in item.stream():
        print(chunk.encode(sys.getdefaultencoding()))
        s = s + chunk
    return s
示例#6
0
def share(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(
        description='Share a file on JottaCloud and get the public URI.',
        epilog='Note: This utility needs to find JOTTACLOUD_USERNAME'
        ' and JOTTACLOUD_PASSWORD in the running environment.')
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    parser.add_argument('localfile',
                        help='The local file that you want to share',
                        type=argparse.FileType('r'))
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()
    jottadev = get_jfs_device(jfs)
    jottashare = jottadev.mountPoints['Shared']
    upload = jottashare.up(args.localfile)  # upload file
    public = upload.share()  # share file
    logging.debug('Shared %r and got: %r (%s)', args.localfile, public,
                  dir(public))
    for (filename, uuid, publicURI) in public.sharedFiles():
        print('%s is now available to the world at %s' % (filename, publicURI))
    return True  # TODO: check return value of command
示例#7
0
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        # Import JottaCloud libraries.
        try:
            from jottalib import JFS
            from jottalib.JFS import JFSNotFoundError, JFSIncompleteFile
        except ImportError:
            raise BackendException(
                u'JottaCloud backend requires jottalib'
                u' (see https://pypi.python.org/pypi/jottalib).')

        # Set jottalib loggers to the same verbosity as duplicity
        duplicity_log_level = get_duplicity_log_level()
        set_jottalib_logging_level(duplicity_log_level)

        # Ensure jottalib and duplicity log to the same handlers
        set_jottalib_log_handlers(log._logger.handlers)

        # Will fetch jottacloud auth from environment or .netrc
        self.client = JFS.JFS()

        self.folder = self.get_or_create_directory(
            parsed_url.path.lstrip(u'/'))
        log.Debug(u"Jottacloud folder for duplicity: %r" % self.folder.path)
示例#8
0
def rm(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(
        description='Delete an item from Jottacloud')
    parser.add_argument('file',
                        type=commandline_text,
                        help='The path to the item that you want to delete')
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    parser.add_argument('-f',
                        '--force',
                        help='Completely deleted, no restore possiblity',
                        action='store_true')
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()
    root_dir = get_root_dir(jfs)
    item_path = posixpath.join(root_dir.path, args.file)
    item = jfs.getObject(item_path)
    if args.force:
        item.hard_delete()
    else:
        item.delete()
    print('%s deleted' % args.file)
    return True  # TODO: check return value of command
示例#9
0
def upload(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(
        description='Upload a file to JottaCloud.')
    parser.add_argument('localfile',
                        help='The local file that you want to upload',
                        type=argparse.FileType('r'))
    parser.add_argument('remote_dir',
                        help='The remote directory to upload the file to',
                        nargs='?',
                        type=commandline_text)
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    jfs = JFS.JFS()
    args = parse_args_and_apply_logging_level(parser, argv)
    decoded_filename = commandline_text(args.localfile.name)
    progress_bar = ProgressBar()
    callback = lambda monitor, size: progress_bar.show(monitor.bytes_read, size
                                                       )
    root_folder = get_root_dir(jfs)
    if args.remote_dir:
        target_dir_path = posixpath.join(root_folder.path, args.remote_dir)
        target_dir = jfs.getObject(target_dir_path)
    else:
        target_dir = root_folder
    upload = target_dir.up(args.localfile,
                           os.path.basename(decoded_filename),
                           upload_callback=callback)
    print('%s uploaded successfully' % decoded_filename)
    return True  # TODO: check return value
示例#10
0
 def __init__(self, username, password, path='.'):
     self.client = JFS.JFS(username, password)
     self.root = path
     self.dirty = False # True if some method has changed/added something and we need to get fresh data from JottaCloud
     # TODO: make self.dirty more smart, to know what path, to get from cache and not
     self.__newfiles = []
     self.__newfolders = []
示例#11
0
文件: cli.py 项目: forsberg/jottalib
def scanner():
    def is_dir(path):
        if not os.path.isdir(path):
            raise argparse.ArgumentTypeError('%s is not a valid directory' % path)
        return path
    parser = argparse.ArgumentParser(description=__doc__,
                                    epilog="""The program expects to find an entry for "jottacloud.com" in your .netrc,
                                    or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
                                    This is not an official JottaCloud project.""")
    parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'), default='warning')
    parser.add_argument('--errorfile', help='A file to write errors to', default='./jottacloudclient.log')
    parser.add_argument('--exclude', type=re.compile, action='append', help='Exclude paths matched by this pattern (can be repeated)')
    parser.add_argument('--version', action='version', version=__version__)
    parser.add_argument('--dry-run', action='store_true',
                        help="don't actually do any uploads or deletes, just show what would be done")
    parser.add_argument('topdir', type=is_dir, help='Path to local dir that needs syncing')
    parser.add_argument('jottapath', help='The path at JottaCloud where the tree shall be synced (must exist)')
    args = parse_args_and_apply_logging_level(parser)
    fh = logging.FileHandler(args.errorfile)
    fh.setLevel(logging.ERROR)
    logging.getLogger('').addHandler(fh)

    jfs = JFS()

    filescanner(args.topdir, args.jottapath, jfs, args.exclude, args.dry_run)
示例#12
0
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        # Import JottaCloud libraries.
        try:
            from jottalib import JFS
        except ImportError:
            raise
            raise BackendException(
                'JottaCloud backend requires jottalib'
                ' (see https://pypi.python.org/pypi/jottalib).')

        # Setup client instance.
        _pass = os.environ.get('JOTTACLOUD_PASSWORD', None)
        if _pass is None:
            _pass = self.get_password()
        self.client = JFS.JFS(parsed_url.username, _pass)
        #self.client.http_client.debug = False

        # Fetch destination folder entry (and create hierarchy if required).
        try:
            self.folder = self.client.getObject('%s/backup' % parsed_url.path)
        except JFS.JFSNotFoundError:
            parentfolder = self.client.getObject(parsed_url.path)
            try:
                self.folder = parentfolder.mkdir('backup')
            except:
                raise
                raise BackendException(
                    "Error while creating destination folder 'Backup')")
        except:
            raise
示例#13
0
    def test_xml(self):
        xml = """<?xml version="1.0" encoding="UTF-8"?>

<device time="2015-09-12-T23:14:25Z" host="dn-093.site-000.jotta.no">
  <name xml:space="preserve">Jotta</name>
  <type>JOTTA</type>
  <sid>ee93a510-907a-4d7c-bbb9-59df7894xxxx</sid>
  <size>58428516774</size>
  <modified>2015-09-12-T23:10:51Z</modified>
  <user>havardgulldahl</user>
  <mountPoints>
    <mountPoint>
      <name xml:space="preserve">Archive</name>
      <size>18577011381</size>
      <modified>2015-09-12-T23:10:51Z</modified>
    </mountPoint>
    <mountPoint>
      <name xml:space="preserve">Shared</name>
      <size>43777</size>
      <modified>2015-09-03-T21:12:55Z</modified>
    </mountPoint>
    <mountPoint>
      <name xml:space="preserve">Sync</name>
      <size>39851461616</size>
      <modified>2015-07-26-T22:26:54Z</modified>
    </mountPoint>
  </mountPoints>
  <metadata first="" max="" total="3" num_mountpoints="3"/>
</device>"""
        o = lxml.objectify.fromstring(xml)
        dev = JFS.JFSDevice(o, jfs, parentpath=jfs.rootpath)
        assert isinstance(o, lxml.objectify.ObjectifiedElement)
        # Test that mountpoints are populated correctly"
        assert sorted(dev.mountPoints.keys()) == ['Archive', 'Shared', 'Sync']
        assert all(
            isinstance(item, JFS.JFSMountPoint)
            for item in dev.mountPoints.values())
        # test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
        assert all(
            isinstance(item, JFS.JFSFile) for item in dev.files('Archive'))
        # test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
        mps = dev.mountpointobjects()
        assert all(isinstance(item, JFS.JFSFile) for item in dev.files(mps[2]))

        #test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
        assert all(
            isinstance(item, JFS.JFSFolder) for item in dev.folders('Archive'))
        #test "mountPoint" may be either an actual mountPoint element from JFSDevice.mountPoints{} or its .name. '
        mps = dev.mountpointobjects()
        assert all(
            isinstance(item, JFS.JFSFolder) for item in dev.folders(mps[0]))

        # test_properties
        assert isinstance(dev.modified, datetime.datetime)
        assert dev.path == '%s/%s' % (jfs.rootpath, 'Jotta')
        assert dev.name == 'Jotta'
        assert dev.type == 'JOTTA'
        assert dev.size == 58428516774
        assert dev.sid == 'ee93a510-907a-4d7c-bbb9-59df7894xxxx'
示例#14
0
def fuse(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    if not HAS_FUSE:
        message = [
            'jotta-fuse requires fusepy (pip install fusepy), install that and try again.'
        ]
        if os.name == 'nt':
            message.append(
                'Note: jotta-fuse is not supported on Windows, but Cygwin might work.'
            )
        print(' '.join(message))
        sys.exit(1)

    from .jottafuse import JottaFuse
    parser = argparse.ArgumentParser(
        description=__doc__,
        epilog=
        """The program expects to find an entry for "jottacloud.com" in your .netrc,
                                     or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
                                     This is not an official JottaCloud project."""
    )
    parser.add_argument(
        '--debug',
        action='store_true',
        help=
        'Run fuse in the foreground and add a lot of messages to help debug')
    parser.add_argument('--debug-fuse',
                        action='store_true',
                        help='Show all low-level filesystem operations')
    parser.add_argument('--debug-http',
                        action='store_true',
                        help='Show all HTTP traffic')
    parser.add_argument('--version', action='version', version=__version__)
    parser.add_argument(
        'mountpoint',
        type=is_dir,
        help=
        'A path to an existing directory where you want your JottaCloud tree mounted'
    )
    args = parser.parse_args(argv)
    if args.debug_http:
        http_client.HTTPConnection.debuglevel = 1
    if args.debug:
        requests_log = logging.getLogger("requests.packages.urllib3")
        requests_log.setLevel(logging.DEBUG)
        requests_log.propagate = True
        logging.basicConfig(level=logging.DEBUG)

    auth = JFS.get_auth_info()
    fuse = FUSE(JottaFuse(auth),
                args.mountpoint,
                debug=args.debug_fuse,
                sync_read=True,
                foreground=args.debug,
                raw_fi=False,
                fsname="JottaCloudFS",
                subtype="fuse")
示例#15
0
文件: cli.py 项目: forsberg/jottalib
def mkdir():
    parser = argparse.ArgumentParser(description='Create a new folder in Jottacloud.')
    parser.add_argument('newdir', help='The path to the folder that you want to create')
    parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
        choices=('debug', 'info', 'warning', 'error'), default='warning')
    args = parse_args_and_apply_logging_level(parser)
    jfs = JFS.JFS()
    root_folder = get_root_dir(jfs)
    root_folder.mkdir(args.newdir)
示例#16
0
 def test_errors(self):
     with pytest.raises(JFS.JFSCredentialsError):  # HTTP 401
         JFS.JFS(auth=('PYTEST', 'PYTEST'))
     with pytest.raises(JFS.JFSNotFoundError):  # HTTP 404
         jfs.get('/Jotta/Archive/FileNot.found')
     with pytest.raises(JFS.JFSRangeError):  # HTTP 416
         p = "/Jotta/Archive/testfile_up_and_readpartial.txt"
         t = jfs.up(p, StringIO.StringIO(TESTFILEDATA))
         f = jfs.getObject(p)
         f.readpartial(10, 3)  # Range start index larger than end index;
         f.delete()
示例#17
0
文件: cli.py 项目: forsberg/jottalib
def restore():
    parser = argparse.ArgumentParser(description='Restore a deleted item from Jottacloud')
    parser.add_argument('file', help='The path to the item that you want to restore')
    parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
        choices=('debug', 'info', 'warning', 'error'), default='warning')
    args = parse_args_and_apply_logging_level(parser)
    jfs = JFS.JFS()
    root_dir = get_root_dir(jfs)
    item_path = posixpath.join(root_dir.path, args.file)
    item = jfs.getObject(item_path)
    item.restore()
    print '%s restored' % args.file
示例#18
0
def ls(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(description='List files in Jotta folder.',
                                     add_help=False)
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    parser.add_argument('-h',
                        '--humanize',
                        help='Print human-readable file sizes.',
                        action='store_true')
    parser.add_argument('-a',
                        '--all',
                        help='Show all files, even deleted ones',
                        action='store_true')
    parser.add_argument('item',
                        nargs='?',
                        help='The file or directory to list. Defaults to the '
                        'root dir')
    parser.add_argument('-H', '--help', help='Print this help', action='help')
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()
    root_folder = get_root_dir(jfs)
    if args.item:
        item_path = posixpath.join(root_folder.path, args.item)
        item = jfs.getObject(item_path)
    else:
        item = root_folder
    if isinstance(item, JFS.JFSFolder):
        files = [(f.created, print_size(f.size, humanize=args.humanize),
                  u'D' if f.deleted else u' ', f.name) for f in item.files()
                 if not f.deleted or args.all]
        folders = [(u' ' * 25, u'', u'D' if f.deleted else u' ', f.name)
                   for f in item.folders()]
        widest_size = 0
        for f in files:
            if len(f[1]) > widest_size:
                widest_size = len(f[1])
        for item in sorted(files + folders, key=lambda t: t[3]):
            if args.all:
                print(u'%s %s %s %s' %
                      (item[0], item[1].rjust(widest_size), item[2], item[3]))
            else:
                print(u'%s %s %s' %
                      (item[0], item[1].rjust(widest_size), item[3]))
    else:
        print(' '.join([
            str(item.created),
            print_size(item.size, humanize=args.humanize), item.name
        ]))
示例#19
0
文件: cli.py 项目: forsberg/jottalib
def share():
    parser = argparse.ArgumentParser(description='Share a file on JottaCloud and get the public URI.',
                                     epilog='Note: This utility needs to find JOTTACLOUD_USERNAME'
                                     ' and JOTTACLOUD_PASSWORD in the running environment.')
    parser.add_argument('localfile', help='The local file that you want to share',
                                     type=argparse.FileType('r'))
    args = parser.parse_args()
    jfs = JFS.JFS()
    jottadev = get_jotta_device(jfs)
    jottashare = jottadev.mountPoints['Shared']
    upload = jottashare.up(args.localfile)  # upload file
    public = upload.share() # share file
    for (filename, uuid, publicURI) in public.sharedFiles():
        print '%s is now available to the world at %s' % (filename, publicURI)
示例#20
0
def mkdir(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(
        description='Create a new folder in Jottacloud.')
    parser.add_argument('newdir',
                        type=commandline_text,
                        help='The path to the folder that you want to create')
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()
    root_folder = get_root_dir(jfs)
    root_folder.mkdir(args.newdir)
    return True  #  TODO: check return value of mkdir
示例#21
0
def fuse(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    if not HAS_FUSE:
        message = ['jotta-fuse requires fusepy (pip install fusepy), install that and try again.']
        if os.name == 'nt':
            message.append('Note: jotta-fuse is not supported on Windows, but Cygwin might work.')
        print(' '.join(message))
        sys.exit(1)


    from .jottafuse import JottaFuse
    parser = argparse.ArgumentParser(description=__doc__,
                                     epilog="""The program expects to find an entry for "jottacloud.com" in your .netrc,
                                     or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
                                     This is not an official JottaCloud project.""")
    parser.add_argument('--debug',
                        action='store_true',
                        help='Run fuse in the foreground and add a lot of messages to help debug')
    parser.add_argument('--debug-fuse',
                        action='store_true',
                        help='Show all low-level filesystem operations')
    parser.add_argument('--debug-http',
                        action='store_true',
                        help='Show all HTTP traffic')
    parser.add_argument('--version',
                        action='version', version=__version__)
    parser.add_argument('mountpoint',
                        type=is_dir,
                        help='A path to an existing directory where you want your JottaCloud tree mounted')
    args = parser.parse_args(argv)
    if args.debug_http:
        http_client.HTTPConnection.debuglevel = 1
    if args.debug:
        requests_log = logging.getLogger("requests.packages.urllib3")
        requests_log.setLevel(logging.DEBUG)
        requests_log.propagate = True
        logging.basicConfig(level=logging.DEBUG)

    auth = JFS.get_auth_info()
    fuse = FUSE(JottaFuse(auth), args.mountpoint, debug=args.debug_fuse,
                sync_read=True, foreground=args.debug, raw_fi=False,
                fsname="JottaCloudFS", subtype="fuse")
示例#22
0
 def download_jfsfile(remote_object, tofolder=None, checksum=False):
     'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
     if tofolder is None:
         tofolder = '.'  # with no arguments, store in current dir
     total_size = remote_object.size
     if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT,
                                JFS.ProtoFile.STATE_INCOMPLETE):
         puts(
             colored.red(
                 '%s was NOT downloaded successfully - Incomplete file' %
                 remote_file.name))
         return False
     topath = os.path.join(tofolder, remote_object.name)
     with open(topath, 'wb') as fh:
         bytes_read = 0
         puts(
             colored.white('Downloading: %s, size: %s \t' %
                           (remote_object.name,
                            print_size(total_size, humanize=True))))
         with ProgressBar(expected_size=total_size) as bar:
             for chunk_num, chunk in enumerate(remote_object.stream()):
                 fh.write(chunk)
                 bytes_read += len(chunk)
                 bar.show(bytes_read)
     if checksum:
         md5_lf = JFS.calculate_md5(open(topath, 'rb'))
         md5_jf = remote_object.md5
         logging.info('%s - Checksum for downloaded file' % md5_lf)
         logging.info('%s - Checksum for server file' % md5_jf)
         if md5_lf != md5_jf:
             puts(colored.blue('%s - Checksum for downloaded file' %
                               md5_lf))
             puts(colored.blue('%s - Checksum for server file' % md5_jf))
             puts(
                 colored.red(
                     '%s was NOT downloaded successfully - cheksum mismatch'
                     % remote_object.name))
             return False
         puts(
             colored.green(
                 '%s was downloaded successfully - checksum  matched' %
                 remote_object.name))
     return True
示例#23
0
    def test_xml(self):
        xml = """<?xml version="1.0" encoding="UTF-8"?>

<file name="testimage.jpg" uuid="9ebcfe1a-98b1-4e38-a73e-f498555da865" time="2015-09-13-T21:22:46Z" host="dn-094.site-000.jotta.no">
  <path xml:space="preserve">/havardgulldahl/Jotta/Archive</path>
  <abspath xml:space="preserve">/havardgulldahl/Jotta/Archive</abspath>
  <currentRevision>
    <number>5</number>
    <state>COMPLETED</state>
    <created>2015-07-25-T21:18:49Z</created>
    <modified>2015-07-25-T21:18:49Z</modified>
    <mime>image/jpeg</mime>
    <mstyle>IMAGE_JPEG</mstyle>
    <size>1816221</size>
    <md5>125073533339a616b99bc53efc509561</md5>
    <updated>2015-07-25-T21:18:50Z</updated>
  </currentRevision>
  <revisions>
    <revision>
      <number>4</number>
      <state>COMPLETED</state>
      <created>2015-07-25-T21:18:05Z</created>
      <modified>2015-07-25-T21:18:05Z</modified>
      <mime>image/jpeg</mime>
      <mstyle>IMAGE_JPEG</mstyle>
      <size>1816221</size>
      <md5>125073533339a616b99bc53efc509561</md5>
      <updated>2015-07-25-T21:18:07Z</updated>
    </revision>
    <revision>
      <number>3</number>
      <state>COMPLETED</state>
      <created>2015-07-25-T21:17:49Z</created>
      <modified>2015-07-25-T21:17:49Z</modified>
      <mime>image/jpeg</mime>
      <mstyle>IMAGE_JPEG</mstyle>
      <size>1816221</size>
      <md5>125073533339a616b99bc53efc509561</md5>
      <updated>2015-07-25-T21:17:50Z</updated>
    </revision>
    <revision>
      <number>2</number>
      <state>COMPLETED</state>
      <created>2015-07-25-T21:01:45Z</created>
      <modified>2015-07-25-T21:01:45Z</modified>
      <mime>image/jpeg</mime>
      <mstyle>IMAGE_JPEG</mstyle>
      <size>1816221</size>
      <md5>125073533339a616b99bc53efc509561</md5>
      <updated>2015-07-25-T21:01:46Z</updated>
    </revision>
    <revision>
      <number>1</number>
      <state>COMPLETED</state>
      <created>2015-07-25-T21:00:02Z</created>
      <modified>2015-07-25-T21:00:02Z</modified>
      <mime>image/jpeg</mime>
      <mstyle>IMAGE_JPEG</mstyle>
      <size>1816221</size>
      <md5>125073533339a616b99bc53efc509561</md5>
      <updated>2015-07-25-T21:00:03Z</updated>
    </revision>
  </revisions>
</file>"""

        o = lxml.objectify.fromstring(xml)
        dev = JFS.JFSFile(o, jfs, parentpath=jfs.rootpath + '/Jotta/Archive')

        #test ProtoFile properties
        assert dev.path == jfs.rootpath + '/Jotta/Archive/testimage.jpg'
        assert dev.name == 'testimage.jpg'
        assert dev.uuid == '9ebcfe1a-98b1-4e38-a73e-f498555da865'
        assert dev.deleted == None
        assert dev.is_deleted() == False

        #test native properties
        assert dev.revisionNumber == 5
        assert dev.created == datetime.datetime(
            2015, 7, 25, 21, 18, 49).replace(tzinfo=dateutil.tz.tzutc())
        assert dev.modified == datetime.datetime(
            2015, 7, 25, 21, 18, 49).replace(tzinfo=dateutil.tz.tzutc())
        assert dev.updated == datetime.datetime(
            2015, 7, 25, 21, 18, 50).replace(tzinfo=dateutil.tz.tzutc())
        assert dev.size == 1816221
        assert dev.md5 == '125073533339a616b99bc53efc509561'
        assert dev.mime == 'image/jpeg'
        assert dev.state == 'COMPLETED'
示例#24
0
    def test_xml(self):
        xml = """<?xml version="1.0" encoding="UTF-8"?>

<mountPoint time="2015-09-13-T00:16:31Z" host="dn-097.site-000.jotta.no">
  <name xml:space="preserve">Sync</name>
  <path xml:space="preserve">/havardgulldahl/Jotta</path>
  <abspath xml:space="preserve">/havardgulldahl/Jotta</abspath>
  <size>39851461616</size>
  <modified>2015-07-26-T22:26:54Z</modified>
  <device>Jotta</device>
  <user>havardgulldahl</user>
  <folders>
    <folder name="folder1"/>
    <folder name="folder2"/>
    <folder name="folder3"/>
    <folder name="folder4"/>
    <folder name="folder5"/>
    <folder name="folder6"/>
    <folder name="folder7"/>
  </folders>
  <files>
    <file name="bigfile" uuid="7a36a217-88d5-4804-99df-bbc42eb4a9f2">
      <latestRevision>
        <number>1</number>
        <state>INCOMPLETE</state>
        <created>2015-05-29-T09:02:53Z</created>
        <modified>2015-05-29-T09:02:53Z</modified>
        <mime>application/octet-stream</mime>
        <mstyle>APPLICATION_OCTET_STREAM</mstyle>
        <md5>4d710d3a12699730976216836a5217a8</md5>
        <updated>2015-05-29-T09:02:53Z</updated>
      </latestRevision>
    </file>
    <file name="test.pdf" uuid="1caec763-2ed0-4e88-9d3c-650f3babecc4">
      <currentRevision>
        <number>3</number>
        <state>COMPLETED</state>
        <created>2015-07-26-T22:26:54Z</created>
        <modified>2015-07-26-T22:26:54Z</modified>
        <mime>application/pdf</mime>
        <mstyle>APPLICATION_PDF</mstyle>
        <size>116153</size>
        <md5>138396327a51ea6bf20caa72cf6d6667</md5>
        <updated>2015-07-26-T22:26:54Z</updated>
      </currentRevision>
    </file>
  </files>
  <metadata first="" max="" total="9" num_folders="7" num_files="2"/>
</mountPoint>"""
        o = lxml.objectify.fromstring(xml)
        dev = JFS.JFSMountPoint(o, jfs, parentpath=jfs.rootpath + '/Jotta')

        #test native properties
        assert dev.name == 'Sync'
        assert dev.size == 39851461616
        assert isinstance(dev.modified, datetime.datetime)
        assert dev.modified == datetime.datetime(
            2015, 7, 26, 22, 26, 54).replace(tzinfo=dateutil.tz.tzutc())

        with pytest.raises(JFS.JFSError):
            dev.delete()
            dev.rename('sillywalkministry')

        # test JFSFolder inheritance
        assert dev.path == jfs.rootpath + '/Jotta/Sync'
        assert dev.deleted == None
        assert dev.is_deleted() == False
        assert all(isinstance(item, JFS.JFSFile) for item in dev.files())
        assert all(isinstance(item, JFS.JFSFolder) for item in dev.folders())
        newf = dev.mkdir('testdir')
        assert isinstance(newf, JFS.JFSFolder)
        newf.delete()

        _f = tempfile.NamedTemporaryFile()
        _f.write('123test')

        newfile = dev.up(_f)
        assert isinstance(newfile, JFS.JFSFile)
        newfile.delete()
        newfile = dev.up(_f, filename='heyhei123.txt')
        assert isinstance(newfile, JFS.JFSFile)
        assert newfile.name == 'heyhei123.txt'
        newfile.delete()
        assert isinstance(dev.filedirlist(), JFS.JFSFileDirList)
示例#25
0
文件: cli.py 项目: antonhagg/jottalib
def download(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(description='Download a file or folder from Jottacloud.')
    parser.add_argument('remoteobject', help='The path to the file or folder that you want to download')
    parser.add_argument('-m', '--mode',
                        help='Mode of operation(decides where the default root should be): DEVICE - Root is above device level (need to specify both Device and Mountpoint in front of the folder i.e. DEVICENAME/mountpoint/folder), ARCHIVE - root is in the archive mountpoint, SYNC - root is in sync mountpoint, Default: %(default)s.',
                        choices=( 'device', 'archive', 'sync'), default='sync')
    parser.add_argument('-l', '--loglevel', help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'), default='warning')
    parser.add_argument('-c', '--checksum', help='Verify checksum of file after download', action='store_true' )
    parser.add_argument('-r', '--resume', help='Will not download the files again if it exist in path', action='store_true' )
    parser.add_argument('-v', '--verbose', help='Increase output verbosity', action='store_true' )
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()
    
    if args.mode == 'sync':      #Device is Jotta and mountpoint is Sync
        device = 'Jotta'
        mountpoint = 'Sync'
        print('Device is: %s' % device)
        print('Mountpoint is: %s' % mountpoint)
        root_folder = get_root_dir(jfs,device,mountpoint)
    elif args.mode == 'archive': #Device is Jotta and mountpoint is Archive
        device = 'Jotta'
        mountpoint = 'Archive'
        print('Device is: %s' % device)
        print('Mountpoint is: %s' % mountpoint)
        root_folder = get_root_dir(jfs,device,mountpoint)
    elif args.mode == 'device':  #Need to get the Device from first part of path and mountpoint from second part
        path_in_parts = os.path.normpath(args.remoteobject).split(os.sep)
        device = path_in_parts[0]
        mountpoint = path_in_parts[1]
        print('Device is: %s' % device)
        print('Mountpoint is: %s' % mountpoint)
        if len(path_in_parts)<3:
            print('You need to specify at least one folder of file in addition to the device and mountpoint (i.e. 3 levels)')
            exit(1)
        else:
            root_folder = get_root_dir(jfs,device,mountpoint)
            del path_in_parts[0:2] #Removing device and mountpoint from path
            args.remoteobject = "/".join(path_in_parts)
    else:
        exit(1)#This shouldn't really be necessary
        
    path_to_object = posixpath.join(root_folder.path, args.remoteobject)

    if args.verbose:
        print('Root folder path: %s' % root_folder.path)
        print('Relative path to object: %s' % args.remoteobject)
        print('Absolute path to object: %s' % path_to_object)
    remote_object = jfs.getObject(path_to_object)
    if hasattr(remote_object, 'size'): #Check if it's a file that is downloaded by checking if the attribute 'size' exist
        remote_file = remote_object
        total_size = remote_file.size
        if total_size == -1: # Indicates an incomplete file
            print('%s was NOT downloaded successfully - Incomplete file' % remote_file.name)
            exit(1)
        with open(remote_file.name, 'wb') as fh:
            bytes_read = 0
            with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
                for chunk_num, chunk in enumerate(remote_file.stream()):
                    fh.write(chunk)
                    bytes_read += len(chunk)
                    bar.show(bytes_read)
        if args.checksum:
            md5_lf = JFS.calculate_md5(open(remote_file.name, 'rb'))
            md5_jf = remote_file.md5
            if md5_lf != md5_jf:
                print('%s - Checksum for downloaded file' % md5_lf)
                print('%s - Checksum for server file' % md5_jf)
                print('%s was NOT downloaded successfully - cheksum mismatch' % remote_file.name)
                exit(1)
            if args.verbose:
                print('%s - Checksum for downloaded file' % md5_lf)
                print('%s - Checksum for server file' % md5_jf)
            print('%s was downloaded successfully - checksum  matched' % remote_file.name)
            exit(1)
        print('%s downloaded successfully - checksum not checked' % remote_file.name)


    else: #if it's not a file it has to be a folder
        incomplete_files = [] #Create an list where we can store incomplete files
        checksum_error_files = [] #Create an list where we can store checksum error files
        zero_files = [] #Create an list where we can store zero files
        long_path = [] #Create an list where we can store skipped files and folders because of long path
        if args.verbose:
            print "It's a folder that is downloaded - getting the folder and file structure. This might take a while if the tree is big..."     
        fileTree = remote_object.filedirlist().tree #Download the folder tree
        if args.verbose:
            print('Total number of folders to download: %d' % len(fileTree))

        #Iterate through each folder
        for folder in fileTree:
            #We need to strip the path to the folder path from account,device and mountpoint details      
            rel_path_to_object = folder[len(jfs.username)+len(device)+len(mountpoint)+4:] 

            if len(rel_path_to_object) > 250: #Windows has a limit of 250 characters in path
                print('%s was NOT downloaded successfully - path to long' % rel_path_to_object)
                long_path.append(rel_path_to_object)
            else:
                if args.verbose:
                    print('Entering a new folder: %s' % rel_path_to_object)
                if not os.path.exists(rel_path_to_object): #Create the folder locally if it doesn't exist
                    os.makedirs(rel_path_to_object)
                for _file in fileTree[folder]: #Enter the folder and download the files within
                    abs_path_to_object = posixpath.join(root_folder.path, posixpath.join(rel_path_to_object, _file[0])) #This is the absolute path to the file that is going to be downloaded
                    if args.verbose:
                        print('Downloading the file from: %s' % abs_path_to_object)
                    if _file[1]==-1: #Corrupt and incomplete files will be skipped 
                        print('%s was NOT downloaded successfully - Incomplete or corrupt file' % _file[0])
                        incomplete_files.append(posixpath.join(rel_path_to_object,_file[0]))
                    else:
                        remote_object = jfs.getObject(abs_path_to_object)
                        remote_file = remote_object
                        total_size = remote_file.size
                        if total_size == 0: # Indicates an zero file
                            print('%s was NOT downloaded successfully - zero file' % remote_file.name)
                            zero_files.append(posixpath.join(rel_path_to_object,remote_file.name)) 
                        else:
                            if len(posixpath.join(rel_path_to_object,remote_file.name)) > 250: #Windows has a limit of 250 characters in path
                                print('%s was NOT downloaded successfully - path to long' % remote_file.name)
                                long_path.append(posixpath.join(rel_path_to_object,remote_file.name))    
                            else:
                                if args.verbose:
                                    print('Downloading the file to: %s' % posixpath.join(rel_path_to_object,remote_file.name))
                                if args.resume: #Check if file exist in path
                                    if os.path.isfile(posixpath.join(rel_path_to_object,remote_file.name)):
                                        print('File exist - skipping downloading: %s' % posixpath.join(rel_path_to_object,remote_file.name))
                                    else:    
                                        with open(posixpath.join(rel_path_to_object,remote_file.name), 'wb') as fh:
                                            bytes_read = 0
                                            with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
                                                for chunk_num, chunk in enumerate(remote_file.stream()):
                                                    fh.write(chunk)
                                                    bytes_read += len(chunk)
                                                    bar.show(bytes_read)
                                else:
                                    with open(posixpath.join(rel_path_to_object,remote_file.name), 'wb') as fh:
                                        bytes_read = 0
                                        with ProgressBar(expected_size=total_size, label='Downloading: %s, Size:%s' % (remote_file.name, print_size(total_size, True))) as bar:
                                            for chunk_num, chunk in enumerate(remote_file.stream()):
                                                fh.write(chunk)
                                                bytes_read += len(chunk)
                                                bar.show(bytes_read)
                                if args.checksum:
                                    md5_lf = JFS.calculate_md5(open(posixpath.join(rel_path_to_object,remote_file.name), 'rb'))
                                    md5_jf = remote_file.md5
                                    if md5_lf != md5_jf:
                                        print('%s - Checksum for downloaded file' % md5_lf)
                                        print('%s - Checksum for server file' % md5_jf)
                                        print('%s was NOT downloaded successfully - cheksum mismatch' % remote_file.name)
                                        checksum_error_files.append(posixpath.join(rel_path_to_object,remote_file.name))
                                    else:
                                        if args.verbose:
                                            print('%s - Checksum for downloaded file' % md5_lf)
                                            print('%s - Checksum for server file' % md5_jf)
                                        print('%s was downloaded successfully - checksum  matched' % remote_file.name)
                                else:    
                                    print('%s downloaded successfully - checksum not checked' % remote_file.name)
        #Incomplete files
        if len(incomplete_files)> 0:
            with codecs.open("incomplete_files.txt", "w", "utf-8") as text_file:
                for item in incomplete_files:        
                    text_file.write("%s\n" % item)
        print('Incomplete files (not downloaded): %d' % len(incomplete_files))
        if args.verbose:
            for _files in incomplete_files:
                print _files

        #Checksum error files
        if len(checksum_error_files)> 0:
            with codecs.open("checksum_error_files.txt", "w", "utf-8") as text_file:
                for item in checksum_error_files:
                    text_file.write("%s\n" % item)
        print('Files with checksum error (not downloaded): %d' % len(checksum_error_files))
        if args.verbose:
            for _files in checksum_error_files:
                print _files

        #zero files
        if len(zero_files)> 0:
            with codecs.open("zero_files.txt", "w", "utf-8") as text_file:
                for item in zero_files:
                    text_file.write("%s\n" % item)
        print('Files with zero size (not downloaded): %d' % len(zero_files))
        if args.verbose:
            for _files in zero_files:
                print _files

        #long path
        if len(long_path)> 0:
            with codecs.open("long_path.txt", "w", "utf-8") as text_file:
                for item in long_path:
                    text_file.write("%s\n" % item)
        print('Folder and files not downloaded because of path to long: %d' % len(long_path))
        if args.verbose:
            for _files in long_path:
                print _files
示例#26
0
# import standardlib
import os, StringIO, logging, datetime
import tempfile, posixpath, urllib

# import dependencies
import lxml
import dateutil
import requests

# import py.test
import pytest  # pip install pytest

# import jotta
from jottalib import JFS, __version__

jfs = JFS.JFS()  # get username and password from environment or .netrc

TESTFILEDATA = """
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla est dolor, convallis fermentum sapien in, fringilla congue ligula. Fusce at justo ac felis vulputate laoreet vel at metus. Aenean justo lacus, porttitor dignissim imperdiet a, elementum cursus ligula. Vivamus eu est viverra, pretium arcu eget, imperdiet eros. Curabitur in bibendum."""


class TestJFS:
    def test_xml(self):
        xml = """<?xml version="1.0" encoding="UTF-8"?>

<user time="2015-09-12-T23:14:23Z" host="dn-093.site-000.jotta.no">
  <username>havardgulldahl</username>
  <account-type>unlimited</account-type>
  <locked>false</locked>
  <capacity>-1</capacity>
  <max-devices>-1</max-devices>
示例#27
0
    return "%.3f%s" % (size / math.pow(1024, p), units[int(p)])


if __name__ == '__main__':
    # we need an active login to test
    import netrc
    try:
        n = netrc.netrc()
        username, account, password = n.authenticators(
            'jottacloud.com')  # read .netrc entry for 'machine jottacloud'
    except Exception as e:
        log.exception(e)
        username = os.environ['JOTTACLOUD_USERNAME']
        password = os.environ['JOTTACLOUD_PASSWORD']

    jfs = JFS.JFS(auth=(username, password))
    lite = LiteJFS(username, password)

    filesize = 1024 * 10 * 10

    data = os.urandom(filesize)
    testfile = tempfile.NamedTemporaryFile()
    puts(colored.blue('Creating test file.'))
    for i in progress.bar(range(0, 1000)):
        testfile.write(data)
    filesize = os.path.getsize(testfile.name)

    p = '/Jotta/Archive/test/%s.data' % os.path.basename(testfile.name)

    # UPLOAD TEST
    puts(
示例#28
0
import sys, os, os.path
import urllib, logging, datetime
import argparse  # 2.7

# import jottacloud client
from jottalib import JFS

if __name__ == '__main__':
    logging.basicConfig(level=logging.WARNING)
    parser = argparse.ArgumentParser(
        description='Share a file on JottaCloud and get the public URI.',
        epilog=
        'Note: This utility needs to find JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.'
    )
    parser.add_argument('localfile',
                        help='The local file that you want to share',
                        type=argparse.FileType('r'))
    args = parser.parse_args()
    #logging.debug(args)
    jfs = JFS.JFS(os.environ['JOTTACLOUD_USERNAME'],
                  password=os.environ['JOTTACLOUD_PASSWORD'])
    jottadev = None
    for j in jfs.devices:  # find Jotta/Shared folder
        if j.name == 'Jotta':
            jottadev = j
    jottashare = jottadev.mountPoints['Shared']
    upload = jottashare.up(args.localfile)  # upload file
    public = upload.share()  # share file
    for (filename, uuid, publicURI) in public.sharedFiles():
        print '%s is now available to the world at %s' % (filename, publicURI)
示例#29
0
    def test_xml(self):
        xml = b"""<?xml version="1.0" encoding="UTF-8"?>

<folder name="test2" time="2015-09-28-T13:49:05Z" host="dn-091.site-000.jotta.no">
  <path xml:space="preserve">/havardgulldahl/Jotta/Archive</path>
  <abspath xml:space="preserve">/havardgulldahl/Jotta/Archive</abspath>
  <folders>
    <folder name="Documents"/>
  </folders>
  <files>
    <file name="boink.txt~" uuid="c6684726-a842-4536-95b9-140584515dd1">
      <currentRevision>
        <number>1</number>
        <state>COMPLETED</state>
        <created>2014-10-05-T10:23:18Z</created>
        <modified>2014-10-05-T10:23:18Z</modified>
        <mime>application/octet-stream</mime>
        <mstyle>APPLICATION_OCTET_STREAM</mstyle>
        <size>40</size>
        <md5>b924ebbc79ad414ded4af442ac7080d3</md5>
        <updated>2014-11-23-T21:12:47Z</updated>
      </currentRevision>
    </file>
    <file name="boink.txx~" uuid="1b243d8e-d6df-412c-a2ce-926ebaa73f47">
      <currentRevision>
        <number>1</number>
        <state>COMPLETED</state>
        <created>2014-10-05-T10:23:18Z</created>
        <modified>2014-10-05-T10:23:18Z</modified>
        <mime>application/octet-stream</mime>
        <mstyle>APPLICATION_OCTET_STREAM</mstyle>
        <size>40</size>
        <md5>0c7652132733ead903dbdb942577fed7</md5>
        <updated>2014-11-23-T21:27:21Z</updated>
      </currentRevision>
    </file>
    <file name="boink.txy~" uuid="ba1ec941-8901-4eec-b797-bde9b6b1958c">
      <currentRevision>
        <number>1</number>
        <state>COMPLETED</state>
        <created>2014-10-05-T10:23:18Z</created>
        <modified>2014-10-05-T10:23:18Z</modified>
        <mime>application/octet-stream</mime>
        <mstyle>APPLICATION_OCTET_STREAM</mstyle>
        <size>40</size>
        <md5>d32f37bf39041d9bb92ad45012e32cb9</md5>
        <updated>2014-11-23-T21:05:11Z</updated>
      </currentRevision>
    </file>
    <file name="boink.txz~" uuid="6724af96-e462-4862-b82a-00b7836a0681">
      <currentRevision>
        <number>1</number>
        <state>COMPLETED</state>
        <created>2014-10-05-T10:23:18Z</created>
        <modified>2014-10-05-T10:23:18Z</modified>
        <mime>application/octet-stream</mime>
        <mstyle>APPLICATION_OCTET_STREAM</mstyle>
        <size>9</size>
        <md5>e3a2cba90ec7630bdf1d0566c8abb93e</md5>
        <updated>2014-11-23-T21:03:25Z</updated>
      </currentRevision>
    </file>
    <file name="dingdong" uuid="95c4bbcc-9a59-4669-b4cb-ee49c186ae1b">
      <currentRevision>
        <number>127</number>
        <state>COMPLETED</state>
        <created>2014-10-05-T10:23:18Z</created>
        <modified>2014-10-05-T10:23:18Z</modified>
        <mime>application/octet-stream</mime>
        <mstyle>APPLICATION_OCTET_STREAM</mstyle>
        <size>20480</size>
        <md5>daa100df6e6711906b61c9ab5aa16032</md5>
        <updated>2014-11-23-T22:56:30Z</updated>
      </currentRevision>
    </file>
    <file name="fisk.txt" uuid="8b6db048-c44b-4656-8024-737a0c38c7ad">
      <currentRevision>
        <number>1</number>
        <state>COMPLETED</state>
        <created>2014-10-05-T10:23:18Z</created>
        <modified>2014-10-05-T10:23:18Z</modified>
        <mime>text/plain</mime>
        <mstyle>TEXT_PLAIN</mstyle>
        <size>18</size>
        <md5>308285a59ae0a4a5ede4f7a0c08d2390</md5>
        <updated>2014-11-23-T19:56:51Z</updated>
      </currentRevision>
    </file>
    <file name="nyboink.txt" uuid="3d0a0a28-4c68-4e6a-8414-fa5c37ec45cb">
      <currentRevision>
        <number>26</number>
        <state>COMPLETED</state>
        <created>2014-10-05-T10:23:18Z</created>
        <modified>2014-10-05-T10:23:18Z</modified>
        <mime>text/plain</mime>
        <mstyle>TEXT_PLAIN</mstyle>
        <size>6</size>
        <md5>6882bf1ef7f4d938a4cf2931d6953fa1</md5>
        <updated>2014-11-23-T21:28:55Z</updated>
      </currentRevision>
    </file>
    <file name="oo.txt" uuid="691aa9b6-eec1-4564-803e-6288dc57aa37">
      <currentRevision>
        <number>3</number>
        <state>COMPLETED</state>
        <created>2014-12-18-T21:23:48Z</created>
        <modified>2014-12-18-T21:23:48Z</modified>
        <mime>text/plain</mime>
        <mstyle>TEXT_PLAIN</mstyle>
        <size>4</size>
        <md5>aa3f5bb8c988fa9b75a1cdb1dc4d93fc</md5>
        <updated>2014-12-18-T21:23:48Z</updated>
      </currentRevision>
    </file>
    <file name="pingpong.data" uuid="f02f1ca1-c6a4-403d-b344-f4e3417d92fd">
      <currentRevision>
        <number>165</number>
        <state>COMPLETED</state>
        <created>2014-12-18-T21:20:32Z</created>
        <modified>2014-12-18-T21:20:32Z</modified>
        <mime>application/octet-stream</mime>
        <mstyle>APPLICATION_OCTET_STREAM</mstyle>
        <size>9216</size>
        <md5>ec041186ebff92a26ab3ef2dd34dd0e7</md5>
        <updated>2014-12-18-T21:20:32Z</updated>
      </currentRevision>
    </file>
    <file name="testfs" uuid="a00034dd-971c-4699-8ee0-e7514045770e">
      <currentRevision>
        <number>2</number>
        <state>COMPLETED</state>
        <created>2014-10-05-T10:23:18Z</created>
        <modified>2014-10-05-T10:23:18Z</modified>
        <mime>application/octet-stream</mime>
        <mstyle>APPLICATION_OCTET_STREAM</mstyle>
        <size>0</size>
        <md5>d41d8cd98f00b204e9800998ecf8427e</md5>
        <updated>2014-11-23-T19:41:03Z</updated>
      </currentRevision>
    </file>
  </files>
  <metadata first="" max="" total="11" num_folders="1" num_files="10"/>
</folder>
"""
        o = lxml.objectify.fromstring(xml)
        dev = JFS.JFSFolder(o, jfs, parentpath=jfs.rootpath + '/Jotta/Archive')
        dev.synced = True  # make sure our tests are based on the xml above, and not live

        #test native properties
        assert dev.path == jfs.rootpath + '/Jotta/Archive/test2'
        assert dev.name == 'test2'
        assert dev.deleted == None
        assert dev.is_deleted() == False

        #test convenience methods
        assert len(list(dev.folders())) == 1
        assert len(list(dev.files())) == 10
        assert all(isinstance(item, JFS.JFSFile) for item in dev.files())
        assert all(isinstance(item, JFS.JFSFolder) for item in dev.folders())
示例#30
0
WIN32 = (sys.platform == "win32")

TESTFILEDATA = u"""
Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nulla est dolor, convallis fermentum sapien in, fringilla congue ligula. Fusce at justo ac felis vulputate laoreet vel at metus. Aenean justo lacus, porttitor dignissim imperdiet a, elementum cursus ligula. Vivamus eu est viverra, pretium arcu eget, imperdiet eros. Curabitur in bibendum."""

EPOCH = datetime(1970, 1, 1)


def timestamp():
    """
    :return: now in unix time, eg. seconds since 1970
    """
    return (datetime.utcnow() - EPOCH).total_seconds()


jfs = JFS.JFS()
dev = cli.get_jfs_device(jfs)
root = cli.get_root_dir(jfs)


def test_get_jotta_device():
    assert isinstance(dev, JFS.JFSDevice)
    assert dev.name == 'Jotta'


def test_get_root_dir():
    assert isinstance(root, JFS.JFSMountPoint)


def test_ls():
    assert cli.ls([])
示例#31
0
def ls(argv=None):
    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(description='List files in Jotta folder.',
                                     add_help=False)
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    parser.add_argument(
        '-h',
        '--humanize',  # this matches ls(1)
        help='Print human-readable file sizes.',
        action='store_true')
    parser.add_argument(
        '-a',
        '--all',
        action='store_true',
        help='Include deleted and incomplete files (otherwise ignored)')
    parser.add_argument(
        'item',
        nargs='?',
        help='The file or directory to list. Defaults to the root dir',
        type=commandline_text)
    parser.add_argument(
        '-H',  # because -h means --humanize
        '--help',
        help='Print this help',
        action='help')
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()
    root_folder = get_root_dir(jfs)
    if args.item:
        if args.item.startswith('//'):
            # break out of root_folder
            item_path = posixpath.join(jfs.rootpath, args.item[1:])
        else:
            item_path = posixpath.join(root_folder.path, args.item)
        item = jfs.getObject(item_path)
    else:
        item = root_folder
    timestamp_width = 25
    logging.debug('about to ls %r', item)
    if isinstance(item, JFS.JFSFolder):
        files = [
            (f.created,
             print_size(f.size, humanize=args.humanize) if f.size else u'',
             u'D' if f.deleted else u'I' if f.state == 'INCOMPLETE' else u' ',
             f.name) for f in item.files()
            if not f.deleted and f.state != 'INCOMPLETE' or args.all
        ]
        folders = [(u' ' * timestamp_width, u'', u'D' if f.deleted else u' ',
                    unicode(f.name)) for f in item.folders()
                   if not f.deleted or args.all]
        widest_size = 0
        for f in files:
            if len(f[1]) > widest_size:
                widest_size = len(f[1])
        for item in sorted(files + folders, key=lambda t: t[3]):
            if args.all:
                print(u'%s %s %s %s' %
                      (item[0], item[1].rjust(widest_size), item[2], item[3]))
            else:
                print(u'%s %s %s' %
                      (item[0], item[1].rjust(widest_size), item[3]))
    else:
        print(' '.join([
            str(item.created),
            print_size(item.size, humanize=args.humanize), item.name
        ]))
    return True  # TODO: check return value of command
示例#32
0
def download(argv=None):
    def download_jfsfile(remote_object, tofolder=None, checksum=False):
        'Helper function to get a jfsfile and store it in a local folder, optionally checksumming it. Returns boolean'
        if tofolder is None:
            tofolder = '.'  # with no arguments, store in current dir
        total_size = remote_object.size
        if remote_object.state in (JFS.ProtoFile.STATE_CORRUPT,
                                   JFS.ProtoFile.STATE_INCOMPLETE):
            puts(
                colored.red(
                    '%s was NOT downloaded successfully - Incomplete file' %
                    remote_file.name))
            return False
        topath = os.path.join(tofolder, remote_object.name)
        with open(topath, 'wb') as fh:
            bytes_read = 0
            puts(
                colored.white('Downloading: %s, size: %s \t' %
                              (remote_object.name,
                               print_size(total_size, humanize=True))))
            with ProgressBar(expected_size=total_size) as bar:
                for chunk_num, chunk in enumerate(remote_object.stream()):
                    fh.write(chunk)
                    bytes_read += len(chunk)
                    bar.show(bytes_read)
        if checksum:
            md5_lf = JFS.calculate_md5(open(topath, 'rb'))
            md5_jf = remote_object.md5
            logging.info('%s - Checksum for downloaded file' % md5_lf)
            logging.info('%s - Checksum for server file' % md5_jf)
            if md5_lf != md5_jf:
                puts(colored.blue('%s - Checksum for downloaded file' %
                                  md5_lf))
                puts(colored.blue('%s - Checksum for server file' % md5_jf))
                puts(
                    colored.red(
                        '%s was NOT downloaded successfully - cheksum mismatch'
                        % remote_object.name))
                return False
            puts(
                colored.green(
                    '%s was downloaded successfully - checksum  matched' %
                    remote_object.name))
        return True

    if argv is None:
        argv = sys.argv[1:]
    parser = argparse.ArgumentParser(
        description='Download a file or folder from Jottacloud.')
    parser.add_argument(
        'remoteobject',
        help='The path to the file or folder that you want to download',
        type=commandline_text)
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    parser.add_argument('-c',
                        '--checksum',
                        help='Verify checksum of file after download',
                        action='store_true')
    #parser.add_argument('-r', '--resume',
    #                    help='Will not download the files again if it exist in path',
    #                    action='store_true' )
    args = parse_args_and_apply_logging_level(parser, argv)
    jfs = JFS.JFS()

    if args.remoteobject.startswith('//'):
        # break out of root_folder
        root_folder = jfs.rootpath
        item_path = posixpath.join(root_folder, args.remoteobject[2:])
    else:
        root_folder = get_root_dir(jfs).path
        item_path = posixpath.join(root_folder, args.remoteobject)

    logging.info('Root folder path: %s' % root_folder)
    logging.info('Command line path to object: %s' % args.remoteobject)
    logging.info('Jotta path to object: %s' % item_path)
    remote_object = jfs.getObject(item_path)
    if isinstance(remote_object, JFS.JFSFile):
        if download_jfsfile(remote_object, checksum=args.checksum):
            logging.info('%r downloaded successfully', remote_object.path)
            return True
        else:
            puts(colored.red('%r download failed' % remote_object.path))
            return False

    else:  #if it's not a file it has to be a folder
        incomplete_files = [
        ]  #Create an list where we can store incomplete files
        checksum_error_files = [
        ]  #Create an list where we can store checksum error files
        zero_files = []  #Create an list where we can store zero files
        long_path = [
        ]  #Create an list where we can store skipped files and folders because of long path
        puts(colored.blue("Getting index for folder: %s" % remote_object.name))
        fileTree = remote_object.filedirlist().tree  #Download the folder tree
        puts(
            colored.blue('Total number of folders to download: %d' %
                         len(fileTree)))
        topdir = os.path.dirname(item_path)
        logging.info("topdir: %r", topdir)

        #Iterate through each folder
        for folder in fileTree:
            #We need to strip the path to the folder path from account,device and mountpoint details
            logging.debug("folder: %r", folder)

            _abs_folder_path = posixpath.join(JFS.JFS_ROOT, folder[1:])
            logging.debug("absolute folder path  : %r", _abs_folder_path)
            _rel_folder_path = _abs_folder_path[len(topdir) + 1:]
            logging.info('relative folder path: %r', _rel_folder_path)

            if len(_rel_folder_path
                   ) > 250:  #Windows has a limit of 250 characters in path
                puts(
                    colored.red(
                        '%s was NOT downloaded successfully - path too long' %
                        _rel_folder_path))
                long_path.append(_rel_folder_path)
            else:
                logging.info('Entering a new folder: %s' % _rel_folder_path)
                if not os.path.exists(
                        _rel_folder_path
                ):  #Create the folder locally if it doesn't exist
                    os.makedirs(_rel_folder_path)
                for _file in fileTree[
                        folder]:  #Enter the folder and download the files within
                    logging.info("file: %r", _file)
                    #This is the absolute path to the file that is going to be downloaded
                    abs_path_to_object = posixpath.join(
                        topdir, _rel_folder_path, _file.name)
                    logging.info('Downloading the file from: %s' %
                                 abs_path_to_object)
                    if _file.state in (JFS.ProtoFile.STATE_CORRUPT,
                                       JFS.ProtoFile.STATE_INCOMPLETE):
                        #Corrupt and incomplete files will be skipped
                        puts(
                            colored.red(
                                '%s was NOT downloaded successfully - Incomplete or corrupt file'
                                % _file.name))
                        incomplete_files.append(
                            posixpath.join(_rel_folder_path, _file.name))
                        continue
                    remote_object = jfs.getObject(abs_path_to_object)
                    remote_file = remote_object
                    total_size = remote_file.size
                    if total_size == 0:  # Indicates an zero file
                        puts(
                            colored.red(
                                '%s was NOT downloaded successfully - zero file'
                                % remote_file.name))
                        zero_files.append(
                            posixpath.join(_rel_folder_path, remote_file.name))
                        continue
                    if len(
                            posixpath.join(_rel_folder_path, remote_file.name)
                    ) > 250:  #Windows has a limit of 250 characters in path
                        puts(
                            colored.red(
                                '%s was NOT downloaded successfully - path too long'
                                % remote_file.name))
                        long_path.append(
                            posixpath.join(_rel_folder_path, remote_file.name))
                        continue
                    #TODO: implement args.resume:
                    if not download_jfsfile(remote_file,
                                            tofolder=_rel_folder_path,
                                            checksum=args.checksum):
                        # download failed
                        puts(
                            colored.red("Download failed: %r" %
                                        remote_file.path))
        #Incomplete files
        if len(incomplete_files) > 0:
            with codecs.open("incomplete_files.txt", "w",
                             "utf-8") as text_file:
                for item in incomplete_files:
                    text_file.write("%s\n" % item)
        print('Incomplete files (not downloaded): %d' % len(incomplete_files))
        for _files in incomplete_files:
            logging.info("Incomplete: %r", _files)

        #Checksum error files
        if len(checksum_error_files) > 0:
            with codecs.open("checksum_error_files.txt", "w",
                             "utf-8") as text_file:
                for item in checksum_error_files:
                    text_file.write("%s\n" % item)
        print('Files with checksum error (not downloaded): %d' %
              len(checksum_error_files))
        for _files in checksum_error_files:
            logging.info("Checksum error: %r", _files)

        #zero files
        if len(zero_files) > 0:
            with codecs.open("zero_files.txt", "w", "utf-8") as text_file:
                for item in zero_files:
                    text_file.write("%s\n" % item)
        print('Files with zero size (not downloaded): %d' % len(zero_files))
        for _files in zero_files:
            logging.info("Zero sized files: %r", _files)

        #long path
        if len(long_path) > 0:
            with codecs.open("long_path.txt", "w", "utf-8") as text_file:
                for item in long_path:
                    text_file.write("%s\n" % item)
        print('Folder and files not downloaded because of path too long: %d' %
              len(long_path))
        for _files in long_path:
            logging.info("Path too long: %r", _files)
        return True
示例#33
0
def scanner(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    parser = argparse.ArgumentParser(
        description=__doc__,
        epilog=
        """The program expects to find an entry for "jottacloud.com" in your .netrc,
                                    or JOTTACLOUD_USERNAME and JOTTACLOUD_PASSWORD in the running environment.
                                    This is not an official JottaCloud project."""
    )
    parser.add_argument('-l',
                        '--loglevel',
                        help='Logging level. Default: %(default)s.',
                        choices=('debug', 'info', 'warning', 'error'),
                        default='warning')
    parser.add_argument('--errorfile',
                        type=commandline_text,
                        help='A file to write errors to',
                        default='./jottacloudclient.log')
    parser.add_argument(
        '--exclude',
        type=re.compile,
        action='append',
        help='Exclude paths matched by this pattern (can be repeated)')
    parser.add_argument('--prune-files',
                        dest='prune_files',
                        help='Delete files that does not exist locally',
                        action='store_true')
    parser.add_argument('--prune-folders',
                        dest='prune_folders',
                        help='Delete folders that does not exist locally',
                        action='store_true')
    parser.add_argument('--prune-all',
                        dest='prune_all',
                        help='Combines --prune-files  and --prune-folders',
                        action='store_true')
    parser.add_argument('--version', action='version', version=__version__)
    parser.add_argument(
        '--dry-run',
        action='store_true',
        help=
        "don't actually do any uploads or deletes, just show what would be done"
    )
    parser.add_argument('topdir',
                        type=is_dir,
                        help='Path to local dir that needs syncing')
    parser.add_argument(
        'jottapath',
        type=commandline_text,
        help=
        'The path at JottaCloud where the tree shall be synced (must exist)')
    args = parse_args_and_apply_logging_level(parser, argv)
    if args.prune_all:
        args.prune_files = True
        args.prune_folders = True

    fh = logging.FileHandler(args.errorfile)
    fh.setLevel(logging.ERROR)
    logging.getLogger('').addHandler(fh)

    jfs = JFS.JFS()

    logging.info('args: topdir %r, jottapath %r', args.topdir, args.jottapath)
    filescanner(args.topdir, args.jottapath, jfs, args.errorfile, args.exclude,
                args.dry_run, args.prune_files, args.prune_folders)