Example #1
0
 def __init__(self):
     "Class constructor."
     config = DriveConfig()
     pidfile = config.getPidFile()
     loglevel = config.getLogLevel()
     logfile = config.getLogFile()
     super(DriveDaemon, self).__init__(pidfile, loglevel, logfile)
def md5(argv):
    """Print the MD5 checksums of the local and remote copies of the specified remote file path.
gdrive md5 <path>

prints the local and remote MD5 checksums of the specified path.

"""
    path = None
    if len(argv) == 0:
        return usage()
    else:
        path = argv[0]
    config = DriveConfig()
    rhash = session.getRemoteFileChecksum(path)
    lpath = config.getLocalPath(path)
    lhash = session.getLocalFileChecksum(lpath)
    print "Local: path=%s md5=%s Remote: path=%s md5=%s" % (lpath, lhash, path, rhash)
    def __init__(self, verbose=False, debug=False, logger=None):
        "Class constructor."
        self._debug = debug
        self._verbose = verbose
        self._config = DriveConfig(verbose, debug, logger)

        self._token = None  ## OAuth 2,0 token object.
        self._client = None  ## Google Docs API client object.

        self._metadata = {}  ## Metadata dict.
        self._metadata[
            "changestamp"] = 0  ## Stores the last changestamp, if any.
        self._metadata["map"] = {}
        self._metadata["map"]["bypath"] = DirectoryTree(
        )  ## Maps paths to resource IDs.
        self._metadata["map"]["byid"] = {}  ## Maps resource IDs to paths.

        self._folder_count = 0
        self._file_count = 0
        self._bar = None

        self._authorise()
        if self._token == None:
            # TODO: throw exception.
            sys.exit("Error: failed to authorise with service!")
        self._setup()
        if self._client == None:
            # TODO: throw exception.
            sys.exit("Error: failed to create Docs client!")

        # Load cached metadata, if any.
        loaded = self._load()

        if not loaded:
            # Initialise metadata.
            self._walk()

        # Save metadata to cache.
        self._save()

        # Ensure local storage tree is available.
        self._config.checkLocalRoot()
Example #4
0
    def __init__(self, verbose=False, debug=False, logger=None):
        "Class constructor."
        self._debug = debug
        self._verbose = verbose
        self._config = DriveConfig(verbose, debug, logger)

        self._token = None      ## OAuth 2,0 token object.
        self._client = None     ## Google Docs API client object.

        self._metadata = {}                                 ## Metadata dict.
        self._metadata["changestamp"] = 0                   ## Stores the last changestamp, if any.
        self._metadata["map"] = {}
        self._metadata["map"]["bypath"] = DirectoryTree()   ## Maps paths to resource IDs.
        self._metadata["map"]["byid"] = {}                  ## Maps resource IDs to paths.

        self._folder_count = 0
        self._file_count = 0
        self._bar = None

        self._authorise()
        if self._token == None:
            # TODO: throw exception.
            sys.exit("Error: failed to authorise with service!")
        self._setup()
        if self._client == None:
            # TODO: throw exception.
            sys.exit("Error: failed to create Docs client!")

        # Load cached metadata, if any.
        loaded = self._load()

        if not loaded:
            # Initialise metadata.
            self._walk()

        # Save metadata to cache.
        self._save()

        # Ensure local storage tree is available.
        self._config.checkLocalRoot()
class Session(object):
    def __init__(self, verbose=False, debug=False, logger=None):
        "Class constructor."
        self._debug = debug
        self._verbose = verbose
        self._config = DriveConfig(verbose, debug, logger)

        self._token = None  ## OAuth 2,0 token object.
        self._client = None  ## Google Docs API client object.

        self._metadata = {}  ## Metadata dict.
        self._metadata[
            "changestamp"] = 0  ## Stores the last changestamp, if any.
        self._metadata["map"] = {}
        self._metadata["map"]["bypath"] = DirectoryTree(
        )  ## Maps paths to resource IDs.
        self._metadata["map"]["byid"] = {}  ## Maps resource IDs to paths.

        self._folder_count = 0
        self._file_count = 0
        self._bar = None

        self._authorise()
        if self._token == None:
            # TODO: throw exception.
            sys.exit("Error: failed to authorise with service!")
        self._setup()
        if self._client == None:
            # TODO: throw exception.
            sys.exit("Error: failed to create Docs client!")

        # Load cached metadata, if any.
        loaded = self._load()

        if not loaded:
            # Initialise metadata.
            self._walk()

        # Save metadata to cache.
        self._save()

        # Ensure local storage tree is available.
        self._config.checkLocalRoot()

    def reset(self):
        "Reset local cached metadata."
        self._metadata = {}
        self._metadata["changestamp"] = 0
        self._metadata["map"] = {}
        self._metadata["map"]["bypath"] = DirectoryTree()
        self._metadata["map"]["byid"] = {}
        self._walk()
        self._save()
        self._folder_count = 0
        self._file_count = 0
        self._num_folders = 0
        self._num_files = 0
        self._bar = None

    def _authorise(self):
        "Perform OAuth 2.0 authorisation."
        saved_auth = False

        # Try to read saved auth blob.
        tokenfile = self._config.getTokenFile()
        if os.path.exists(tokenfile):
            logging.debug("Reading token...")
            f = open(tokenfile, 'r')
            blob = f.read()
            f.close()
            if blob:
                self._token = gdata.gauth.token_from_blob(blob)
                if self._token:
                    saved_auth = True

        # Generate the OAuth 2.0 request token.
        logging.info("Generating the request token...")
        if saved_auth:
            self._token = gdata.gauth.OAuth2Token(
                client_id=self._config.CLIENT_ID,
                client_secret=self._config.CLIENT_SECRET,
                scope=" ".join(self._config.SCOPES),
                user_agent=self._config.USER_AGENT,
                refresh_token=self._token.refresh_token)
        else:
            self._token = gdata.gauth.OAuth2Token(
                client_id=self._config.CLIENT_ID,
                client_secret=self._config.CLIENT_SECRET,
                scope=" ".join(self._config.SCOPES),
                user_agent=self._config.USER_AGENT)

            # Authorise the OAuth 2.0 request token.
            print 'Visit the following URL in your browser to authorise this app:'
            print str(
                self._token.generate_authorize_url(
                    redirect_url=self._config.REDIRECT_URI))
            print 'After agreeing to authorise the app, copy the verification code from the browser.'
            access_code = raw_input('Please enter the verification code: ')

            # Get the OAuth 2.0 Access Token.
            self._token.get_access_token(access_code)

        # Save the refresh token.
        if self._token.refresh_token and not saved_auth:
            logging.debug("Saving token...")
            f = open(tokenfile, 'w')
            blob = gdata.gauth.token_to_blob(self._token)
            f.write(blob)
            f.close()

    def _setup(self):
        "Setup Google Docs session."
        # Create the Google Documents List API client.
        logging.info("Creating the Docs client...")
        self._client = gdata.docs.client.DocsClient(
            source=self._config.APP_NAME)
        #client.ssl = True  # Force HTTPS use.
        #client.http_client.debug = True  # Turn on HTTP debugging.

        # Authorise the client.
        logging.info("Authorising the Docs client API...")
        self._client = self._token.authorize(self._client)

    # Wrapper for gdata.docs.client.GetAllResources.
    def _getAllResources(self, uri):
        "Get all resources, with exponential backoff and retry."
        for n in range(0, 5):
            try:
                items = self._client.GetAllResources(uri=uri)
                return items
            except:
                time.sleep((2**n) + (random.randint(0, 1000) / 1000))
        logging.fatal(
            "An error occurred contacting the Google servers, the request never succeeded, aborting."
        )
        return None

    # Wrapper for gdata.docs.client.GetResourceById.
    def _getResourceById(self, res_id, show_root=True):
        "Get resource by ID, with exponential backoff and retry."
        for n in range(0, 5):
            try:
                resource = self._client.GetResourceById(res_id,
                                                        show_root=show_root)
                return resource
            except:
                time.sleep((2**n) + (random.randint(0, 1000) / 1000))
        logging.fatal(
            "An error occurred contacting the Google servers, the request never succeeded, aborting."
        )
        return None

    # Wrapper for gdata.docs.client.GetResourceBySelfLink.
    def _getResourceBySelfLink(self, href, show_root=True):
        "Get resource by self link, with exponential backoff and retry."
        for n in range(0, 5):
            try:
                resource = self._client.GetResourceBySelfLink(
                    href, show_root=show_root)
                return resource
            except:
                time.sleep((2**n) + (random.randint(0, 1000) / 1000))
        logging.fatal(
            "An error occurred contacting the Google servers, the request never succeeded, aborting."
        )
        return None

    # Wrapper for gdata.docs.client.GetChanges.
    def _getChanges(self, changestamp=None, max_results=100, show_root=True):
        "Get a change feed, with exponential backoff and retry."
        for n in range(0, 5):
            try:
                feed = self._client.GetChanges(changestamp, max_results,
                                               show_root)
                return feed
            except:
                time.sleep((2**n) + (random.randint(0, 1000) / 1000))
        logging.fatal(
            "An error occurred contacting the Google servers, the request never succeeded, aborting."
        )
        return None

    # Wrapper for gdata.docs.client.GetNext.
    def _getNext(self, feed):
        "Get next chink of entries from feed, with exponential backoff and retry."
        for n in range(0, 5):
            try:
                feed = self._client.GetNext(feed)
                return feed
            except:
                time.sleep((2**n) + (random.randint(0, 1000) / 1000))
        logging.fatal(
            "An error occurred contacting the Google servers, the request never succeeded, aborting."
        )
        return None

    # Wrapper for gdata.docs.client.GetRevisions.
    def _getRevisions(self, resource):
        "Get a revisions feed, with exponential backoff and retry."
        for n in range(0, 5):
            try:
                revisions = self._client.GetRevisions(resource)
                return revisions
            except gdata.client.Unauthorized:
                logging.warn("Unauthorised error on resource")
                return None
            except:
                time.sleep((2**n) + (random.randint(0, 1000) / 1000))
        logging.fatal(
            "An error occurred contacting the Google servers, the request never succeeded, aborting."
        )
        return None

    def getMetadata(self):
        "Return Google Docs user metadata."
        metadata = self._client.GetMetadata()
        metadict = {
            'quota': {
                'total': metadata.quota_bytes_total.text,
                'used': metadata.quota_bytes_used.text,
                'trashed': metadata.quota_bytes_used_in_trash.text
            },
            'import': [
                "%s to %s" % (input_format.source, input_format.target)
                for input_format in metadata.import_formats
            ],
            'export': [
                "%s to %s" % (export_format.source, export_format.target)
                for export_format in metadata.export_formats
            ],
            'features': [feature.name.text for feature in metadata.features]
        }
        metadict["upload_sizes"] = {}
        for upload_size in metadata.max_upload_sizes:
            metadict["upload_sizes"][upload_size.kind] = upload_size.text
        for child in metadata.children:
            if child.tag == "largestChangestamp":
                metadict["changestamp"] = int(child.attributes["value"])
                break
        return metadict

    def _printresource(self, resource):
        "Print debugging information about a resource."
        logging.debug("Resource: id=%s type=%s updated=%s name=%s" %
                      (resource.id.text, resource.content.type,
                       resource.updated.text, resource.title.text))

    def _resourceToUri(self, resource):
        "Get the URI for a resource."
        return resource.content.src

    def _resourceIdToPath(self, resource_id):
        "Get the path for a resource ID."
        try:
            path = self._metadata["map"]["byid"][resource_id]
        except KeyError:
            path = None
        return path

    def _pathToUri(self, path):
        "Get the URI for a path."
        if path == '/':
            uri = self._config.ROOT_FEED_URI
        else:
            if path in self._metadata["map"]["bypath"]:
                uri = self._metadata["map"]["bypath"][path]["uri"]
            else:
                # TODO: try to handle this better.
                logging.error("Path \"%s\" is unknown!" % path)
                raise KeyError
        return uri

    def _pathToResourceId(self, path):
        "Get the resource ID for a path."
        root = self._config.getLocalRoot()
        if path.startswith(root):
            path = path[len(root):]
        if path == '/':
            res_id = "folder:root"
        else:
            if path in self._metadata["map"]["bypath"]:
                res_id = self._metadata["map"]["bypath"][path]["resource_id"]
            else:
                # TODO: try to handle this better.
                logging.error("Path \"%s\" is unknown!" % path)
                raise KeyError
        return res_id

    def _readFolder(self, path):
        "Read the contents of a folder."
        logging.debug("Reading folder \"%s\"" % path)
        uri = self._pathToUri(path)
        items = self._getAllResources(uri)
        folders = []
        files = []
        for entry in items:
            itempath = os.path.join(path, entry.title.text)
            itemid = entry.resource_id.text
            item = {
                "path": itempath,
                "resource_id": itemid,
                "uri": entry.content.src,
                "size": entry.quota_bytes_used.text
            }
            item["shared"] = "false"
            if entry.get_resource_type() == 'folder':
                item["type"] = "folder"
                folders.append(itempath)
            else:
                files.append(itempath)
                item["type"] = "file"
                logging.debug("Getting metadata for path %s" % itempath)
                metadata = self._getResourceMetadata(entry)
                if metadata:
                    item.update(metadata)
                else:
                    logging.warn(
                        "No metadata found for path %s, assuming shared resource"
                        % itempath)
                    item["shared"] = "true"
            self._metadata["map"]["bypath"].add(itempath, item)
            self._metadata["map"]["byid"][itemid] = itempath
        folders.sort()
        files.sort()
        return folders, files

    def readFolder(self, path):
        "Get the list of items in the specified folder."
        return self._readFolder(path)

    def readRoot(self):
        "Get the list of items in the root folder."
        return self._readFolder('/')

    def _walk(self, root='/'):
        "Walk the server-side tree, populating the local maps."
        folders, files = self._readFolder(root)
        for folder in folders:
            self._walk(root=folder)

    def _load(self):
        "Load metadata from local file, if it exists."
        metafile = self._config.getMetadataFile()
        if os.path.exists(metafile):
            logging.debug("Reading cached metadata...")
            f = open(metafile, 'rb')
            self._metadata = pickle.load(f)
            f.close()
            return True
        return False

    def _save(self):
        "Save metadata to local file."
        metafile = self._config.getMetadataFile()
        logging.debug("Saving metadata...")
        f = open(metafile, 'wb')
        pickle.dump(self._metadata, f)
        f.close()

    def isFolder(self, path):
        "Return true if the specified path is a folder."
        if path in self._metadata["map"]["bypath"]:
            if self._metadata["map"]["bypath"][path]["type"] == "folder":
                return True
            else:
                return False
        elif path == '/':
            return True
        else:
            # TODO: Handle this better. Raise an exception?
            return False
            #sys.exit("Error: path \"%s\" is unknown!" % path)

    def isFile(self, path):
        "Return true if the specified path is a file."
        return not self.isFolder(path)

    def _getResourceMetadata(self, resource):
        metadata = {}
        revisions = self._getRevisions(resource)
        if revisions == None or len(revisions.entry) == 0:
            logging.warn("No revisions found for resource!")
            return None
        revision = revisions.entry[0]
        metadata["author"] = ""
        metadata["email"] = ""
        if len(revision.author) > 0:
            metadata["author-name"] = revision.author[0].name.text
            metadata["author-email"] = revision.author[0].email.text
        metadata["updated"] = revision.updated.text
        for child in revision.children:
            if child.tag == "edited":
                metadata["edited"] = child.text
            if child.tag == "md5Checksum":
                metadata["md5checksum"] = child.text
        return metadata

    def getRemoteFileAuthor(self, path):
        "Return the author of the specified remote path, if it is a file."
        author = {}
        if not self.isFolder(path):
            if path.startswith(self._config.getLocalRoot()):
                path = self._config.getRemotePath(path)
            try:
                author["name"] = self._metadata["map"]["bypath"][path][
                    "author-name"]
                author["email"] = self._metadata["map"]["bypath"][path][
                    "author-email"]
            except KeyError:
                author["name"] = None
                author["email"] = None
        return author

    def getLocalFileDate(self, path):
        "Return the last modified date of the specified local path, if it is a file."
        date = None
        if os.path.exists(path) and not os.path.isdir(path):
            size = os.path.getmtime(path)
        return date

    def getRemoteFileDate(self, path):
        "Return the last modified date of the specified remote path, if it is a file."
        date = None
        if not self.isFolder(path):
            if path.startswith(self._config.getLocalRoot()):
                path = self._config.getRemotePath(path)
            try:
                date = self._metadata["map"]["bypath"][path]["updated"]
            except KeyError:
                pass
        return date

    def getFileDate(self, path):
        "Return the last modified date of the specified local or remote path, if it is a file."
        date = None
        if os.path.exists(path):
            date = self.getLocalFileDate(path)
        else:
            date = self.getRemoteFileDate(path)
        return date

    def getLocalFileSize(self, path):
        "Return the size in bytes of the specified local path, if it is a file."
        size = 0
        if os.path.exists(path) and not os.path.isdir(path):
            size = os.path.getsize(path)
        return size

    def getRemoteFileSize(self, path):
        "Return the size in bytes of the specified remote path, if it is a file."
        size = 0
        if not self.isFolder(path):
            if path.startswith(self._config.getLocalRoot()):
                path = self._config.getRemotePath(path)
            try:
                size = int(self._metadata["map"]["bypath"][path]["size"])
            except KeyError:
                pass
        return size

    def getFileSize(self, path):
        "Return the size in bytes of the specified local or remote path, if it is a file."
        size = 0
        if os.path.exists(path):
            size = self.getLocalFileSize(path)
        else:
            size = self.getRemoteFileSize(path)
        return size

    def getLocalFileChecksum(self, path):
        "Return the MD5 checksum of the specified local path, if it is a file."
        checksum = None
        logging.debug("Getting MD5 for %s..." % path)
        if os.path.exists(path) and not os.path.isdir(path):
            f = open(path, 'rb')
            m = hashlib.md5()
            while True:
                data = f.read(64 * 1024)
                if not data:
                    break
                m.update(data)
            checksum = m.hexdigest()
        return checksum

    def getRemoteFileChecksum(self, path):
        "Return the MD5 checksum of the specified remote path, if it is a file."
        checksum = None
        logging.debug("Getting MD5 for %s..." % path)
        if not self.isFolder(path):
            if path.startswith(self._config.getLocalRoot()):
                path = self._config.getRemotePath(path)
            if path not in self._metadata["map"][
                    "bypath"] or "md5checksum" not in self._metadata["map"][
                        "bypath"][path]:
                shared = None
                try:
                    shared = self._metadata["map"]["bypath"][path]["shared"]
                except KeyError:
                    # This will really only happen if the metadata is from an old version of the app.
                    pass
                if shared:
                    return None
                parent = '/' + '/'.join(path.split('/')[:-2])
                self._readFolder(parent)
            try:
                checksum = self._metadata["map"]["bypath"][path]["md5checksum"]
            except KeyError:
                logging.error("Path \"%s\" is not recognised!" % path)
        return checksum

    def getFileChecksum(self, path):
        "Return the MD5 checksum of the specified local or remote path, if it is a file."
        checksum = None
        if os.path.exists(path):
            checksum = self.getLocalFileChecksum(path)
        else:
            checksum = self.getRemoteFileChecksum(path)
        return checksum

    def _getLargestChangestamp(self):
        "Returns the largest changestamp."
        metadict = self.getMetadata()
        logging.debug("Max changestamp: %d" % metadict["changestamp"])
        return metadict["changestamp"]

    def _getChangeList(self, changestamp=0):
        "Get a list of resource IDs that have changed since the specified changestamp."
        changes = []
        resource_ids = []
        if changestamp == 0:
            logging.debug("Getting all changes...")
            feed = self._getChanges(max_results=self._config.MAX_RESULTS,
                                    show_root=True)
        else:
            logging.debug("Getting changes since changestamp=%s..." %
                          changestamp)
            feed = self._getChanges(changestamp=str(changestamp),
                                    max_results=self._config.MAX_RESULTS,
                                    show_root=True)
        if feed:
            changes.extend(feed.entry)
        while feed and len(feed.entry) == self._config.MAX_RESULTS:
            feed = self._getNext(feed)
            changes.extend(feed.entry)
        if len(changes) > 0:
            # Save a changestamp of one beyond the last.
            self._metadata["changestamp"] = int(
                changes[-1].changestamp.value) + 1
            logging.debug("Got %d changes, last changestamp is %d" %
                          (len(changes), self._metadata["changestamp"]))
            for change in changes:
                resource_ids.append(change.resource_id.text)
        else:
            logging.debug("No changes found")
        return resource_ids

    def update(self, path='/', download=False, interactive=True):
        "Update the local tree at the specified path to match the server."
        logging.debug("Updating %s ..." % path)
        localpath = self._config.getLocalPath(path)
        if path == '/' or not os.path.exists(localpath):
            logging.debug("Local copy does not exist, fetching...")
            self.download(path,
                          localpath,
                          overwrite=True,
                          interactive=interactive)
        else:
            # Request change feed from the last changestamp.
            # If no stored changestamp, then start at the beginning.
            if self._metadata["changestamp"] == 0:
                logging.debug(
                    "Stored changestamp is zero, walking the tree...")
                self._metadata["changestamp"] = self._getLargestChangestamp(
                ) + 1
                self._walk(root=path)
                if download:
                    self.download(path,
                                  localpath,
                                  overwrite=True,
                                  interactive=interactive)
            # Now check for changes again, since before we walked.
            resource_ids = self._getChangeList(self._metadata["changestamp"])
            if len(resource_ids) > 0:
                # Iterate over the changes, downloading each resource.
                for res_id in resource_ids:
                    res_path = self._resourceIdToPath(res_id)
                    if res_path == None:
                        logging.debug("No local path for resource ID %s" %
                                      res_id)
                        # The resource is not in our cache.
                        resource = self._getResourceById(res_id,
                                                         show_root=True)
                        if not resource:
                            # TODO: This should never fail.
                            logging.error("Failed to get resource \"%s\"" %
                                          res_id)
                            break
                        self._printresource(resource)
                        # Repeatedly get the parent until we find one in our cache, or else reach the root,
                        # which should always exist. If it has no parent, and is not in root, then it must
                        # be shared.
                        # TODO: support shared resources somehow.
                        parents = resource.InCollections()
                        for parent in parents:
                            logging.debug("parent: %s" % parent.href)
                            if parent.href == self._config.ROOT_FOLDER_HREF:
                                logging.debug("Parent is root folder")
                                top_path = '/'
                            else:
                                parent_resource = self._getResourceBySelfLink(
                                    parent.href, show_root=True)
                                parent_resid = parent_resource.resource_id.text
                                logging.debug("Parent resource ID %s" %
                                              parent_resid)
                                parent_resids = [parent_resid]
                                while parent_resid not in self._metadata[
                                        "map"]["byid"]:
                                    logging.debug(
                                        "Parent resource ID %s not in cache" %
                                        parent_resid)
                                    parent = parent.InCollections()
                                    parent_resource = self._getResourceBySelfLink(
                                        parent.href, show_root=True)
                                    parent_resid = parent_resource.resource_id.text
                                    parent_resids.insert(0, parent_resid)
                                top_path = self._resourceIdToPath(parent_resid)
                                logging.debug(
                                    "Found parent path %s in cache for resource ID %s"
                                    % (top_path, parent_resid))
                            self._walk(top_path)
                            # Download the top_path subtree here.
                            if download:
                                self.download(
                                    top_path,
                                    self._config.getLocalPath(top_path),
                                    overwrite=True,
                                    interactive=interactive)
                        res_path = self._resourceIdToPath(res_id)
                        if res_path == None:
                            logging.warn(
                                "No parent path found, must be a shared resource, skipping..."
                            )
                            continue
                    # Check if resource path is in the path specified.
                    if res_path.startswith(path):
                        logging.debug("Get resource %s (%s)" %
                                      (res_id, res_path))
                        if download:
                            self.download(res_path,
                                          self._config.getLocalPath(res_path),
                                          overwrite=True,
                                          interactive=interactive)
                    else:
                        logging.debug(
                            "Ignoring change to path %s, not in target path %s"
                            % (res_path, path))
        self._save()

    def getNumResources(self, path=None):
        "Returns the total number of resources (files, folders) in the specified path, and all subtrees."
        return len(self._metadata["map"]["bypath"].keys(path))

    def getNumRemoteFolders(self, path=None):
        "Returns the total number of folders in the specified remote path, and all subtrees."
        count = 0
        if path == '/':
            values = self._metadata["map"]["bypath"].itervalues()
        else:
            values = self._metadata["map"]["bypath"].itervalues(path)
        for value in values:
            if value["type"] == "folder":
                count += 1
        return count

    def getNumRemoteFiles(self, path=None):
        "Returns the total number of files in the specified remote path, and all subtrees."
        count = 0
        if path == '/':
            values = self._metadata["map"]["bypath"].itervalues()
        else:
            values = self._metadata["map"]["bypath"].itervalues(path)
        for value in values:
            if value["type"] != "folder":
                count += 1
        return count

    def getNumLocalFolders(self, path):
        "Returns the total number of folders in the specified local path, and all subtrees."
        count = 0
        for root, dirs, files in os.walk(path):
            count += len(dirs)
        return count

    def getNumLocalFiles(self, path):
        "Returns the total number of files in the specified local path, and all subtrees."
        count = 0
        for root, dirs, files in os.walk(path):
            count += len(files)
        return count

    def _download(self, path, localpath, overwrite=False):
        "Download a file."
        entry = None
        if path != '/':
            entry = self._getResourceById(self._pathToResourceId(path))
            if not entry:
                logging.error("Failed to download path \"%s\"" % path)
                return False
        if self.isFolder(path):
            if not self._config.checkLocalFolder(localpath, overwrite):
                logging.error("Cannot overwrite local path \"%s\", exiting!" %
                              localpath)
                return
            logging.info("Downloading folder %s (%d of %d)..." %
                         (localpath, self._folder_count, self._num_folders))
            (folders, files) = self._readFolder(path)
            for fname in files:
                lpath = os.path.join(localpath, os.path.basename(fname))
                self._download(fname, lpath, overwrite)
                self._file_count += 1
            for folder in folders:
                lpath = os.path.join(localpath, os.path.basename(folder))
                self._download(folder, lpath, overwrite)
            self._folder_count += 1
        else:
            logging.info("Downloading file %s (%d bytes) (%d of %d)..." %
                         (localpath, self.getRemoteFileSize(path),
                          self._file_count, self._num_files))
            if self._bar:
                self._bar.render(self._file_count * 100 / self._num_files,
                                 localpath)
            if not self._config.checkLocalFile(localpath, overwrite):
                return False
            self._client.DownloadResource(entry, localpath)
            os.chmod(localpath,
                     stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
        return True

    def download(self,
                 path,
                 localpath=None,
                 overwrite=False,
                 interactive=False):
        "Download a file or a folder tree."
        self._folder_count = 1
        self._num_folders = self.getNumRemoteFolders(path)
        self._file_count = 1
        self._num_files = self.getNumRemoteFiles(path)
        if interactive:
            if self._num_folders + self._num_files > 2:
                self._bar = progressbar.ProgressBar(width=80)
        if localpath is None:
            localpath = self._config.getLocalPath(path)
            logging.debug("Using local path %s" % localpath)
        for exclude in self._config.getExcludes():
            if path == '/' + exclude:
                logging.debug("Skipping folder on exclude list")
                return
        self._download(path, localpath, overwrite)
        self._folder_count = 0
        self._file_count = 0

    def _upload(self, localpath, path):
        "Upload a file."
        logging.error("Upload is not yet implemented!")

    def upload(self, localpath, path=None, interactive=False):
        "Upload a file or a folder tree."
        if path is None:
            if localpath.startswith(self._config.getLocalRoot()):
                path = localpath[len(self._config.getLocalRoot()):]
            else:
                path = '/'
        self._folder_count = 1
        self._num_folders = self.getNumLocalFolders(path)
        self._file_count = 1
        self._num_files = self.getNumLocalFiles(path)
        if interactive:
            if self._num_folders + self._num_files > 2:
                self._bar = progressbar.ProgressBar(width=80)
        self._upload(localpath, path)
        self._folder_count = 0
        self._file_count = 0

    def getInfo(self):
        "Return general information."
        userdata = self.getMetadata()
        userdata["Total resources"] = self.getNumResources()
        return userdata

    def dump(self):
        "Dump metadata."
        pprint.pprint(self._metadata, indent=2)

    def filestatus(self, path, interactive=False):
        "Get the status of a file."
        res_id = self._pathToResourceId(path)
        resource = self._getResourceById(res_id, show_root=True)
        if not resource:
            logging.error("Failed to get resource \"%s\"" % res_id)
        return self._getResourceMetadata(resource)
Example #6
0
class Session(object):

    def __init__(self, verbose=False, debug=False, logger=None):
        "Class constructor."
        self._debug = debug
        self._verbose = verbose
        self._config = DriveConfig(verbose, debug, logger)

        self._token = None      ## OAuth 2,0 token object.
        self._client = None     ## Google Docs API client object.

        self._metadata = {}                                 ## Metadata dict.
        self._metadata["changestamp"] = 0                   ## Stores the last changestamp, if any.
        self._metadata["map"] = {}
        self._metadata["map"]["bypath"] = DirectoryTree()   ## Maps paths to resource IDs.
        self._metadata["map"]["byid"] = {}                  ## Maps resource IDs to paths.

        self._folder_count = 0
        self._file_count = 0
        self._bar = None

        self._authorise()
        if self._token == None:
            # TODO: throw exception.
            sys.exit("Error: failed to authorise with service!")
        self._setup()
        if self._client == None:
            # TODO: throw exception.
            sys.exit("Error: failed to create Docs client!")

        # Load cached metadata, if any.
        loaded = self._load()

        if not loaded:
            # Initialise metadata.
            self._walk()

        # Save metadata to cache.
        self._save()

        # Ensure local storage tree is available.
        self._config.checkLocalRoot()

    def reset(self):
        "Reset local cached metadata."
        self._metadata = {}
        self._metadata["changestamp"] = 0
        self._metadata["map"] = {}
        self._metadata["map"]["bypath"] = DirectoryTree()
        self._metadata["map"]["byid"] = {}
        self._walk()
        self._save()
        self._folder_count = 0
        self._file_count = 0
        self._num_folders = 0
        self._num_files = 0
        self._bar = None

    def _authorise(self):
        "Perform OAuth 2.0 authorisation."
        saved_auth = False

        # Try to read saved auth blob.
        tokenfile = self._config.getTokenFile()
        if os.path.exists(tokenfile):
            logging.debug("Reading token...")
            f = open(tokenfile, 'r')
            blob = f.read()
            f.close()
            if blob:
                self._token = gdata.gauth.token_from_blob(blob)
                if self._token:
                    saved_auth = True

        # Generate the OAuth 2.0 request token.
        logging.info("Generating the request token...")
        if saved_auth:
            self._token = gdata.gauth.OAuth2Token(client_id=self._config.CLIENT_ID,
                                                  client_secret=self._config.CLIENT_SECRET,
                                                  scope=" ".join(self._config.SCOPES),
                                                  user_agent=self._config.USER_AGENT,
                                                  refresh_token=self._token.refresh_token)
        else:
            self._token = gdata.gauth.OAuth2Token(client_id=self._config.CLIENT_ID,
                                                  client_secret=self._config.CLIENT_SECRET,
                                                  scope=" ".join(self._config.SCOPES),
                                                  user_agent=self._config.USER_AGENT)

            # Authorise the OAuth 2.0 request token.
            print 'Visit the following URL in your browser to authorise this app:'
            print str(self._token.generate_authorize_url(redirect_url=self._config.REDIRECT_URI))
            print 'After agreeing to authorise the app, copy the verification code from the browser.'
            access_code = raw_input('Please enter the verification code: ')

            # Get the OAuth 2.0 Access Token.
            self._token.get_access_token(access_code)

        # Save the refresh token.
        if self._token.refresh_token and not saved_auth:
            logging.debug("Saving token...")
            f = open(tokenfile, 'w')
            blob = gdata.gauth.token_to_blob(self._token)
            f.write(blob)
            f.close()

    def _setup(self):
        "Setup Google Docs session."
        # Create the Google Documents List API client.
        logging.info("Creating the Docs client...")
        self._client = gdata.docs.client.DocsClient(source=self._config.APP_NAME)
        #client.ssl = True  # Force HTTPS use.
        #client.http_client.debug = True  # Turn on HTTP debugging.

        # Authorise the client.
        logging.info("Authorising the Docs client API...")
        self._client = self._token.authorize(self._client)

    # Wrapper for gdata.docs.client.GetAllResources.
    def _getAllResources(self, uri):
        "Get all resources, with exponential backoff and retry." 
        for n in range(0, 5):
            try:
                items = self._client.GetAllResources(uri=uri)
                return items
            except:
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
        logging.fatal("An error occurred contacting the Google servers, the request never succeeded, aborting.")
        return None
    
    # Wrapper for gdata.docs.client.GetResourceById.
    def _getResourceById(self, res_id, show_root=True):
        "Get resource by ID, with exponential backoff and retry." 
        for n in range(0, 5):
            try:
                resource = self._client.GetResourceById(res_id, show_root=show_root)
                return resource
            except:
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
        logging.fatal("An error occurred contacting the Google servers, the request never succeeded, aborting.")
        return None
    
    # Wrapper for gdata.docs.client.GetResourceBySelfLink.
    def _getResourceBySelfLink(self, href, show_root=True):
        "Get resource by self link, with exponential backoff and retry." 
        for n in range(0, 5):
            try:
                resource = self._client.GetResourceBySelfLink(href, show_root=show_root)
                return resource
            except:
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
        logging.fatal("An error occurred contacting the Google servers, the request never succeeded, aborting.")
        return None
    
    # Wrapper for gdata.docs.client.GetChanges.
    def _getChanges(self, changestamp=None, max_results=100, show_root=True):
        "Get a change feed, with exponential backoff and retry." 
        for n in range(0, 5):
            try:
                feed = self._client.GetChanges(changestamp, max_results, show_root)
                return feed
            except:
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
        logging.fatal("An error occurred contacting the Google servers, the request never succeeded, aborting.")
        return None

    # Wrapper for gdata.docs.client.GetNext.
    def _getNext(self, feed):
        "Get next chink of entries from feed, with exponential backoff and retry." 
        for n in range(0, 5):
            try:
                feed = self._client.GetNext(feed)
                return feed
            except:
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
        logging.fatal("An error occurred contacting the Google servers, the request never succeeded, aborting.")
        return None

    # Wrapper for gdata.docs.client.GetRevisions.
    def _getRevisions(self, resource):
        "Get a revisions feed, with exponential backoff and retry." 
        for n in range(0, 5):
            try:
                revisions = self._client.GetRevisions(resource)
                return revisions
            except gdata.client.Unauthorized:
                logging.warn("Unauthorised error on resource")
                return None
            except:
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
        logging.fatal("An error occurred contacting the Google servers, the request never succeeded, aborting.")
        return None

    def getMetadata(self):
        "Return Google Docs user metadata."
        metadata = self._client.GetMetadata()
        metadict = { 'quota': { 'total':   metadata.quota_bytes_total.text,
                                'used':    metadata.quota_bytes_used.text,
                                'trashed': metadata.quota_bytes_used_in_trash.text },
                     'import': [ "%s to %s" % (input_format.source, input_format.target) for input_format in metadata.import_formats ],
                     'export': [ "%s to %s" % (export_format.source, export_format.target) for export_format in metadata.export_formats ],
                     'features': [ feature.name.text for feature in metadata.features ]
        }
        metadict["upload_sizes"] = {}
        for upload_size in metadata.max_upload_sizes:
            metadict["upload_sizes"][upload_size.kind] = upload_size.text
        for child in metadata.children:
            if child.tag == "largestChangestamp":
                metadict["changestamp"] = int(child.attributes["value"])
                break
        return metadict

    def _printresource(self, resource):
        "Print debugging information about a resource."
        logging.debug("Resource: id=%s type=%s updated=%s name=%s" % (resource.id.text, resource.content.type, resource.updated.text, resource.title.text))

    def _resourceToUri(self, resource):
        "Get the URI for a resource."
        return resource.content.src

    def _resourceIdToPath(self, resource_id):
        "Get the path for a resource ID."
        try:
            path = self._metadata["map"]["byid"][resource_id]
        except KeyError:
            path = None
        return path

    def _pathToUri(self, path):
        "Get the URI for a path."
        if path == '/':
            uri = self._config.ROOT_FEED_URI
        else:
            if path in self._metadata["map"]["bypath"]:
                uri = self._metadata["map"]["bypath"][path]["uri"]
            else:
                # TODO: try to handle this better.
                logging.error("Path \"%s\" is unknown!" % path)
                raise KeyError
        return uri

    def _pathToResourceId(self, path):
        "Get the resource ID for a path."
        root = self._config.getLocalRoot()
        if path.startswith(root):
            path = path[len(root):]
        if path == '/':
            res_id = "folder:root"
        else:
            if path in self._metadata["map"]["bypath"]:
                res_id = self._metadata["map"]["bypath"][path]["resource_id"]
            else:
                # TODO: try to handle this better.
                logging.error("Path \"%s\" is unknown!" % path)
                raise KeyError
        return res_id

    def _readFolder(self, path):
        "Read the contents of a folder."
        logging.debug("Reading folder \"%s\"" % path)
        uri = self._pathToUri(path)
        items = self._getAllResources(uri)
        folders = []
        files = []
        for entry in items:
            itempath = os.path.join(path, entry.title.text)
            itemid = entry.resource_id.text
            item = { "path": itempath,
                     "resource_id": itemid,
                     "uri": entry.content.src,
                     "size": entry.quota_bytes_used.text }
            item["shared"] = "false"
            if entry.get_resource_type() == 'folder':
                item["type"] = "folder"
                folders.append(itempath)
            else:
                files.append(itempath)
                item["type"] = "file"
                logging.debug("Getting metadata for path %s" % itempath)
                metadata = self._getResourceMetadata(entry)
                if metadata:
                    item.update(metadata)
                else:
                    logging.warn("No metadata found for path %s, assuming shared resource" % itempath)
                    item["shared"] = "true"
            self._metadata["map"]["bypath"].add(itempath, item)
            self._metadata["map"]["byid"][itemid] = itempath
        folders.sort()
        files.sort()
        return folders, files

    def readFolder(self, path):
        "Get the list of items in the specified folder."
        return self._readFolder(path)

    def readRoot(self):
        "Get the list of items in the root folder."
        return self._readFolder('/')

    def _walk(self, root='/'):
        "Walk the server-side tree, populating the local maps."
        folders, files = self._readFolder(root)
        for folder in folders:
            self._walk(root=folder)

    def _load(self):
        "Load metadata from local file, if it exists."
        metafile = self._config.getMetadataFile()
        if os.path.exists(metafile):
            logging.debug("Reading cached metadata...")
            f = open(metafile, 'rb')
            self._metadata = pickle.load(f)
            f.close()
            return True
        return False

    def _save(self):
        "Save metadata to local file."
        metafile = self._config.getMetadataFile()
        logging.debug("Saving metadata...")
        f = open(metafile, 'wb')
        pickle.dump(self._metadata, f)
        f.close()

    def isFolder(self, path):
        "Return true if the specified path is a folder."
        if path in self._metadata["map"]["bypath"]:
            if self._metadata["map"]["bypath"][path]["type"] == "folder":
                return True
            else:
                return False
        elif path == '/':
            return True
        else:
            # TODO: Handle this better. Raise an exception?
            return False
            #sys.exit("Error: path \"%s\" is unknown!" % path)

    def isFile(self, path):
        "Return true if the specified path is a file."
        return not self.isFolder(path)

    def _getResourceMetadata(self, resource):
        metadata = {}
        revisions = self._getRevisions(resource)
        if revisions == None or len(revisions.entry) == 0:
            logging.warn("No revisions found for resource!")
            return None
        revision = revisions.entry[0]
        metadata["author"] = ""
        metadata["email"] = ""
        if len(revision.author) > 0:
            metadata["author-name"] = revision.author[0].name.text
            metadata["author-email"] = revision.author[0].email.text
        metadata["updated"] = revision.updated.text
        for child in revision.children:
            if child.tag == "edited":
                metadata["edited"] = child.text
            if child.tag == "md5Checksum":
                metadata["md5checksum"] = child.text
        return metadata

    def getRemoteFileAuthor(self, path):
        "Return the author of the specified remote path, if it is a file."
        author = {}
        if not self.isFolder(path):
            if path.startswith(self._config.getLocalRoot()):
                path = self._config.getRemotePath(path)
            try:
                author["name"] = self._metadata["map"]["bypath"][path]["author-name"]
                author["email"] = self._metadata["map"]["bypath"][path]["author-email"]
            except KeyError:
                author["name"] = None
                author["email"] = None
        return author

    def getLocalFileDate(self, path):
        "Return the last modified date of the specified local path, if it is a file."
        date = None
        if os.path.exists(path) and not os.path.isdir(path):
            size = os.path.getmtime(path)
        return date

    def getRemoteFileDate(self, path):
        "Return the last modified date of the specified remote path, if it is a file."
        date = None
        if not self.isFolder(path):
            if path.startswith(self._config.getLocalRoot()):
                path = self._config.getRemotePath(path)
            try:
                date = self._metadata["map"]["bypath"][path]["updated"]
            except KeyError:
                pass
        return date

    def getFileDate(self, path):
        "Return the last modified date of the specified local or remote path, if it is a file."
        date = None
        if os.path.exists(path):
            date = self.getLocalFileDate(path)
        else:
            date = self.getRemoteFileDate(path)
        return date

    def getLocalFileSize(self, path):
        "Return the size in bytes of the specified local path, if it is a file."
        size = 0
        if os.path.exists(path) and not os.path.isdir(path):
            size = os.path.getsize(path)
        return size

    def getRemoteFileSize(self, path):
        "Return the size in bytes of the specified remote path, if it is a file."
        size = 0
        if not self.isFolder(path):
            if path.startswith(self._config.getLocalRoot()):
                path = self._config.getRemotePath(path)
            try:
                size = int(self._metadata["map"]["bypath"][path]["size"])
            except KeyError:
                pass
        return size

    def getFileSize(self, path):
        "Return the size in bytes of the specified local or remote path, if it is a file."
        size = 0
        if os.path.exists(path):
            size = self.getLocalFileSize(path)
        else:
            size = self.getRemoteFileSize(path)
        return size

    def getLocalFileChecksum(self, path):
        "Return the MD5 checksum of the specified local path, if it is a file."
        checksum = None
        logging.debug("Getting MD5 for %s..." % path)
        if os.path.exists(path) and not os.path.isdir(path):
            f = open(path, 'rb')
            m = hashlib.md5()
            while True:
                data = f.read(64 * 1024)
                if not data:
                    break
                m.update(data)
            checksum = m.hexdigest()
        return checksum

    def getRemoteFileChecksum(self, path):
        "Return the MD5 checksum of the specified remote path, if it is a file."
        checksum = None
        logging.debug("Getting MD5 for %s..." % path)
        if not self.isFolder(path):
            if path.startswith(self._config.getLocalRoot()):
                path = self._config.getRemotePath(path)
            if path not in self._metadata["map"]["bypath"] or "md5checksum" not in self._metadata["map"]["bypath"][path]:
                shared = None
                try:
                    shared = self._metadata["map"]["bypath"][path]["shared"]
                except KeyError:
                    # This will really only happen if the metadata is from an old version of the app.
                    pass
                if shared:
                    return None
                parent = '/' + '/'.join(path.split('/')[:-2])
                self._readFolder(parent)
            try:
                checksum = self._metadata["map"]["bypath"][path]["md5checksum"]
            except KeyError:
                logging.error("Path \"%s\" is not recognised!" % path)
        return checksum

    def getFileChecksum(self, path):
        "Return the MD5 checksum of the specified local or remote path, if it is a file."
        checksum = None
        if os.path.exists(path):
            checksum = self.getLocalFileChecksum(path)
        else:
            checksum = self.getRemoteFileChecksum(path)
        return checksum

    def _getLargestChangestamp(self):
        "Returns the largest changestamp."
        metadict = self.getMetadata()
        logging.debug("Max changestamp: %d" % metadict["changestamp"])
        return metadict["changestamp"]

    def _getChangeList(self, changestamp=0):
        "Get a list of resource IDs that have changed since the specified changestamp."
        changes = []
        resource_ids = []
        if changestamp == 0:
            logging.debug("Getting all changes...")
            feed = self._getChanges(max_results=self._config.MAX_RESULTS, show_root=True)
        else:
            logging.debug("Getting changes since changestamp=%s..." % changestamp)
            feed = self._getChanges(changestamp=str(changestamp), max_results=self._config.MAX_RESULTS, show_root=True)
        if feed:
            changes.extend(feed.entry)
        while feed and len(feed.entry) == self._config.MAX_RESULTS:
            feed = self._getNext(feed)
            changes.extend(feed.entry)
        if len(changes) > 0:
            # Save a changestamp of one beyond the last.
            self._metadata["changestamp"] = int(changes[-1].changestamp.value) + 1
            logging.debug("Got %d changes, last changestamp is %d" % (len(changes), self._metadata["changestamp"]))
            for change in changes:
                resource_ids.append(change.resource_id.text)
        else:
            logging.debug("No changes found")
        return resource_ids

    def update(self, path='/', download=False, interactive=True):
        "Update the local tree at the specified path to match the server."
        logging.debug("Updating %s ..." % path)
        localpath = self._config.getLocalPath(path)
        if path == '/' or not os.path.exists(localpath):
            logging.debug("Local copy does not exist, fetching...")
            self.download(path, localpath, overwrite=True, interactive=interactive)
        else:
            # Request change feed from the last changestamp.
            # If no stored changestamp, then start at the beginning.
            if self._metadata["changestamp"] == 0:
                logging.debug("Stored changestamp is zero, walking the tree...")
                self._metadata["changestamp"] = self._getLargestChangestamp() + 1
                self._walk(root=path)
                if download:
                    self.download(path, localpath, overwrite=True, interactive=interactive)
            # Now check for changes again, since before we walked.
            resource_ids = self._getChangeList(self._metadata["changestamp"])
            if len(resource_ids) > 0:
                # Iterate over the changes, downloading each resource.
                for res_id in resource_ids:
                    res_path = self._resourceIdToPath(res_id)
                    if res_path == None:
                        logging.debug("No local path for resource ID %s" % res_id)
                        # The resource is not in our cache.
                        resource = self._getResourceById(res_id, show_root=True)
                        if not resource:
                            # TODO: This should never fail.
                            logging.error("Failed to get resource \"%s\"" % res_id)
                            break
                        self._printresource(resource)
                        # Repeatedly get the parent until we find one in our cache, or else reach the root,
                        # which should always exist. If it has no parent, and is not in root, then it must
                        # be shared.
                        # TODO: support shared resources somehow.
                        parents = resource.InCollections()
                        for parent in parents:
                            logging.debug("parent: %s" % parent.href)
                            if parent.href == self._config.ROOT_FOLDER_HREF:
                                logging.debug("Parent is root folder")
                                top_path = '/'
                            else:
                                parent_resource = self._getResourceBySelfLink(parent.href, show_root=True)
                                parent_resid = parent_resource.resource_id.text
                                logging.debug("Parent resource ID %s" % parent_resid)
                                parent_resids = [parent_resid]
                                while parent_resid not in self._metadata["map"]["byid"]:
                                    logging.debug("Parent resource ID %s not in cache" % parent_resid)
                                    parent = parent.InCollections()
                                    parent_resource = self._getResourceBySelfLink(parent.href, show_root=True)
                                    parent_resid = parent_resource.resource_id.text
                                    parent_resids.insert(0, parent_resid)
                                top_path = self._resourceIdToPath(parent_resid)
                                logging.debug("Found parent path %s in cache for resource ID %s" % (top_path, parent_resid))
                            self._walk(top_path)
                            # Download the top_path subtree here.
                            if download:
                                self.download(top_path, self._config.getLocalPath(top_path), overwrite=True, interactive=interactive)
                        res_path = self._resourceIdToPath(res_id)
                        if res_path == None:
                            logging.warn("No parent path found, must be a shared resource, skipping...")
                            continue
                    # Check if resource path is in the path specified.
                    if res_path.startswith(path):
                        logging.debug("Get resource %s (%s)" % (res_id, res_path))
                        if download:
                            self.download(res_path, self._config.getLocalPath(res_path), overwrite=True, interactive=interactive)
                    else:
                        logging.debug("Ignoring change to path %s, not in target path %s" % (res_path, path))
        self._save()

    def getNumResources(self, path=None):
        "Returns the total number of resources (files, folders) in the specified path, and all subtrees."
        return len(self._metadata["map"]["bypath"].keys(path))

    def getNumRemoteFolders(self, path=None):
        "Returns the total number of folders in the specified remote path, and all subtrees."
        count = 0
        if path == '/':
            values = self._metadata["map"]["bypath"].itervalues()
        else:
            values = self._metadata["map"]["bypath"].itervalues(path)
        for value in values:
            if value["type"] == "folder":
                count += 1
        return count

    def getNumRemoteFiles(self, path=None):
        "Returns the total number of files in the specified remote path, and all subtrees."
        count = 0
        if path == '/':
            values = self._metadata["map"]["bypath"].itervalues()
        else:
            values = self._metadata["map"]["bypath"].itervalues(path)
        for value in values:
            if value["type"] != "folder":
                count += 1
        return count

    def getNumLocalFolders(self, path):
        "Returns the total number of folders in the specified local path, and all subtrees."
        count = 0
        for root, dirs, files in os.walk(path):
            count += len(dirs)
        return count

    def getNumLocalFiles(self, path):
        "Returns the total number of files in the specified local path, and all subtrees."
        count = 0
        for root, dirs, files in os.walk(path):
            count += len(files)
        return count

    def _download(self, path, localpath, overwrite=False):
        "Download a file."
        entry = None
        if path != '/':
            entry = self._getResourceById(self._pathToResourceId(path))
            if not entry:
                logging.error("Failed to download path \"%s\"" % path)
                return False
        if self.isFolder(path):
            if not self._config.checkLocalFolder(localpath, overwrite):
                logging.error("Cannot overwrite local path \"%s\", exiting!" % localpath)
                return
            logging.info("Downloading folder %s (%d of %d)..." % (localpath, self._folder_count, self._num_folders))
            (folders, files) = self._readFolder(path)
            for fname in files:
                lpath = os.path.join(localpath, os.path.basename(fname))
                self._download(fname, lpath, overwrite)
                self._file_count += 1
            for folder in folders:
                lpath = os.path.join(localpath, os.path.basename(folder))
                self._download(folder, lpath, overwrite)
            self._folder_count += 1
        else:
            logging.info("Downloading file %s (%d bytes) (%d of %d)..." % (localpath, self.getRemoteFileSize(path), self._file_count, self._num_files))
            if self._bar:
                self._bar.render(self._file_count * 100 / self._num_files, localpath)
            if not self._config.checkLocalFile(localpath, overwrite):
                return False
            self._client.DownloadResource(entry, localpath)
            os.chmod(localpath, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
        return True

    def download(self, path, localpath=None, overwrite=False, interactive=False):
        "Download a file or a folder tree."
        self._folder_count = 1
        self._num_folders = self.getNumRemoteFolders(path)
        self._file_count = 1
        self._num_files = self.getNumRemoteFiles(path)
        if interactive:
            if self._num_folders + self._num_files > 2:
                self._bar = progressbar.ProgressBar(width=80)
        if localpath is None:
            localpath = self._config.getLocalPath(path)
            logging.debug("Using local path %s" % localpath)
        for exclude in self._config.getExcludes():
            if path == '/' + exclude:
                logging.debug("Skipping folder on exclude list")
                return
        self._download(path, localpath, overwrite)
        self._folder_count = 0
        self._file_count = 0

    def _upload(self, localpath, path):
        "Upload a file."

	# http://planzero.org/blog/2012/04/13/uploading_any_file_to_google_docs_with_python

	# The default root collection URI
	uri = 'https://docs.google.com/feeds/upload/create-session/default/private/full'

	# if collection path is set
	if path <> "/":
		resources = self._client.GetAllResources(uri='https://docs.google.com/feeds/default/private/full/-/folder?title=' + path + '&title-exact=true')

		uri = resources[0].get_resumable_create_media_link().href
	
	# Make sure Google doesn't try to do any conversion on the upload (e.g. convert images to documents)
	uri += '?convert=false'

	# Create an uploader and upload the file
	# Hint: it should be possible to use UploadChunk() to allow display of upload statistics for large uploads
	t1 = time.time()

	fh = open(localpath)
	import magic
	import atom
	file_type = magic.Magic(mime=True).from_file(fh.name)
	file_size = os.path.getsize(fh.name)

	logging.info("Uploading...")
	uploader = gdata.client.ResumableUploader(self._client, fh, file_type, file_size, chunk_size=1048576, desired_class=gdata.data.GDEntry)
	res = uploader.UploadFile(uri, entry=gdata.data.GDEntry(title=atom.data.Title(text=os.path.basename(fh.name))))

	logging.info('Uploaded', '{0:.2f}'.format(file_size / 1024 / 1024) + ' MiB in ' + str(round(time.time() - t1, 2)) + ' seconds')
	logging.info("Created: %s" % res)

        #logging.error("Upload is not yet implemented!")

    def upload(self, localpath, path=None, interactive=False):
        "Upload a file or a folder tree."
        if path is None:
            if localpath.startswith(self._config.getLocalRoot()):
                path = localpath[len(self._config.getLocalRoot()):]
            else:
                path = '/'
        self._folder_count = 1
        self._num_folders = self.getNumLocalFolders(path)
        self._file_count = 1
        self._num_files = self.getNumLocalFiles(path)
        if interactive:
            if self._num_folders + self._num_files > 2:
                self._bar = progressbar.ProgressBar(width=80)
        self._upload(localpath, path)
        self._folder_count = 0
        self._file_count = 0

    def getInfo(self):
        "Return general information."
        userdata = self.getMetadata()
        userdata["Total resources"] = self.getNumResources()
        return userdata

    def dump(self):
        "Dump metadata."
        pprint.pprint(self._metadata, indent=2)

    def filestatus(self, path, interactive=False):
        "Get the status of a file."
        res_id = self._pathToResourceId(path)
        resource = self._getResourceById(res_id, show_root=True)
        if not resource:
            logging.error("Failed to get resource \"%s\"" % res_id)
        return self._getResourceMetadata(resource)