def revisions(self): try: revisions = self._drive.service().revisions().list(fileId=self._info.id).execute() return revisions.get("items", []) except Exception, e: # pragma: no cover debug.exception(e) return None
def update(self, path, properties, **kwargs): """ Updates the content and attributes of a remote file. """ progress_callback = kwargs.get('progress_callback') options = kwargs.get('options', {}) info = self.stat(path) if not info: raise FileNotFoundError(path) debug("Updating: %s" % repr(path)) # Merge properties for key, val in properties.iteritems(): # Do not update the ID, always use the path obtained ID. if key == 'id': continue debug(" * with: %s = %s" % (repr(key), repr(val))) setattr(info, key, Drive.utf8(val)) with self.service() as service: res = None req = service.files().update( fileId=info.id, body=info.copy(), setModifiedDate=options.get('setModifiedDate', False), newRevision=True, media_body=kwargs.get('media_body') ) if progress_callback is None: res = req.execute() else: try: while res is None: debug(" * uploading next chunk...") status, res = req.next_chunk() if status: progress_callback(status) elif res: file_size = int(res['fileSize']) progress_callback( MediaUploadProgress(file_size, file_size) ) except Exception, ex: debug("Exception: %s" % str(ex)) debug.exception() # Refresh the cache with the latest revision self._pcache.put(path, res) return res
def update(self, path, properties, media_body=None, progress_callback=None, options={}): info = self.stat(path) if not info: debug("No such file: %s" % repr(path)) return None debug("Updating: %s" % repr(path)) # Merge properties for k, v in properties.iteritems(): # Do not update the ID, always use the path obtained ID. if k == "id": continue debug(" * with: %s = %s" % (repr(k), repr(v))) setattr(info, k, _Drive.utf8(v)) debug("media_body type = %s" % type(media_body)) try: req = ( self.service() .files() .update( fileId=info.id, body=info.dict(), setModifiedDate=options.get("setModifiedDate", False), newRevision=True, media_body=media_body, ) ) if progress_callback is None: res = req.execute() else: status, res = None, None while res is None: debug(" * uploading next chunk...") try: status, res = req.next_chunk() except Exception, e: debug("Exception: %s" % str(e)) debug.exception() break if status: progress_callback(status) elif res: fileSize = int(res["fileSize"]) progress_callback(MediaUploadProgress(fileSize, fileSize)) # Refresh the cache with the latest revision self._pcache.put(path, res) return res
def update(self, path, properties, media_body = None, progress_callback = None, options = {} ): info = self.stat(path) if not info: debug("No such file: %s" % repr(path)) return None debug("Updating: %s" % repr(path)) # Merge properties for k, v in properties.iteritems(): # Do not update the ID, always use the path obtained ID. if k == 'id': continue debug(" * with: %s = %s" % (repr(k), repr(v))) setattr(info, k, _Drive.utf8(v)) debug("mdeia_body type = %s" % type(media_body)) try: req = self.service().files().update( fileId = info.id, body = info.dict(), setModifiedDate = options.get('setModifiedDate', False), newRevision = True, media_body = media_body ) if progress_callback is None: return req.execute() else: status, res = None, None while res is None: debug(" * uploading next chunk...") try: status, res = req.next_chunk() except Exception, e: debug("Exception: %s" % str(e)) debug.exception() break if status: progress_callback(status) return res except Exception, e: debug("Update failed: %s" % repr(e)) debug.exception() raise
def _md5Checksum(self, path): try: import hashlib m = hashlib.md5() with open(path, "r") as f: m.update(f.read()) return m.hexdigest() except Exception, e: debug.exception() debug("Exception: %s" % repr(e)) return None
def mkdir(self, path): debug("path = %s" % repr(path)) self.validatepath(path) spath = self.strippath(path) normpath = self.normpath(spath) debug("spath = %s" % repr(spath)) debug("normpath = %s" % repr(normpath)) try: dirname, basename = os.path.split(normpath) debug("dirname = %s, basename = %s" % ( repr(dirname), repr(basename) )) if dirname in [ "/", "drive:" ]: parentId = "root" else: parent = self.stat(dirname) debug("Failed to stat directory: %s" % repr(dirname)) if not parent: if normpath != dirname: parent = self.mkdir(dirname) if not parent: debug("Failed to create parent: %s" % repr(dirname)) return None debug("Got parent: %s" % repr(parent)) parentId = parent.id debug("Creating directory: %s" % repr(normpath)) info = self.service().files().insert( body = { 'title': basename, 'mimeType': MimeTypes.FOLDER, 'parents': [{ 'id': parentId }] } ).execute() if info: self._clearCache(path) self._pcache[path] = info ent = DriveFile(path = _Drive.unicode(normpath), **info) return ent except Exception, e: debug.exception() debug("Failed to create directory: %s" % repr(e))
def walk(self, top, topdown = True, onerror = None, followlinks = False): join = os.path.join names = None debug("Walking: %s" % repr(top)) try: names = self.listdir(top) except Exception, e: debug.exception() debug("Exception: %s" % repr(e)) if onerror is not None: onerror(e) return
def _md5Checksum(self, path): if os.path.isdir(path): return None try: import hashlib m = hashlib.md5() with open(path, "r") as f: # Read the file in 1K chunks to avoid memory consumption while True: chunk = f.read(1024) if not chunk: break m.update(chunk) return m.hexdigest() except Exception, e: debug.exception(e)
def _md5Checksum(self, path): if os.path.isdir(path): return None try: import hashlib m = hashlib.md5() with open(path, "r") as f: # Read the file in 1K chunks to avoid memory consumption while True: chunk = f.read(1024) if not chunk: break m.update(chunk) return m.hexdigest() except Exception, e: # pragma: no cover debug.exception(e)
def walk(self, top, topdown=True, onerror=None, followlinks=False): """ Walks the Google Drive directory structure one directory at a time and processes all files at each directory by yielding a tuple of the directory, list of directories and list of files. """ join = os.path.join names = None debug("Walking: %s" % repr(top)) try: names = self.listdir(top) except Exception as ex: debug.exception() debug("Exception: %s" % repr(ex)) if onerror is not None: onerror(ex) return debug("Separating directories from files...") dirs, nondirs = [], [] for name in names: if self.isdir(join(top, name)): dirs.append(name) else: nondirs.append(name) if topdown: yield top, dirs, nondirs debug("Iterating directories...") for name in dirs: new_path = join(top, name) for vals in self.walk(new_path, topdown, onerror, followlinks): yield vals debug("Yeilding on non-directories...") if not topdown: yield top, dirs, nondirs
def walk(self, top, topdown=True, onerror=None, followlinks=False): """ Walks the Google Drive directory structure one directory at a time and processes all files at each directory by yielding a tuple of the directory, list of directories and list of files. """ join = os.path.join names = None debug("Walking: %s" % repr(top)) try: names = self.listdir(top) except Exception, ex: debug.exception() debug("Exception: %s" % repr(ex)) if onerror is not None: onerror(ex) return
def _md5_checksum(self, path): """Returns the checksum of the file""" if os.path.isdir(path): return None try: md5_gen = hashlib.new("md5") with open(path, "r") as fd: # Read the file in 1K chunks to avoid memory consumption while True: chunk = fd.read(1024) if not chunk: break md5_gen.update(chunk) return md5_gen.hexdigest() except Exception, ex: # pragma: no cover debug.exception(ex)
def run(self): """ Worker method called synchronously or as part of an asynchronous thread or subprocess. """ srcpath = self._src basepath, path = os.path.split(srcpath) if self._drive.is_drivepath(self._src): basepath = self._drive.normpath(basepath) debug("Source srcpath: %s" % repr(srcpath)) debug("Source basepath: %s" % repr(basepath)) debug("Source path: %s" % repr(path)) if GsyncOptions.relative: # Supports the foo/./bar notation in rsync. path = re.sub(r'^.*/\./', "", path) self._sync = Sync(basepath, self._dst) debug("Enumerating: %s" % repr(srcpath)) try: self._walk(srcpath, self._walk_callback, self._dev) except KeyboardInterrupt: print("\nInterrupted") raise except Exception as ex: debug.exception(ex) print("Error: %s" % repr(ex)) finally: verbose("sent %d bytes received %d bytes %.2f bytes/sec" % (self._sync.total_bytes_sent, self._sync.total_bytes_received, self._sync.rate()))
class Crawler(object): def __init__(self, src, dst): self._dev = None self._src = None self._dst = None force_dest_file = GsyncOptions.force_dest_file self._drive = Drive() if self._drive.is_drivepath(src): self._walkCallback = bind("walk", self._drive) self._src = self._drive.normpath(src) info = self._drive.stat(self._src) if info and info.mimeType != MimeTypes.FOLDER: debug("Source is not a directory, forcing dest file: %s" % (repr(self._src))) force_dest_file = True else: self._walkCallback = os_walk_wrapper self._src = os.path.normpath(src) st_info = os.stat(self._src) if os.path.isfile(self._src): debug("Source is not a directory, forcing dest file: %s" % (repr(self._src))) force_dest_file = True if GsyncOptions.one_file_system: self._dev = st_info.st_dev if self._drive.is_drivepath(dst): self._dst = self._drive.normpath(dst) info = self._drive.stat(self._dst) if info and info.mimeType == MimeTypes.FOLDER: debug("Dest is a directory, not forcing dest file: %s" % (repr(self._dst))) force_dest_file = False else: self._dst = os.path.normpath(dst) if os.path.isdir(self._dst): debug("Dest is a directory, not forcing dest file: %s" % (repr(self._dst))) force_dest_file = False if src[-1] == "/": self._src += "/" if dst[-1] == "/": self._dst += "/" debug("Dest has trailing slash, not forcing dest file: %s" % (self._dst)) force_dest_file = False # Only update if not already set. if GsyncOptions.force_dest_file is None: debug("force_dest_file = %s" % force_dest_file) GsyncOptions.force_dest_file = force_dest_file #super(Crawler, self).__init__(name = "Crawler: %s" % src) def _devCheck(self, dev, path): if dev is not None: st_info = os.stat(path) if st_info.st_dev != dev: debug("Not on same dev: %s" % repr(path)) return False return True def _walk(self, path, generator, dev): for d, dirs, files in generator(path): debug("Walking: %s" % repr(d)) if not self._devCheck(dev, d): debug("Not on same device: %s" % repr(d)) continue if not GsyncOptions.force_dest_file: if GsyncOptions.dirs or GsyncOptions.recursive: # Sync the directory but not its contents debug("Synchronising directory: %s" % repr(d)) self._sync(d) else: sys.stdout.write("skipping directory %s\n" % d) break for f in files: f = os.path.join(d, f) if not self._devCheck(dev, f): continue debug("Synchronising file: %s" % repr(f)) self._sync(f) if not GsyncOptions.recursive: break def run(self): srcpath = self._src basepath, path = os.path.split(srcpath) if self._drive.is_drivepath(self._src): basepath = self._drive.normpath(basepath) debug("Source srcpath: %s" % repr(srcpath)) debug("Source basepath: %s" % repr(basepath)) debug("Source path: %s" % repr(path)) if GsyncOptions.relative: # Supports the foo/./bar notation in rsync. path = re.sub(r'^.*/\./', "", path) self._sync = Sync(basepath, self._dst) debug("Enumerating: %s" % repr(srcpath)) try: self._walk(srcpath, self._walkCallback, self._dev) except KeyboardInterrupt, e: print("\nInterrupted") raise except Exception, e: debug.exception(e) print("Error: %s" % repr(e))
break if status: progress_callback(status) elif res: fileSize = int(res["fileSize"]) progress_callback(MediaUploadProgress(fileSize, fileSize)) # Refresh the cache with the latest revision self._pcache.put(path, res) return res except Exception, e: # pragma: no cover debug("Update failed: %s" % repr(e)) debug.exception() raise def _query(self, **kwargs): parentId = kwargs.get("parentId") mimeType = kwargs.get("mimeType") fileId = kwargs.get("id") includeTrash = kwargs.get("includeTrash", False) result = [] page_token = None service = self.service() query, ents = [], [] param = {} if fileId is not None:
class Crawler(object): """ Crawler class that defines an instance of a crawler that is bound to either a local or remote filesystem. """ def __init__(self, src, dst): self._dev = None self._src = None self._dst = None self._sync = None force_dest_file = GsyncOptions.force_dest_file self._drive = Drive() if self._drive.is_drivepath(src): self._walk_callback = bind("walk", self._drive) self._src = self._drive.normpath(src) info = self._drive.stat(self._src) if info and info.mimeType != MimeTypes.FOLDER: debug("Source is not a directory, forcing dest file: %s" % (repr(self._src))) force_dest_file = True else: self._walk_callback = os_walk_wrapper self._src = os.path.normpath(src) st_info = os.stat(self._src) if os.path.isfile(self._src): debug("Source is not a directory, forcing dest file: %s" % (repr(self._src))) force_dest_file = True if GsyncOptions.one_file_system: self._dev = st_info.st_dev if self._drive.is_drivepath(dst): self._dst = self._drive.normpath(dst) info = self._drive.stat(self._dst) if info and info.mimeType == MimeTypes.FOLDER: debug("Dest is a directory, not forcing dest file: %s" % (repr(self._dst))) force_dest_file = False else: self._dst = os.path.normpath(dst) if os.path.isdir(self._dst): debug("Dest is a directory, not forcing dest file: %s" % (repr(self._dst))) force_dest_file = False if src[-1] == "/": self._src += "/" if dst[-1] == "/": self._dst += "/" debug("Dest has trailing slash, not forcing dest file: %s" % (self._dst)) force_dest_file = False # Only update if not already set. if GsyncOptions.force_dest_file is None: debug("force_dest_file = %s" % force_dest_file) GsyncOptions.force_dest_file = force_dest_file #super(Crawler, self).__init__(name = "Crawler: %s" % src) def _dev_check(self, device_id, path): """ Checks if the path provided resides on the device specified by the device ID provided. @param {int} device_id The device ID. @param {String} path Path to verify. @return {bool} True if the path resides on device with the specified ID. """ if device_id is not None: st_info = os.stat(path) if st_info.st_dev != device_id: debug("Not on same device: %s" % repr(path)) return False return True def _walk(self, path, generator, device_id): """ Walks the path provided, calling the generator function on the path, which yields the subdirectories and files. It then iterates these lists and calls the sync method '_sync'. @param {String} path Path to walk. @param {Function} generator Generator function to call on path. @param {int} device_id Device ID for the path, None if device cannot be determined. """ for dirpath, _, files in generator(path): debug("Walking: %s" % repr(dirpath)) if not self._dev_check(device_id, dirpath): debug("Not on same device: %s" % repr(dirpath)) continue if not GsyncOptions.force_dest_file: if GsyncOptions.dirs or GsyncOptions.recursive: # Sync the directory but not its contents debug("Synchronising directory: %s" % repr(dirpath)) self._sync(dirpath) else: sys.stdout.write("skipping directory %s\n" % dirpath) break for filename in files: absfile = os.path.join(dirpath, filename) if not self._dev_check(device_id, absfile): continue debug("Synchronising file: %s" % repr(absfile)) self._sync(absfile) if not GsyncOptions.recursive: break def run(self): """ Worker method called synchronously or as part of an asynchronous thread or subprocess. """ srcpath = self._src basepath, path = os.path.split(srcpath) if self._drive.is_drivepath(self._src): basepath = self._drive.normpath(basepath) debug("Source srcpath: %s" % repr(srcpath)) debug("Source basepath: %s" % repr(basepath)) debug("Source path: %s" % repr(path)) if GsyncOptions.relative: # Supports the foo/./bar notation in rsync. path = re.sub(r'^.*/\./', "", path) self._sync = Sync(basepath, self._dst) debug("Enumerating: %s" % repr(srcpath)) try: self._walk(srcpath, self._walk_callback, self._dev) except KeyboardInterrupt, ex: print("\nInterrupted") raise except Exception, ex: debug.exception(ex) print("Error: %s" % repr(ex))