def revisions(self):
try:
revisions = self._drive.service().revisions().list(fileId=self._info.id).execute()
return revisions.get("items", [])
except Exception, e: # pragma: no cover
debug.exception(e)
return None
def update(self, path, properties, **kwargs):
"""
Updates the content and attributes of a remote file.
"""
progress_callback = kwargs.get('progress_callback')
options = kwargs.get('options', {})
info = self.stat(path)
if not info:
raise FileNotFoundError(path)
debug("Updating: %s" % repr(path))
# Merge properties
for key, val in properties.iteritems():
# Do not update the ID, always use the path obtained ID.
if key == 'id':
continue
debug(" * with: %s = %s" % (repr(key), repr(val)))
setattr(info, key, Drive.utf8(val))
with self.service() as service:
res = None
req = service.files().update(
fileId=info.id,
body=info.copy(),
setModifiedDate=options.get('setModifiedDate', False),
newRevision=True,
media_body=kwargs.get('media_body')
)
if progress_callback is None:
res = req.execute()
else:
try:
while res is None:
debug(" * uploading next chunk...")
status, res = req.next_chunk()
if status:
progress_callback(status)
elif res:
file_size = int(res['fileSize'])
progress_callback(
MediaUploadProgress(file_size, file_size)
)
except Exception, ex:
debug("Exception: %s" % str(ex))
debug.exception()
# Refresh the cache with the latest revision
self._pcache.put(path, res)
return res
def update(self, path, properties, media_body=None, progress_callback=None, options={}):
info = self.stat(path)
if not info:
debug("No such file: %s" % repr(path))
return None
debug("Updating: %s" % repr(path))
# Merge properties
for k, v in properties.iteritems():
# Do not update the ID, always use the path obtained ID.
if k == "id":
continue
debug(" * with: %s = %s" % (repr(k), repr(v)))
setattr(info, k, _Drive.utf8(v))
debug("media_body type = %s" % type(media_body))
try:
req = (
self.service()
.files()
.update(
fileId=info.id,
body=info.dict(),
setModifiedDate=options.get("setModifiedDate", False),
newRevision=True,
media_body=media_body,
)
)
if progress_callback is None:
res = req.execute()
else:
status, res = None, None
while res is None:
debug(" * uploading next chunk...")
try:
status, res = req.next_chunk()
except Exception, e:
debug("Exception: %s" % str(e))
debug.exception()
break
if status:
progress_callback(status)
elif res:
fileSize = int(res["fileSize"])
progress_callback(MediaUploadProgress(fileSize, fileSize))
# Refresh the cache with the latest revision
self._pcache.put(path, res)
return res
def update(self, path, properties, **kwargs):
"""
Updates the content and attributes of a remote file.
"""
progress_callback = kwargs.get('progress_callback')
options = kwargs.get('options', {})
info = self.stat(path)
if not info:
raise FileNotFoundError(path)
debug("Updating: %s" % repr(path))
# Merge properties
for key, val in properties.iteritems():
# Do not update the ID, always use the path obtained ID.
if key == 'id':
continue
debug(" * with: %s = %s" % (repr(key), repr(val)))
setattr(info, key, Drive.utf8(val))
with self.service() as service:
res = None
req = service.files().update(
fileId=info.id,
body=info.copy(),
setModifiedDate=options.get('setModifiedDate', False),
newRevision=True,
media_body=kwargs.get('media_body')
)
if progress_callback is None:
res = req.execute()
else:
try:
while res is None:
debug(" * uploading next chunk...")
status, res = req.next_chunk()
if status:
progress_callback(status)
elif res:
file_size = int(res['fileSize'])
progress_callback(
MediaUploadProgress(file_size, file_size)
)
except Exception, ex:
debug("Exception: %s" % str(ex))
debug.exception()
# Refresh the cache with the latest revision
self._pcache.put(path, res)
return res
def update(self,
path, properties,
media_body = None,
progress_callback = None,
options = {}
):
info = self.stat(path)
if not info:
debug("No such file: %s" % repr(path))
return None
debug("Updating: %s" % repr(path))
# Merge properties
for k, v in properties.iteritems():
# Do not update the ID, always use the path obtained ID.
if k == 'id': continue
debug(" * with: %s = %s" % (repr(k), repr(v)))
setattr(info, k, _Drive.utf8(v))
debug("mdeia_body type = %s" % type(media_body))
try:
req = self.service().files().update(
fileId = info.id,
body = info.dict(),
setModifiedDate = options.get('setModifiedDate', False),
newRevision = True,
media_body = media_body
)
if progress_callback is None:
return req.execute()
else:
status, res = None, None
while res is None:
debug(" * uploading next chunk...")
try:
status, res = req.next_chunk()
except Exception, e:
debug("Exception: %s" % str(e))
debug.exception()
break
if status:
progress_callback(status)
return res
except Exception, e:
debug("Update failed: %s" % repr(e))
debug.exception()
raise
def _md5Checksum(self, path):
try:
import hashlib
m = hashlib.md5()
with open(path, "r") as f:
m.update(f.read())
return m.hexdigest()
except Exception, e:
debug.exception()
debug("Exception: %s" % repr(e))
return None
def mkdir(self, path):
debug("path = %s" % repr(path))
self.validatepath(path)
spath = self.strippath(path)
normpath = self.normpath(spath)
debug("spath = %s" % repr(spath))
debug("normpath = %s" % repr(normpath))
try:
dirname, basename = os.path.split(normpath)
debug("dirname = %s, basename = %s" % (
repr(dirname), repr(basename)
))
if dirname in [ "/", "drive:" ]:
parentId = "root"
else:
parent = self.stat(dirname)
debug("Failed to stat directory: %s" % repr(dirname))
if not parent:
if normpath != dirname:
parent = self.mkdir(dirname)
if not parent:
debug("Failed to create parent: %s" % repr(dirname))
return None
debug("Got parent: %s" % repr(parent))
parentId = parent.id
debug("Creating directory: %s" % repr(normpath))
info = self.service().files().insert(
body = {
'title': basename,
'mimeType': MimeTypes.FOLDER,
'parents': [{ 'id': parentId }]
}
).execute()
if info:
self._clearCache(path)
self._pcache[path] = info
ent = DriveFile(path = _Drive.unicode(normpath), **info)
return ent
except Exception, e:
debug.exception()
debug("Failed to create directory: %s" % repr(e))
def walk(self, top, topdown = True, onerror = None, followlinks = False):
join = os.path.join
names = None
debug("Walking: %s" % repr(top))
try:
names = self.listdir(top)
except Exception, e:
debug.exception()
debug("Exception: %s" % repr(e))
if onerror is not None:
onerror(e)
return
def _md5Checksum(self, path):
if os.path.isdir(path):
return None
try:
import hashlib
m = hashlib.md5()
with open(path, "r") as f:
# Read the file in 1K chunks to avoid memory consumption
while True:
chunk = f.read(1024)
if not chunk: break
m.update(chunk)
return m.hexdigest()
except Exception, e:
debug.exception(e)
def _md5Checksum(self, path):
if os.path.isdir(path):
return None
try:
import hashlib
m = hashlib.md5()
with open(path, "r") as f:
# Read the file in 1K chunks to avoid memory consumption
while True:
chunk = f.read(1024)
if not chunk: break
m.update(chunk)
return m.hexdigest()
except Exception, e: # pragma: no cover
debug.exception(e)
def walk(self, top, topdown=True, onerror=None, followlinks=False):
"""
Walks the Google Drive directory structure one directory at a time
and processes all files at each directory by yielding a tuple of
the directory, list of directories and list of files.
"""
join = os.path.join
names = None
debug("Walking: %s" % repr(top))
try:
names = self.listdir(top)
except Exception as ex:
debug.exception()
debug("Exception: %s" % repr(ex))
if onerror is not None:
onerror(ex)
return
debug("Separating directories from files...")
dirs, nondirs = [], []
for name in names:
if self.isdir(join(top, name)):
dirs.append(name)
else:
nondirs.append(name)
if topdown:
yield top, dirs, nondirs
debug("Iterating directories...")
for name in dirs:
new_path = join(top, name)
for vals in self.walk(new_path, topdown, onerror, followlinks):
yield vals
debug("Yeilding on non-directories...")
if not topdown:
yield top, dirs, nondirs
def walk(self, top, topdown=True, onerror=None, followlinks=False):
"""
Walks the Google Drive directory structure one directory at a time
and processes all files at each directory by yielding a tuple of
the directory, list of directories and list of files.
"""
join = os.path.join
names = None
debug("Walking: %s" % repr(top))
try:
names = self.listdir(top)
except Exception, ex:
debug.exception()
debug("Exception: %s" % repr(ex))
if onerror is not None:
onerror(ex)
return
def walk(self, top, topdown=True, onerror=None, followlinks=False):
"""
Walks the Google Drive directory structure one directory at a time
and processes all files at each directory by yielding a tuple of
the directory, list of directories and list of files.
"""
join = os.path.join
names = None
debug("Walking: %s" % repr(top))
try:
names = self.listdir(top)
except Exception, ex:
debug.exception()
debug("Exception: %s" % repr(ex))
if onerror is not None:
onerror(ex)
return
def _md5_checksum(self, path):
"""Returns the checksum of the file"""
if os.path.isdir(path):
return None
try:
md5_gen = hashlib.new("md5")
with open(path, "r") as fd:
# Read the file in 1K chunks to avoid memory consumption
while True:
chunk = fd.read(1024)
if not chunk:
break
md5_gen.update(chunk)
return md5_gen.hexdigest()
except Exception, ex: # pragma: no cover
debug.exception(ex)
def _md5_checksum(self, path):
"""Returns the checksum of the file"""
if os.path.isdir(path):
return None
try:
md5_gen = hashlib.new("md5")
with open(path, "r") as fd:
# Read the file in 1K chunks to avoid memory consumption
while True:
chunk = fd.read(1024)
if not chunk:
break
md5_gen.update(chunk)
return md5_gen.hexdigest()
except Exception, ex: # pragma: no cover
debug.exception(ex)
def run(self):
"""
Worker method called synchronously or as part of an asynchronous
thread or subprocess.
"""
srcpath = self._src
basepath, path = os.path.split(srcpath)
if self._drive.is_drivepath(self._src):
basepath = self._drive.normpath(basepath)
debug("Source srcpath: %s" % repr(srcpath))
debug("Source basepath: %s" % repr(basepath))
debug("Source path: %s" % repr(path))
if GsyncOptions.relative:
# Supports the foo/./bar notation in rsync.
path = re.sub(r'^.*/\./', "", path)
self._sync = Sync(basepath, self._dst)
debug("Enumerating: %s" % repr(srcpath))
try:
self._walk(srcpath, self._walk_callback, self._dev)
except KeyboardInterrupt:
print("\nInterrupted")
raise
except Exception as ex:
debug.exception(ex)
print("Error: %s" % repr(ex))
finally:
verbose("sent %d bytes received %d bytes %.2f bytes/sec" %
(self._sync.total_bytes_sent,
self._sync.total_bytes_received, self._sync.rate()))
class Crawler(object):
def __init__(self, src, dst):
self._dev = None
self._src = None
self._dst = None
force_dest_file = GsyncOptions.force_dest_file
self._drive = Drive()
if self._drive.is_drivepath(src):
self._walkCallback = bind("walk", self._drive)
self._src = self._drive.normpath(src)
info = self._drive.stat(self._src)
if info and info.mimeType != MimeTypes.FOLDER:
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
else:
self._walkCallback = os_walk_wrapper
self._src = os.path.normpath(src)
st_info = os.stat(self._src)
if os.path.isfile(self._src):
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
if GsyncOptions.one_file_system:
self._dev = st_info.st_dev
if self._drive.is_drivepath(dst):
self._dst = self._drive.normpath(dst)
info = self._drive.stat(self._dst)
if info and info.mimeType == MimeTypes.FOLDER:
debug("Dest is a directory, not forcing dest file: %s" %
(repr(self._dst)))
force_dest_file = False
else:
self._dst = os.path.normpath(dst)
if os.path.isdir(self._dst):
debug("Dest is a directory, not forcing dest file: %s" %
(repr(self._dst)))
force_dest_file = False
if src[-1] == "/":
self._src += "/"
if dst[-1] == "/":
self._dst += "/"
debug("Dest has trailing slash, not forcing dest file: %s" %
(self._dst))
force_dest_file = False
# Only update if not already set.
if GsyncOptions.force_dest_file is None:
debug("force_dest_file = %s" % force_dest_file)
GsyncOptions.force_dest_file = force_dest_file
#super(Crawler, self).__init__(name = "Crawler: %s" % src)
def _devCheck(self, dev, path):
if dev is not None:
st_info = os.stat(path)
if st_info.st_dev != dev:
debug("Not on same dev: %s" % repr(path))
return False
return True
def _walk(self, path, generator, dev):
for d, dirs, files in generator(path):
debug("Walking: %s" % repr(d))
if not self._devCheck(dev, d):
debug("Not on same device: %s" % repr(d))
continue
if not GsyncOptions.force_dest_file:
if GsyncOptions.dirs or GsyncOptions.recursive:
# Sync the directory but not its contents
debug("Synchronising directory: %s" % repr(d))
self._sync(d)
else:
sys.stdout.write("skipping directory %s\n" % d)
break
for f in files:
f = os.path.join(d, f)
if not self._devCheck(dev, f):
continue
debug("Synchronising file: %s" % repr(f))
self._sync(f)
if not GsyncOptions.recursive:
break
def run(self):
srcpath = self._src
basepath, path = os.path.split(srcpath)
if self._drive.is_drivepath(self._src):
basepath = self._drive.normpath(basepath)
debug("Source srcpath: %s" % repr(srcpath))
debug("Source basepath: %s" % repr(basepath))
debug("Source path: %s" % repr(path))
if GsyncOptions.relative:
# Supports the foo/./bar notation in rsync.
path = re.sub(r'^.*/\./', "", path)
self._sync = Sync(basepath, self._dst)
debug("Enumerating: %s" % repr(srcpath))
try:
self._walk(srcpath, self._walkCallback, self._dev)
except KeyboardInterrupt, e:
print("\nInterrupted")
raise
except Exception, e:
debug.exception(e)
print("Error: %s" % repr(e))
break
if status:
progress_callback(status)
elif res:
fileSize = int(res["fileSize"])
progress_callback(MediaUploadProgress(fileSize, fileSize))
# Refresh the cache with the latest revision
self._pcache.put(path, res)
return res
except Exception, e: # pragma: no cover
debug("Update failed: %s" % repr(e))
debug.exception()
raise
def _query(self, **kwargs):
parentId = kwargs.get("parentId")
mimeType = kwargs.get("mimeType")
fileId = kwargs.get("id")
includeTrash = kwargs.get("includeTrash", False)
result = []
page_token = None
service = self.service()
query, ents = [], []
param = {}
if fileId is not None:
class Crawler(object):
"""
Crawler class that defines an instance of a crawler that is bound to
either a local or remote filesystem.
"""
def __init__(self, src, dst):
self._dev = None
self._src = None
self._dst = None
self._sync = None
force_dest_file = GsyncOptions.force_dest_file
self._drive = Drive()
if self._drive.is_drivepath(src):
self._walk_callback = bind("walk", self._drive)
self._src = self._drive.normpath(src)
info = self._drive.stat(self._src)
if info and info.mimeType != MimeTypes.FOLDER:
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
else:
self._walk_callback = os_walk_wrapper
self._src = os.path.normpath(src)
st_info = os.stat(self._src)
if os.path.isfile(self._src):
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
if GsyncOptions.one_file_system:
self._dev = st_info.st_dev
if self._drive.is_drivepath(dst):
self._dst = self._drive.normpath(dst)
info = self._drive.stat(self._dst)
if info and info.mimeType == MimeTypes.FOLDER:
debug("Dest is a directory, not forcing dest file: %s" %
(repr(self._dst)))
force_dest_file = False
else:
self._dst = os.path.normpath(dst)
if os.path.isdir(self._dst):
debug("Dest is a directory, not forcing dest file: %s" %
(repr(self._dst)))
force_dest_file = False
if src[-1] == "/":
self._src += "/"
if dst[-1] == "/":
self._dst += "/"
debug("Dest has trailing slash, not forcing dest file: %s" %
(self._dst))
force_dest_file = False
# Only update if not already set.
if GsyncOptions.force_dest_file is None:
debug("force_dest_file = %s" % force_dest_file)
GsyncOptions.force_dest_file = force_dest_file
#super(Crawler, self).__init__(name = "Crawler: %s" % src)
def _dev_check(self, device_id, path):
"""
Checks if the path provided resides on the device specified by the
device ID provided.
@param {int} device_id The device ID.
@param {String} path Path to verify.
@return {bool} True if the path resides on device with the
specified ID.
"""
if device_id is not None:
st_info = os.stat(path)
if st_info.st_dev != device_id:
debug("Not on same device: %s" % repr(path))
return False
return True
def _walk(self, path, generator, device_id):
"""
Walks the path provided, calling the generator function on the path,
which yields the subdirectories and files. It then iterates these
lists and calls the sync method '_sync'.
@param {String} path Path to walk.
@param {Function} generator Generator function to call on path.
@param {int} device_id Device ID for the path, None if device
cannot be determined.
"""
for dirpath, _, files in generator(path):
debug("Walking: %s" % repr(dirpath))
if not self._dev_check(device_id, dirpath):
debug("Not on same device: %s" % repr(dirpath))
continue
if not GsyncOptions.force_dest_file:
if GsyncOptions.dirs or GsyncOptions.recursive:
# Sync the directory but not its contents
debug("Synchronising directory: %s" % repr(dirpath))
self._sync(dirpath)
else:
sys.stdout.write("skipping directory %s\n" % dirpath)
break
for filename in files:
absfile = os.path.join(dirpath, filename)
if not self._dev_check(device_id, absfile):
continue
debug("Synchronising file: %s" % repr(absfile))
self._sync(absfile)
if not GsyncOptions.recursive:
break
def run(self):
"""
Worker method called synchronously or as part of an asynchronous
thread or subprocess.
"""
srcpath = self._src
basepath, path = os.path.split(srcpath)
if self._drive.is_drivepath(self._src):
basepath = self._drive.normpath(basepath)
debug("Source srcpath: %s" % repr(srcpath))
debug("Source basepath: %s" % repr(basepath))
debug("Source path: %s" % repr(path))
if GsyncOptions.relative:
# Supports the foo/./bar notation in rsync.
path = re.sub(r'^.*/\./', "", path)
self._sync = Sync(basepath, self._dst)
debug("Enumerating: %s" % repr(srcpath))
try:
self._walk(srcpath, self._walk_callback, self._dev)
except KeyboardInterrupt, ex:
print("\nInterrupted")
raise
except Exception, ex:
debug.exception(ex)
print("Error: %s" % repr(ex))
