def test_open_for_read_and_read_data(self):
drive = Drive()
f = drive.open("drive://gsync_unittest/open_for_read.txt", "r")
contents = f.read()
self.assertIsNotNone(contents)
self.assertNotEqual(contents, "")
def _update_data(self, path, src):
debug("Updating remote file: %s" % repr(path))
total_bytes_written = self.bytes_written
bytes_written = 0
info = src.get_info()
def __callback(status):
bytes_written = int(status.resumable_progress)
self.bytes_written = total_bytes_written + bytes_written
progress = Progress(GsyncOptions.progress, __callback)
if GsyncOptions.dry_run:
bytes_written = info.fileSize
progress(MediaUploadProgress(bytes_written, bytes_written))
else:
progress.bytesTotal = info.fileSize
drive = Drive()
info = drive.update(
path, info, media_body=src.get_uploader(),
progress_callback=progress
)
if info is not None:
bytes_written = long(info.get('fileSize', '0'))
debug("Final file size: %d" % bytes_written)
else:
debug("Update failed")
progress.complete(bytes_written)
self.bytes_written = total_bytes_written + bytes_written
def test_revisions(self):
drive = Drive()
num_revisions = 6
info = drive.create("drive://gsync_unittest/revision_test",
{"description": "revision-0"})
self.assertEqual(info['description'], "revision-0")
for revision in range(1, num_revisions):
description = "revision-%d" % revision
info = drive.update("drive://gsync_unittest/revision_test",
{"description": description},
media_body=MediaFileUpload(
"tests/data/open_for_read.txt",
mimetype=MimeTypes.BINARY_FILE,
resumable=True))
self.assertEqual(info['description'], description)
f = drive.open("drive://gsync_unittest/revision_test", "r")
revisions = f.revisions()
self.assertEqual(len(revisions), num_revisions)
self.assertEqual(int(revisions[0]['fileSize']), 0)
self.assertNotEqual(int(revisions[-1]['fileSize']), 0)
def test_revisions(self):
drive = Drive()
num_revisions = 6
info = drive.create("drive://gsync_unittest/revision_test", {
"description": "revision-0"
})
self.assertEqual(info['description'], "revision-0")
for revision in range(1, num_revisions):
description = "revision-%d" % revision
info = drive.update("drive://gsync_unittest/revision_test", {
"description": description
},
media_body=MediaFileUpload("tests/data/open_for_read.txt",
mimetype=MimeTypes.BINARY_FILE, resumable=True
)
)
self.assertEqual(info['description'], description)
f = drive.open("drive://gsync_unittest/revision_test", "r")
revisions = f.revisions()
self.assertEqual(len(revisions), num_revisions)
self.assertEqual(int(revisions[0]['fileSize']), 0)
self.assertNotEqual(int(revisions[-1]['fileSize']), 0)
def test_strippath(self):
drive = Drive()
paths = [
"drive:",
"drive:/",
"drive://",
"drive://gsync_unittest",
"drive://gsync_unittest/",
"drive://gsync_unittest/a/b/c",
"drive://gsync_unittest/a/b/c/.",
"drive://gsync_unittest/a/b/c/..",
]
expected_paths = [
"/",
"/",
"/",
"/gsync_unittest",
"/gsync_unittest",
"/gsync_unittest/a/b/c",
"/gsync_unittest/a/b/c",
"/gsync_unittest/a/b",
]
for i in xrange(0, len(paths)):
expected = str(expected_paths[i])
actual = str(drive.strippath(paths[i]))
self.assertEqual(
expected, actual, "From %s expected %s but got %s" %
(paths[i], expected, actual))
def __init__(self, src, dst):
self._dev = None
self._src = None
self._dst = None
self._sync = None
force_dest_file = GsyncOptions.force_dest_file
self._drive = Drive()
if self._drive.is_drivepath(src):
self._walk_callback = bind("walk", self._drive)
self._src = self._drive.normpath(src)
info = self._drive.stat(self._src)
if info and info.mimeType != MimeTypes.FOLDER:
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
else:
self._walk_callback = os_walk_wrapper
self._src = os.path.normpath(src)
st_info = os.stat(self._src)
if os.path.isfile(self._src):
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
if GsyncOptions.one_file_system:
self._dev = st_info.st_dev
if self._drive.is_drivepath(dst):
self._dst = self._drive.normpath(dst)
info = self._drive.stat(self._dst)
if info and info.mimeType == MimeTypes.FOLDER:
debug("Dest is a directory, not forcing dest file: %s" %
(repr(self._dst)))
force_dest_file = False
else:
self._dst = os.path.normpath(dst)
if os.path.isdir(self._dst):
debug("Dest is a directory, not forcing dest file: %s" %
(repr(self._dst)))
force_dest_file = False
if src[-1] == "/":
self._src += "/"
if dst[-1] == "/":
self._dst += "/"
debug("Dest has trailing slash, not forcing dest file: %s" %
(self._dst))
force_dest_file = False
# Only update if not already set.
if GsyncOptions.force_dest_file is None:
debug("force_dest_file = %s" % force_dest_file)
GsyncOptions.force_dest_file = force_dest_file
def test_pathlist(self):
drive = Drive()
paths = [
"drive://",
"drive://gsync_unittest",
"drive://gsync_unittest/",
"drive://gsync_unittest/a/b/c",
"drive://gsync_unittest/a/b/c/.",
"drive://gsync_unittest/a/b/c/..",
]
expected_paths = [
["drive://"],
["drive://", "gsync_unittest"],
["drive://", "gsync_unittest"],
["drive://", "gsync_unittest", "a", "b", "c"],
["drive://", "gsync_unittest", "a", "b", "c"],
["drive://", "gsync_unittest", "a", "b"],
]
for i in xrange(0, len(paths)):
expected = str(expected_paths[i])
actual = str(drive.pathlist(paths[i]))
self.assertEqual(
expected, actual, "From %s expected %s but got %s" %
(paths[i], expected, actual))
def _update_data(self, path, src):
debug("Updating remote file: %s" % repr(path))
total_bytes_written = self.bytes_written
bytes_written = 0
info = src.get_info()
def __callback(status):
bytes_written = int(status.resumable_progress)
self.bytes_written = total_bytes_written + bytes_written
progress = Progress(GsyncOptions.progress, __callback)
if GsyncOptions.dry_run:
bytes_written = info.fileSize
progress(MediaUploadProgress(bytes_written, bytes_written))
else:
progress.bytesTotal = info.fileSize
drive = Drive()
info = drive.update(path,
info,
media_body=src.get_uploader(),
progress_callback=progress)
if info is not None:
bytes_written = int(info.get('fileSize', '0'))
debug("Final file size: %d" % bytes_written)
else:
debug("Update failed")
progress.complete(bytes_written)
self.bytes_written = total_bytes_written + bytes_written
def test_open_for_read_and_read_data(self):
drive = Drive()
f = drive.open("drive://unittest/open_for_read.txt", "r")
contents = f.read()
self.assertIsNotNone(contents)
self.assertNotEqual(contents, "")
def _updateFile(self, path, src):
debug("Updating remote file: %s" % repr(path))
totalBytesWritten = self.bytesWritten
bytesWritten = 0
info = src.getInfo()
def _callback(status):
bytesWritten = int(status.resumable_progress)
self.bytesWritten = totalBytesWritten + bytesWritten
progress = Progress(GsyncOptions.progress, _callback)
if GsyncOptions.dry_run:
bytesWritten = info.fileSize
progress(MediaUploadProgress(bytesWritten, bytesWritten))
else:
drive = Drive()
info = drive.update(path, info, src.getUploader(), progress)
if info is not None:
bytesWritten = long(info.get('fileSize', '0'))
else:
debug("Update failed")
progress.complete(bytesWritten)
self.bytesWritten = totalBytesWritten + bytesWritten
def test_Drive_pathlist(self):
drive = Drive()
paths = [
"drive://",
"drive://unittest",
"drive://unittest/",
"drive://unittest/a/b/c",
"drive://unittest/a/b/c/.",
"drive://unittest/a/b/c/..",
]
expected_paths = [
[ "drive://" ],
[ "drive://", "unittest" ],
[ "drive://", "unittest" ],
[ "drive://", "unittest", "a", "b", "c" ],
[ "drive://", "unittest", "a", "b", "c" ],
[ "drive://", "unittest", "a", "b" ],
]
for i in xrange(0, len(paths)):
expected = str(expected_paths[i])
actual = str(drive.pathlist(paths[i]))
self.assertEqual(expected, actual,
"From %s expected %s but got %s" % (
paths[i], expected, actual
)
)
def test_Drive_strippath(self):
drive = Drive()
paths = [
"drive:",
"drive:/",
"drive://",
"drive://unittest",
"drive://unittest/",
"drive://unittest/a/b/c",
"drive://unittest/a/b/c/.",
"drive://unittest/a/b/c/..",
]
expected_paths = [
"/",
"/",
"/",
"/unittest",
"/unittest",
"/unittest/a/b/c",
"/unittest/a/b/c",
"/unittest/a/b",
]
for i in xrange(0, len(paths)):
expected = str(expected_paths[i])
actual = str(drive.strippath(paths[i]))
self.assertEqual(expected, actual,
"From %s expected %s but got %s" % (
paths[i], expected, actual
)
)
def test_open_for_read_and_seek(self):
drive = Drive()
f = drive.open("drive://gsync_unittest/open_for_read.txt", "r")
self.assertNotEqual(int(f._info.fileSize), 0)
f.seek(0, os.SEEK_END)
self.assertNotEqual(f.tell(), 0)
def test_open_for_read_and_seek(self):
drive = Drive()
f = drive.open("drive://gsync_unittest/open_for_read.txt", "r")
self.assertNotEqual(int(f._info.fileSize), 0)
f.seek(0, os.SEEK_END)
self.assertNotEqual(f.tell(), 0)
def test_mkdir(self):
drive = Drive()
info = drive.mkdir("drive://gsync_unittest/test_mkdir/a/b/c/d/e/f/g")
self.assertIsNotNone(info)
self.assertEqual(info.title, "g")
drive.delete("drive://gsync_unittest/test_mkdir", skip_trash=True)
def test_mkdir(self):
drive = Drive()
info = drive.mkdir("drive://gsync_unittest/test_mkdir/a/b/c/d/e/f/g")
self.assertIsNotNone(info)
self.assertEqual(info.title, "g")
drive.delete("drive://gsync_unittest/test_mkdir", skip_trash=True)
def test_mkdir(self):
self.skipTest("slow")
drive = Drive()
info = drive.mkdir("drive://unittest/test_mkdir/a/b/c/d/e/f/g")
self.assertIsNotNone(info)
self.assertEqual(info.title, "g")
drive.rm("drive://unittest/test_mkdir", recursive=True)
def test_listdir(self):
drive = Drive()
info = drive.create("drive://gsync_unittest/a_file_to_list", {})
self.assertIsNotNone(info)
items = drive.listdir("drive://gsync_unittest/")
self.assertTrue(isinstance(items, list))
self.assertTrue("a_file_to_list" in items)
def test_mkdir(self):
self.skipTest("slow")
drive = Drive()
info = drive.mkdir("drive://unittest/test_mkdir/a/b/c/d/e/f/g")
self.assertIsNotNone(info)
self.assertEqual(info.title, "g")
drive.rm("drive://unittest/test_mkdir", recursive=True)
def test_listdir(self):
drive = Drive()
info = drive.create("drive://gsync_unittest/a_file_to_list", {})
self.assertIsNotNone(info)
items = drive.listdir("drive://gsync_unittest/")
self.assertTrue(isinstance(items, list))
self.assertTrue("a_file_to_list" in items)
def test_mimetypes(self):
drive = Drive()
drive.mkdir("drive://gsync_unittest/mimetype_test_dir")
f = drive.open("drive://gsync_unittest/mimetype_test_dir", "r")
self.assertEqual(f.mimetype(), MimeTypes.FOLDER)
f.mimetype(MimeTypes.BINARY_FILE)
self.assertEqual(f.mimetype(), MimeTypes.BINARY_FILE)
def _createFile(self, path, src):
debug("Creating remote file: %s" % repr(path))
if GsyncOptions.dry_run: return
drive = Drive()
info = drive.create(path, src.getInfo())
if info is None:
debug("Creation failed")
def _createFile(self, path, src):
debug("Creating remote file: %s" % repr(path))
if GsyncOptions.dry_run: return
drive = Drive()
info = drive.create(path, src.getInfo())
if info is None:
debug("Creation failed")
def test_mimetypes(self):
drive = Drive()
drive.mkdir("drive://gsync_unittest/mimetype_test_dir")
f = drive.open("drive://gsync_unittest/mimetype_test_dir", "r")
self.assertEqual(f.mimetype(), MimeTypes.FOLDER)
f.mimetype(MimeTypes.BINARY_FILE)
self.assertEqual(f.mimetype(), MimeTypes.BINARY_FILE)
def test_stat(self):
drive = Drive()
info = drive.stat("drive://")
self.assertIsNotNone(info)
self.assertEqual("root", info.id)
info = drive.stat("drive://gsync_unittest/")
self.assertIsNotNone(info)
self.assertIsNotNone(info.id)
self.assertEqual(info.title, "gsync_unittest")
def test_stat(self):
drive = Drive()
info = drive.stat("drive://")
self.assertIsNotNone(info)
self.assertEqual("root", info.id)
info = drive.stat("drive://unittest/")
self.assertIsNotNone(info)
self.assertIsNotNone(info.id)
self.assertEqual(info.title, "unittest")
def create(path):
"""Creates a new SyncFile instance"""
drive = Drive()
if drive.is_drivepath(path):
filepath = drive.normpath(path)
from libgsync.sync.file.remote import SyncFileRemote
return SyncFileRemote(filepath)
filepath = os.path.normpath(path)
from libgsync.sync.file.local import SyncFileLocal
return SyncFileLocal(filepath)
def test_isdir(self):
drive = Drive()
self.assertFalse(drive.isdir("drive://gsync_unittest/is_a_dir"))
drive.mkdir("drive://gsync_unittest/is_a_dir")
self.assertTrue(drive.isdir("drive://gsync_unittest/is_a_dir"))
drive.create("drive://gsync_unittest/not_a_dir", {})
self.assertFalse(drive.isdir("drive://gsync_unittest/not_a_dir"))
def setup_drive_data(testcase):
# Ironic, using Gsync to setup the tests, but if it fails the tests
# will fail anyway, so we will be okay.
assert os.path.exists("tests/data")
drive = Drive()
drive.delete("drive://gsync_unittest/", skip_trash=True)
drive.mkdir("drive://gsync_unittest/")
drive.create("drive://gsync_unittest/open_for_read.txt", {})
drive.update("drive://gsync_unittest/open_for_read.txt", {},
media_body=MediaFileUpload("tests/data/open_for_read.txt",
mimetype=MimeTypes.BINARY_FILE,
resumable=True))
def test_close(self):
drive = Drive()
f = drive.open("drive://gsync_unittest/open_for_read.txt", "r")
contents = f.read()
self.assertNotEqual(contents, None)
f.close()
self.assertTrue(f.closed)
try:
f.seek(0)
self.assertEqual("Expected IOError for seek on closed file", None)
except IOError:
pass
def get_uploader(self, path = None):
info = self.get_info(path)
if info is None:
raise Exception("Could not obtain file information: %s" % path)
path = self.get_path(path)
drive = Drive()
debug("Opening remote file for reading: %s" % repr(path))
fd = drive.open(path, "r")
if fd is None:
raise Exception("Open failed: %s" % path)
return MediaIoBaseUpload(fd, info.mimeType, resumable=True)
def get_uploader(self, path=None):
info = self.get_info(path)
if info is None:
raise Exception("Could not obtain file information: %s" % path)
path = self.get_path(path)
drive = Drive()
debug("Opening remote file for reading: %s" % repr(path))
fd = drive.open(path, "r")
if fd is None:
raise Exception("Open failed: %s" % path)
return MediaIoBaseUpload(fd, info.mimeType, resumable=True)
def test_close(self):
drive = Drive()
f = drive.open("drive://gsync_unittest/open_for_read.txt", "r")
contents = f.read()
self.assertNotEqual(contents, None)
f.close()
self.assertTrue(f.closed)
try:
f.seek(0)
self.assertEqual("Expected IOError for seek on closed file", None)
except IOError:
pass
def test_write(self):
drive = Drive()
try:
drive.open("drive://gsync_unittest/open_for_read.txt", "w")
self.assertEqual("Expected IOError for unsupported mode", None)
except IOError:
pass
f = drive.open("drive://gsync_unittest/open_for_read.txt", "r")
try:
f.write("Some data")
self.assertEqual("Expected IOError for writing to readable file",
None)
except IOError:
pass
def test_create(self):
drive = Drive()
info = drive.create("drive://gsync_unittest/create_test", {
"title": "Will be overwritten",
"description": "Will be kept"
})
self.assertEqual(info['title'], "create_test")
self.assertEqual(info['description'], "Will be kept")
info2 = drive.create("drive://gsync_unittest/create_test", {
"description": "This file will replace the first one"
})
self.assertNotEqual(info['id'], info2['id'])
self.assertEqual(info2['title'], "create_test")
self.assertEqual(info2['description'], "This file will replace the first one")
def __init__(self, src, dst):
self._dev = None
self._src = None
self._dst = None
self._drive = Drive()
if self._drive.is_drivepath(src):
self._walkCallback = bind("walk", self._drive)
self._src = self._drive.normpath(src)
else:
self._walkCallback = os.walk
self._src = os.path.normpath(src)
st_info = os.stat(self._src)
if GsyncOptions.one_file_system:
self._dev = st_info.st_dev
if self._drive.is_drivepath(dst):
self._dst = self._drive.normpath(dst)
else:
self._dst = os.path.normpath(dst)
if src[-1] == "/": self._src += "/"
if dst[-1] == "/": self._dst += "/"
def _updateStats(self, path, src, mode, uid, gid, mtime, atime):
debug("Updating remote file stats: %s" % repr(path))
if GsyncOptions.dry_run: return
info = self.getInfo(path)
if not info: return
st_info = list(tuple(info.statInfo))
if mode is not None:
st_info[0] = mode
if uid is not None:
st_info[4] = uid
if gid is not None:
st_info[5] = gid
if atime is not None:
st_info[7] = atime
info._setStatInfo(st_info)
mtime_utc = datetime.datetime.utcfromtimestamp(mtime).isoformat()
Drive().update(path, properties = {
'description': info.description,
'modifiedDate': mtime_utc,
}, options = {
'setModifiedDate': GsyncOptions.times
})
def create(path):
"""Creates a new SyncFile instance"""
drive = Drive()
if drive.is_drivepath(path):
filepath = drive.normpath(path)
from libgsync.sync.file.remote import SyncFileRemote
return SyncFileRemote(filepath)
else:
filepath = os.path.normpath(path)
from libgsync.sync.file.local import SyncFileLocal
return SyncFileLocal(filepath)
def __init__(self, src, dst):
self._dev = None
self._src = None
self._dst = None
self._sync = None
force_dest_file = GsyncOptions.force_dest_file
self._drive = Drive()
if self._drive.is_drivepath(src):
self._walk_callback = bind("walk", self._drive)
self._src = self._drive.normpath(src)
info = self._drive.stat(self._src)
if info and info.mimeType != MimeTypes.FOLDER:
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
else:
self._walk_callback = os_walk_wrapper
self._src = os.path.normpath(src)
st_info = os.stat(self._src)
if os.path.isfile(self._src):
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
if GsyncOptions.one_file_system:
self._dev = st_info.st_dev
if self._drive.is_drivepath(dst):
self._dst = self._drive.normpath(dst)
info = self._drive.stat(self._dst)
if info and info.mimeType == MimeTypes.FOLDER:
debug("Dest is a directory, not forcing dest file: %s" % (repr(
self._dst)))
force_dest_file = False
else:
self._dst = os.path.normpath(dst)
if os.path.isdir(self._dst):
debug("Dest is a directory, not forcing dest file: %s" % (repr(
self._dst)))
force_dest_file = False
if src[-1] == "/":
self._src += "/"
if dst[-1] == "/":
self._dst += "/"
debug("Dest has trailing slash, not forcing dest file: %s" %
(self._dst))
force_dest_file = False
# Only update if not already set.
if GsyncOptions.force_dest_file is None:
debug("force_dest_file = %s" % force_dest_file)
GsyncOptions.force_dest_file = force_dest_file
def _update_attrs(self, path, src, attrs):
debug("Updating remote file attrs: %s" % repr(path))
if GsyncOptions.dry_run:
return
info = self.get_info(path)
if not info:
return
st_info = list(tuple(info.statInfo))
if attrs.mode is not None:
st_info[0] = attrs.mode
if attrs.uid is not None:
st_info[4] = attrs.uid
if attrs.gid is not None:
st_info[5] = attrs.gid
if attrs.atime is not None:
st_info[7] = attrs.atime
info.set_stat_info(st_info)
mtime_utc = datetime.datetime.utcfromtimestamp(
# attrs.mtime).isoformat()
# attrs.mtime).replace(tzinfo=tzutc()).isoformat()
attrs.mtime).replace(
tzinfo=tzutc()).strftime("%Y-%m-%dT%H:%M:%S.%f%z")
Drive().update(path,
properties={
'description': info.description,
'modifiedDate': mtime_utc,
},
options={'setModifiedDate': GsyncOptions.times})
def test_isdir(self):
drive = Drive()
self.assertFalse(drive.isdir("drive://gsync_unittest/is_a_dir"))
drive.mkdir("drive://gsync_unittest/is_a_dir")
self.assertTrue(drive.isdir("drive://gsync_unittest/is_a_dir"))
drive.create("drive://gsync_unittest/not_a_dir", {})
self.assertFalse(drive.isdir("drive://gsync_unittest/not_a_dir"))
def test_write(self):
drive = Drive()
try:
drive.open("drive://gsync_unittest/open_for_read.txt", "w")
self.assertEqual("Expected IOError for unsupported mode", None)
except IOError:
pass
f = drive.open("drive://gsync_unittest/open_for_read.txt", "r")
try:
f.write("Some data")
self.assertEqual(
"Expected IOError for writing to readable file", None
)
except IOError:
pass
def test_create(self):
drive = Drive()
info = drive.create("drive://gsync_unittest/create_test", {
"title": "Will be overwritten",
"description": "Will be kept"
})
self.assertEqual(info['title'], "create_test")
self.assertEqual(info['description'], "Will be kept")
info2 = drive.create(
"drive://gsync_unittest/create_test",
{"description": "This file will replace the first one"})
self.assertNotEqual(info['id'], info2['id'])
self.assertEqual(info2['title'], "create_test")
self.assertEqual(info2['description'],
"This file will replace the first one")
def setup_drive_data(testcase):
# Ironic, using Gsync to setup the tests, but if it fails the tests
# will fail anyway, so we will be okay.
assert os.path.exists("tests/data")
drive = Drive()
drive.delete("drive://gsync_unittest/", skip_trash=True)
drive.mkdir("drive://gsync_unittest/")
drive.create("drive://gsync_unittest/open_for_read.txt", {})
drive.update("drive://gsync_unittest/open_for_read.txt", {},
media_body=MediaFileUpload("tests/data/open_for_read.txt",
mimetype=MimeTypes.BINARY_FILE, resumable=True
)
)
def get_info(self, path = None):
path = self.get_path(path)
debug("Fetching remote file metadata: %s" % repr(path))
# The Drive() instance is self caching.
drive = Drive()
info = drive.stat(path)
if info is None:
debug("File not found: %s" % repr(path))
return None
info = SyncFileInfo(**info)
debug("Remote file = %s" % repr(info), 3)
debug("Remote mtime: %s" % repr(info.modifiedDate))
return info
def get_info(self, path=None):
path = self.get_path(path)
debug("Fetching remote file metadata: %s" % repr(path))
# The Drive() instance is self caching.
drive = Drive()
info = drive.stat(path)
if info is None:
debug("File not found: %s" % repr(path))
return None
info = SyncFileInfo(**info)
debug("Remote file = %s" % repr(info), 3)
debug("Remote mtime: %s" % repr(info.modifiedDate))
return info
def create(path):
debug("SyncFileFactory.create(%s)" % repr(path))
drive = Drive()
if drive.is_drivepath(path):
filepath = drive.normpath(path)
debug("Creating SyncFileRemote(%s)" % repr(filepath))
from libgsync.sync.file.remote import SyncFileRemote
return SyncFileRemote(filepath)
else:
filepath = os.path.normpath(path)
debug("Creating SyncFileLocal(%s)" % repr(filepath))
from libgsync.sync.file.local import SyncFileLocal
return SyncFileLocal(filepath)
def create(path):
debug("SyncFileFactory.create(%s)" % repr(path))
drive = Drive()
if drive.is_drivepath(path):
filepath = drive.normpath(path)
debug("Creating SyncFileRemote(%s)" % repr(filepath))
from libgsync.sync.file.remote import SyncFileRemote
return SyncFileRemote(filepath)
else:
filepath = os.path.normpath(path)
debug("Creating SyncFileLocal(%s)" % repr(filepath))
from libgsync.sync.file.local import SyncFileLocal
return SyncFileLocal(filepath)
def __requires_auth(testcase, *args, **kwargs):
config_dir = Drive()._get_config_dir()
credentials = os.path.join(config_dir, "credentials")
if os.path.exists(credentials):
return func(testcase, *args, **kwargs)
if inspect.isclass(testcase):
return None
testcase.skipTest("Authentication not established")
return None
def test_update_with_progress(self):
drive = Drive()
info = drive.create("drive://gsync_unittest/update_test",
{"description": "Old description"})
self.assertEqual(info['title'], "update_test")
def progress_callback(status):
progress_callback.called = True
progress_callback.called = False
info = drive.update("drive://gsync_unittest/update_test",
{"description": "New description"},
media_body=MediaFileUpload(
"tests/data/open_for_read.txt",
mimetype=MimeTypes.BINARY_FILE,
resumable=True),
progress_callback=progress_callback)
self.assertEqual(info['description'], "New description")
self.assertTrue(int(info['fileSize']) > 0)
self.assertTrue(progress_callback.called)
def test_update_with_progress(self):
drive = Drive()
info = drive.create("drive://gsync_unittest/update_test", {
"description": "Old description"
})
self.assertEqual(info['title'], "update_test")
def progress_callback(status):
progress_callback.called = True
progress_callback.called = False
info = drive.update("drive://gsync_unittest/update_test", {
"description": "New description"
},
media_body=MediaFileUpload("tests/data/open_for_read.txt",
mimetype=MimeTypes.BINARY_FILE, resumable=True
),
progress_callback=progress_callback
)
self.assertEqual(info['description'], "New description")
self.assertTrue(int(info['fileSize']) > 0)
self.assertTrue(progress_callback.called)
def strippath(path):
"""Strips path of the 'drive://' prefix using the Drive() method"""
return Drive().strippath(path)
class Crawler(object):
"""
Crawler class that defines an instance of a crawler that is bound to
either a local or remote filesystem.
"""
def __init__(self, src, dst):
self._dev = None
self._src = None
self._dst = None
self._sync = None
force_dest_file = GsyncOptions.force_dest_file
self._drive = Drive()
if self._drive.is_drivepath(src):
self._walk_callback = bind("walk", self._drive)
self._src = self._drive.normpath(src)
info = self._drive.stat(self._src)
if info and info.mimeType != MimeTypes.FOLDER:
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
else:
self._walk_callback = os_walk_wrapper
self._src = os.path.normpath(src)
st_info = os.stat(self._src)
if os.path.isfile(self._src):
debug("Source is not a directory, forcing dest file: %s" %
(repr(self._src)))
force_dest_file = True
if GsyncOptions.one_file_system:
self._dev = st_info.st_dev
if self._drive.is_drivepath(dst):
self._dst = self._drive.normpath(dst)
info = self._drive.stat(self._dst)
if info and info.mimeType == MimeTypes.FOLDER:
debug("Dest is a directory, not forcing dest file: %s" % (repr(
self._dst)))
force_dest_file = False
else:
self._dst = os.path.normpath(dst)
if os.path.isdir(self._dst):
debug("Dest is a directory, not forcing dest file: %s" % (repr(
self._dst)))
force_dest_file = False
if src[-1] == "/":
self._src += "/"
if dst[-1] == "/":
self._dst += "/"
debug("Dest has trailing slash, not forcing dest file: %s" %
(self._dst))
force_dest_file = False
# Only update if not already set.
if GsyncOptions.force_dest_file is None:
debug("force_dest_file = %s" % force_dest_file)
GsyncOptions.force_dest_file = force_dest_file
#super(Crawler, self).__init__(name = "Crawler: %s" % src)
def _dev_check(self, device_id, path):
"""
Checks if the path provided resides on the device specified by the
device ID provided.
@param {int} device_id The device ID.
@param {String} path Path to verify.
@return {bool} True if the path resides on device with the
specified ID.
"""
if device_id is not None:
st_info = os.stat(path)
if st_info.st_dev != device_id:
debug("Not on same device: %s" % repr(path))
return False
return True
def _walk(self, path, generator, device_id):
"""
Walks the path provided, calling the generator function on the path,
which yields the subdirectories and files. It then iterates these
lists and calls the sync method '_sync'.
@param {String} path Path to walk.
@param {Function} generator Generator function to call on path.
@param {int} device_id Device ID for the path, None if device
cannot be determined.
"""
for dirpath, _, files in generator(path):
debug("Walking: %s" % repr(dirpath))
if not self._dev_check(device_id, dirpath):
debug("Not on same device: %s" % repr(dirpath))
continue
if not GsyncOptions.force_dest_file:
if GsyncOptions.dirs or GsyncOptions.recursive:
# Sync the directory but not its contents
debug("Synchronising directory: %s" % repr(dirpath))
self._sync(dirpath)
else:
sys.stdout.write("skipping directory %s\n" % dirpath)
break
for filename in files:
absfile = os.path.join(dirpath, filename)
if not self._dev_check(device_id, absfile):
continue
debug("Synchronising file: %s" % repr(absfile))
self._sync(absfile)
if not GsyncOptions.recursive:
break
def run(self):
"""
Worker method called synchronously or as part of an asynchronous
thread or subprocess.
"""
srcpath = self._src
basepath, path = os.path.split(srcpath)
if self._drive.is_drivepath(self._src):
basepath = self._drive.normpath(basepath)
debug("Source srcpath: %s" % repr(srcpath))
debug("Source basepath: %s" % repr(basepath))
debug("Source path: %s" % repr(path))
if GsyncOptions.relative:
# Supports the foo/./bar notation in rsync.
path = re.sub(r'^.*/\./', "", path)
self._sync = Sync(basepath, self._dst)
debug("Enumerating: %s" % repr(srcpath))
try:
self._walk(srcpath, self._walk_callback, self._dev)
except KeyboardInterrupt, ex:
print("\nInterrupted")
raise
except Exception, ex:
debug.exception(ex)
print("Error: %s" % repr(ex))
def _create_dir(self, path, src = None):
debug("Creating remote directory: %s" % repr(path))
if not GsyncOptions.dry_run:
drive = Drive()
drive.mkdir(path)
def test_open_for_read_and_seek(self):
drive = Drive()
f = drive.open("drive://unittest/open_for_read.txt", "r")
f.seek(0, os.SEEK_END)
self.assertNotEqual(f.tell(), 0)
def normpath(self, path):
return Drive().normpath(path)
def test_open_for_read(self):
drive = Drive()
f = drive.open("drive://unittest/open_for_read.txt", "r")
self.assertIsNotNone(f)
def _create_dir(self, path, src=None):
debug("Creating remote directory: %s" % repr(path))
if not GsyncOptions.dry_run:
drive = Drive()
drive.mkdir(path)
class Crawler(object):
def __init__(self, src, dst):
self._dev = None
self._src = None
self._dst = None
self._drive = Drive()
if self._drive.is_drivepath(src):
self._walkCallback = bind("walk", self._drive)
self._src = self._drive.normpath(src)
else:
self._walkCallback = os.walk
self._src = os.path.normpath(src)
st_info = os.stat(self._src)
if GsyncOptions.one_file_system:
self._dev = st_info.st_dev
if self._drive.is_drivepath(dst):
self._dst = self._drive.normpath(dst)
else:
self._dst = os.path.normpath(dst)
if src[-1] == "/": self._src += "/"
if dst[-1] == "/": self._dst += "/"
#super(Crawler, self).__init__(name = "Crawler: %s" % src)
def _devCheck(self, dev, path):
if dev is not None:
st_info = os.stat(path)
if st_info.st_dev != dev:
debug("Not on same dev: %s" % path)
return False
return True
def _walk(self, path, generator, dev):
for d, dirs, files in generator(path):
debug("Walking: %s" % d)
if not self._devCheck(dev, d):
debug("Not on same device: %s" % d)
continue
if GsyncOptions.dirs or GsyncOptions.recursive:
# Sync the directory but not its contents
debug("Synchronising directory: %s" % d)
self._sync(d)
else:
sys.stdout.write("skipping directory %s\n" % d)
break
for f in files:
f = os.path.join(d, f)
if not self._devCheck(dev, f):
continue
debug("Synchronising file: %s" % f)
self._sync(f)
if not GsyncOptions.recursive:
break
def run(self):
srcpath = self._src
basepath, path = os.path.split(srcpath)
if self._drive.is_drivepath(self._src):
basepath = self._drive.normpath(basepath)
debug("Source srcpath: %s" % srcpath)
debug("Source basepath: %s" % basepath)
debug("Source path: %s" % path)
if GsyncOptions.relative:
# Supports the foo/./bar notation in rsync.
path = re.sub(r'^.*/\./', "", path)
self._sync = Sync(basepath, self._dst)
debug("Enumerating: %s" % srcpath)
try:
self._walk(srcpath, self._walkCallback, self._dev)
except KeyboardInterrupt, e:
print("\nInterrupted")
raise
except Exception, e:
debug.exception(e)
print("Error: %s" % str(e))
def test_open_for_read(self):
drive = Drive()
f = drive.open("drive://gsync_unittest/open_for_read.txt", "r")
self.assertIsNotNone(f)
