def _delete_folders(self):
"""
"""
lst, tmp, ids, to_delete = self._get_all_inos(), [], [], []
stored_folders = db().getAllFolders()
if stored_folders:
for folder in stored_folders:
if folder[0] not in lst:
tmp.append({
'ino' : folder[0],
'id' : folder[1],
'parent' : folder[2],
'title' : folder[3]
})
ids.append(folder[1])
for i in tmp:
if i['parent'] not in ids or i['parent'] == "":
# these are the items that need to be deleted
entry = self.client.GetResourceById(i['id'])
self.client.DeleteResource(entry, force=True)
db().deleteById(i['id'])
# delete all the children
if ids:
db().deleteByMultiId(",".join(ids))
sleep(5) # there tends to be some lag
def _init_fetch_documents(self):
documents = self.client.GetResources(uri='/feeds/default/private/full')
from os.path import getmtime
for document in documents.entry:
sleep(1)
# check if the document has slashes in its name
doctype = re.search("(\w*(?=:))", document.resource_id.text).group(0)
# add functionality for tables later on
if doctype != "table":
data = {
'id' : document.resource_id.text,
'title' : document.title.text,
'modified' : document.updated.text,
'type' : doctype,
'location' : '',
'parent' : self.get_parent(document),
'parent_ino' : self.get_parent_ino(self.get_parent(self.resource))
}
fileExtension = self.fileType(doctype)
if document.InCollections():
fileLocation = db().getLocation(data['parent'])[3] + unicode(document.title.text).replace('/', '\\') + fileExtension
if not path.exists(fileLocation):
if self.fileType(doctype) != "":
self.client.DownloadResource(document, fileLocation, \
extra_params={'exportFormat': fileExtension[1:]})
else:
self.client.DownloadResource(document, fileLocation)
data['ino'] = stat(fileLocation).st_ino
data['local_modified'] = int(getmtime(fileLocation))
db().insertResource(data)
else:
fileLocation = SYNC_DIRECTORY + unicode(document.title.text).replace('/', '\\') + fileExtension
if self.fileType(doctype) != "":
self.client.DownloadResource(document, fileLocation, \
extra_params={'exportFormat': fileExtension[1:]})
else:
self.client.DownloadResource(document, fileLocation)
data['ino'] = stat(fileLocation).st_ino
data['local_modified'] = int(getmtime(fileLocation))
db().insertResource(data)
def get_location(self, id, resource_title):
if id=="":
return SYNC_DIRECTORY + "/%s/" % resource_title
else:
location = []
docDetail = db().getDetails("parent, title", id)
if docDetail:
if resource_title == "":
resource_title = docDetail[1]
while db().getDetails("parent, title", docDetail[0]):
docDetail = db().getDetails("parent, title", docDetail[0])
location.append(docDetail[1])
else:
return ""
return SYNC_DIRECTORY + "/".join(location[::-1]) + "/%s/" % resource_title
def get_location(self, id, resource_title, files=False):
if id=="":
return path.join(SYNC_DIRECTORY, resource_title)
else:
location = []
docDetail = db().getDetails("parent, title", id)
if docDetail:
if files:
location.append(docDetail[1])
if resource_title == "":
resource_title = docDetail[1]
while db().getDetails("parent, title", docDetail[0]):
docDetail = db().getDetails("parent, title", docDetail[0])
location.append(docDetail[1])
else:
return ""
return path.join(SYNC_DIRECTORY, "/".join(location[::-1]), resource_title)
def _init_fetch_folders(self):
# Fetch all the 'collections' from Google Documents
folders = self.client.GetResources(uri='/feeds/default/private/full/-/folder')
# Iterate through all folders
for self.resource in folders.entry:
data = {
'id' : self.resource.resource_id.text,
'title' : self.resource.title.text,
'location' : '',
'type' : 'folder',
'ino' : '',
'parent' : self.get_parent(self.resource),
'parent_ino' : self.get_parent_ino(self.get_parent(self.resource))
}
# if the document has no parent, set file path to root directory
if data['parent'] == "":
self.file_path = SYNC_DIRECTORY + self.resource.title.text + "/"
data['location'] = self.file_path
# create the folder if it doesn't exist
if not path.exists(self.file_path):
mkdir(self.file_path)
# get the new folder inode
data['ino'] = stat(self.file_path).st_ino
# insert folder reference to database
db().insertResource(data)
else:
# Insert a reference only to the database for non-root folders
db().insertResource(data)
self.file_path = ""
# Now add all the other folders
# Create the referenced folders (nested)
while db().getRest():
upd_data = {} # reset update data
for row in db().getRest():
upd_data['location'] = self.get_location(row[0], row[1])
if not path.exists(upd_data['location']):
mkdir(upd_data['location'])
upd_data['ino'] = stat(upd_data['location']).st_ino
upd_data['parent_ino'] = self.get_parent_ino(row[2])
db().updateResource(row[0], upd_data)
def _upload_files(self):
"""
"""
from gdata.docs import data as gd
from gdds.magic.magic import Magic
lst = self._get_all_files()
for file in lst:
data = {}
dbStatus = db().resourceExists(file['ino'])
if dbStatus:
print "File: %s : exists" % file['title']
# check if the location of the file has changed
localData = db().getDetailsByIno("*", file['ino'])
fileTitle = file['title'].split(".")[0]
data['location'] = fileTitle
if file['parent_ino'] != localData[8]:
if file['parent_location'] == SYNC_DIRECTORY[:-1]:
data['parent'] = ""
else:
data['parent'] = db().getKeyByIno(file['parent_ino'])
if data['parent']:
data['parent'] = data['parent'][0]
print "Modifying existing folder: %s" % file['title']
data['parent_ino'] = file['parent_ino']
data['title'] = file['title']
data['ino'] = file['ino']
if data['parent']:
data['parent'] = data['parent'][0]
else: data['parent'] = ""
resourceToMove = self.client.GetResourceById(localData[0])
try:
collection = self.client.GetResourceById(data['parent'])
except gdata.client.RequestError: collection = None
self.client.MoveResource(resourceToMove, collection=collection)
db().updateResource(localData[0], data)
elif file['title'] != localData[1]:
"""
If the file name only changes
"""
entry = self.client.GetResourceById(localData[0])
entry.title.text = fileTitle
data['title'] = fileTitle
self.client.UpdateResource(entry)
db().updateResource(localData[0], data)
else:
print "Creating a new file: %s " % file['title']
fileTitle = file['title'].split(".")[0]
mime = Magic(mime=True).from_file(file['location'])
if mime == "application/zip":
mime = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
elif mime == "application/vnd.ms-office":
mime = "application/vnd.ms-excel"
# add functionality to import Pages
# check if the folder is in the root directory
if file['parent_location'] == SYNC_DIRECTORY[:-1]:
data['parent'] = ""
data['location'] = fileTitle
data['parent_ino'] = stat(SYNC_DIRECTORY).st_ino
else:
data['parent'] = db().getKeyByIno(file['parent_ino'])
if not data['parent']:
data['parent'] = ""
else: data['parent'] = data['parent'][0]
data['location'] = self.get_location(data['parent'], file['title'], files=True)
print data['location']
data['ino'] = file['ino']
data['title'] = file['title']
data['type'] = mime
data['parent_ino'] = self.get_parent_ino(data['location'])
filePath = file['location']
newResource = gdata.docs.data.Resource(filePath, fileTitle)
media = gdata.data.MediaSource()
media.SetFileHandle(filePath, mime)
try:
collection = self.client.GetResourceById(data['parent'])
except gdata.client.RequestError: collection = None
try:
newDocument = self.client.CreateResource(newResource, media=media, collection=collection)
data['id'] = newDocument.resource_id.text
db().insertResource(data)
except gdata.client.RequestError:
print "There was a problem uploading: %s " % data['title']
pass
def _upload_folders(self):
"""
"""
from gdata.docs import data as gd
lst = self._get_all_folders()
for resource in lst:
data = {}
resource_exists = db().resourceExists(resource['ino'])
# check if the resource exists locally, and if it does check for changes
if resource_exists:
print "Resource: '%s' : already exists" % resource['title']
# check if the location of the file has changed
localData = db().getDetailsByIno("*", resource['ino'])
# if the folder location has changed
if resource['parent_ino'] != localData[8]:
print "Modifying existing folder: %s" % resource['title']
parent_key = db().getDetailsByIno("*", resource['parent_ino'])
if parent_key:
data['parent_ino'] = resource['parent_ino']
data['title'] = resource['title']
data['ino'] = resource['ino']
data['parent'] = db().getKeyByIno(resource['parent_ino'])
if data['parent']:
data['parent'] = data['parent'][0]
else: data['parent'] = ""
print "Parent key: %s" % parent_key[0]
print "Resource key: %s " % localData[0]
resourceToMove = self.client.GetResourceById(localData[0])
collection = self.client.GetResourceById(parent_key[0])
self.client.MoveResource(resourceToMove, collection=collection)
db().updateResource(localData[0], data)
else:
lst.append(resource); continue
elif resource['title'] != localData[1]:
"""
If the folder name only changes
"""
print "Folder '%s' has been renamed" % resource['title']
entry = self.client.GetResourceById(localData[0])
entry.title.text = resource['title']
data['title'] = resource['title']
self.client.UpdateResource(entry)
db().updateResource(localData[0], data)
else:
print "Creating a new folder: %s " % resource['title']
# check if the folder is in the root directory
if resource['parent_location'] == SYNC_DIRECTORY[:-1]:
data['parent'] = ""
data['location'] = ""
data['parent_ino'] = stat(SYNC_DIRECTORY).st_ino
else:
data['parent'] = db().getKeyByIno(resource['parent_ino'])
if not data['parent']:
lst.append(resource)
continue
else:
data['parent'] = data['parent'][0]
data['location'] = self.get_location(data['parent'], resource['title'])
data['ino'] = resource['ino']
data['title'] = resource['title']
data['type'] = "folder"
data['parent_ino'] = self.get_parent_ino(data['location'])
collection = self.client.GetResourceById(data['parent'])
newResource = gd.Resource(gd.COLLECTION_LABEL, resource['title'])
# create a new folder
newFolder = self.client.CreateResource(newResource, collection=collection)
data['id'] = newFolder.resource_id.text
db().insertResource(data)
def _update_folders(self):
duplicates = []
folders = self.client.GetResources(uri='/feeds/default/private/full/-/folder')
# Iterate through all folders ( All items are Resource objects )
for resource in folders.entry:
print "Checking Folder: %s" % resource.title.text
# If a folder is a reference only, it means its parent doesn't exist locally yet
reference_only, insert_reference = False, False
# get the local data for folder
local_data = db().getDetails("*", resource.resource_id.text)
# Set the folder parent if one exists
self.parent_key = self.get_parent(resource)
# data to be inserted/updated
data = {
"id" : resource.resource_id.text,
"title" : resource.title.text,
"parent" : self.parent_key,
"type" : "folder",
"location" : ""
}
# if the folder exists locally
if local_data:
print "Folder exists locally"
# check whether the folder has moved directories/changed name
parent_changed = True if self.parent_key != local_data[2] else False
title_changed = True if resource.title.text != local_data[1] else False
print parent_changed, title_changed
# if any changes were made to a file
if parent_changed or title_changed:
print "Making changes to folder"
prev_location = self.get_location(local_data[2], local_data[1])
new_parent_location = self.get_location(self.parent_key, "")
print new_parent_location, prev_location
# check whether to only create a reference
if new_parent_location and self.parent_key != "":
try:
if path.exists(prev_location):
move(prev_location, new_parent_location)
sleep(4)
except shutil.Error as E:
print E
exit()
db().updateResource(resource.resource_id.text, data)
else:
# Create the new folders
print "Creating new folder: %s" % resource.title.text
self.file_path = self.get_location(self.parent_key, resource.title.text, files=True)
parentAlive = db().getDetails("location", self.parent_key)
if self.parent_key != "":
if not parentAlive:
reference_only = True
self.file_path = ""
else:
if parentAlive[0] == "":
reference_only = True
self.file_path = ""
print "File location: '%s'" % self.file_path
if not reference_only:
if not path.exists(self.file_path):
mkdir(self.file_path)
data['location'] = self.file_path
data['ino'] = stat(self.file_path).st_ino
data['parent_ino'] = self.get_parent_ino(self.parent_key)
db().insertResource(data)
else:
duplicates.append(resource)
continue
else:
# update database with parent key
data['location'] = ""
db().insertResource(data)
self.parent_key = ""
self.file_path = ""
print "--------------------------------"
# handle the duplicates
print duplicates
# Create the rest of the folders in db
while db().getRest():
upd_data = {} # stores the k/v data to be updated
for row in db().getRest():
upd_data['location'] = self.get_location(row[0], row[1])
if not path.exists(upd_data['location']):
mkdir(upd_data['location'])
upd_data['ino'] = stat(upd_data['location']).st_ino
upd_data['parent_ino'] = self.get_parent_ino(row[2])
db().updateResource(row[0], upd_data)
def _update_files(self):
# upload the local files
self._upload_files()
# delete files
documents = self.client.GetResources(uri= '/feeds/default/private/full')
for document in documents.entry:
file_changed_online, file_changed_offline, changed_parent = True, True, False
documentParent = "" if not document.InCollections() else search("(folder%3.*)", \
document.InCollections()[0].href.strip()).group(0).replace("%3A", ":")
doctype = search("(\w*(?=:))", document.resource_id.text).group(0)
exportFormat = None if not self.fileType(doctype) else {'exportFormat': self.fileType(doctype)[1:]}
localData = db().getDetails("*", document.resource_id.text)
if localData: # if document already in database
if path.exists(localData[3]): # check if it exists locally
# Check for online changes
if document.updated.text == localData[5]: # check if the document hasn't been changed remotely
file_changed_online = False
# Check for offline changes
if int(path.getmtime(localData[3])) == localData[6]:
file_changed_offline = False
if file_changed_offline == True and file_changed_online == True:
'''
Handle clashes when the files are out of sync
'''
print "changed both"
else:
if file_changed_online: # if the file has been changed online
print "online version is newer"
parentFolder = None
# check if the file directory, or the file name has been changed
# if it has query database to check for parentDirectory
if documentParent != localData[2] or document.title.text != localData[1]:
# check if the parent exists locally
file_changed = True
if documentParent:
try:
parentFolder = db().getDetails('location', documentParent)[0]
except TypeError:
# create folder
parentFolder = "new"
else:
parentFolder = "root"
else:
self.parent_directory = localData[3]
file_changed = False
print parentFolder
exit()
if file_changed:
if documentParent != localData[2]:
if parentFolder:
if parentFolder == "":
self.parent_directory = SYNC_DIRECTORY
else:
self.parent_directory = parentFolder
else:
self.parent_directory = self.tmpdir
self.moved_to_tmp = True
self.parent_directory += document.title.text + self.fileType(doctype)
else:
self.parent_directory = parentFolder + document.title.text + self.fileType(doctype)
# the moving and doing a whateva of ze file
if not file_changed:
# if only the contents of the file have changed
# download the resource
self.client.DownloadResource(document, self.parent_directory, extra_params=exportFormat)
# update the database
localtime = int(path.getmtime(localData[3]))
database.updateDocument(document.resource_id.text, localData[1], documentParent, self.parent_directory, doctype, document.updated.text, localtime)
else:
# if the file parent directory exists locally
print "Move file to its new directory"
rename(localData[3], self.parent_directory)
# download the resource
sleep(1) # delay, not sure if needed
self.client.DownloadResource(document, self.parent_directory, extra_params=exportFormat)
localtime = int(path.getmtime(self.parent_directory))
database.updateDocument(document.resource_id.text, localData[1], documentParent, self.parent_directory, doctype, document.updated.text, localtime)
else:
print "file up to date"
if file_changed_offline:
print "offline version is newer"
sleep(1)
