def googledrive_upload(self, filename, file_handle, parent_id=None):
mime = MimeTypes()
guessed_type = mime.guess_type(filename)[0]
file_metadata = {gdu.NAME: filename}
if self.write_as_google_doc and guessed_type == gdu.CSV:
file_metadata[gdu.MIME_TYPE] = gdu.SPREADSHEET
if filename.lower().endswith(".csv"):
file_metadata[gdu.NAME] = filename + ".csv"
if guessed_type is None:
guessed_type = gdu.BINARY_STREAM
media = MediaIoBaseUpload(file_handle,
mimetype=guessed_type,
resumable=True)
query = gdu.query_parents_in([parent_id], name=filename, trashed=False)
files = self.googledrive_list(query)
if len(files) == 0:
self.googledrive_create(body=file_metadata,
media_body=media,
parent_id=parent_id)
else:
self.googledrive_update(file_id=gdu.get_id(files[0]),
body=file_metadata,
media_body=media,
parent_id=parent_id)
def get_item_from_path(self, path_and_file):
tokens = gdu.split_path(path_and_file)
if len(tokens) == 1:
return {
gdu.MIME_TYPE: gdu.FOLDER,
gdu.SIZE: u'0',
gdu.ID: self.root_id,
gdu.NAME: u'/'
}
parent_ids = [self.root_id]
for token in tokens:
if token == '/':
token = ''
continue
query = gdu.query_parents_in(parent_ids,
name_contains=token,
trashed=False)
files = self.googledrive_list(query)
files = gdu.keep_files_with(files, name_starting_with=token)
files = gdu.keep_files_with(
files, name=token
) # we only keep files / parent_ids for names = current token for the next loop
if len(files) == 0:
return None
parent_ids = gdu.get_files_ids(files)
return files[0]
def __init__(self, root, config, plugin_config):
if len(root) > 0 and root[0] == '/':
root = root[1:]
self.root = root
self.provider_root = "/"
gdu.check_path_format(self.get_normalized_path(self.root))
self.root_id = gdu.get_root_id(config)
self.session = GoogleDriveSession(config, plugin_config)
def __init__(self, config, plugin_config):
scopes = ['https://www.googleapis.com/auth/drive']
connection = plugin_config.get("googledrive_connection")
self.auth_type = config.get("auth_type")
self.write_as_google_doc = config.get(
"googledrive_write_as_google_doc")
self.nodir_mode = False # Future development
if self.auth_type == "oauth":
self.access_token = config.get("oauth_credentials")["access_token"]
credentials = AccessTokenCredentials(self.access_token,
"dss-googledrive-plugin/2.0")
http_auth = credentials.authorize(Http())
else:
credentials_dict = eval(connection['credentials'])
credentials = ServiceAccountCredentials.from_json_keyfile_dict(
credentials_dict, scopes)
http_auth = credentials.authorize(Http())
self.root_id = config.get("googledrive_root_id")
if not self.root_id:
self.root_id = gdu.ROOT_ID
self.max_attempts = 5
self.root_id = gdu.get_root_id(config)
self.drive = build(
gdu.API,
gdu.API_VERSION,
http=http_auth,
cache=MemoryCache(
) # Fix for ImportError messages https://github.com/googleapis/google-api-python-client/issues/325
)
def create_directory_from_path(self, path):
tokens = gdu.split_path(path)
parent_ids = [self.root_id]
current_path = ""
for token in tokens:
current_path = os.path.join(current_path, token)
item = self.get_item_from_path(current_path)
if item is None:
new_directory_id = self.create_directory(token, parent_ids)
parent_ids = [new_directory_id]
else:
new_directory_id = gdu.get_id(item)
parent_ids = [new_directory_id]
return new_directory_id
def list_recursive(self, path, folder, first_non_empty):
paths = []
if path == "/":
path = ""
children = self.session.directory(folder, root_path=self.get_rel_path(path))
for child in children:
if gdu.is_directory(child):
paths.extend(self.list_recursive(path + '/' + gdu.get_name(child), child, first_non_empty))
else:
paths.append({
DSSConstants.PATH: path + '/' + gdu.get_name(child),
DSSConstants.SIZE: gdu.file_size(child),
DSSConstants.LAST_MODIFIED: gdu.get_last_modified(child)
})
if first_non_empty:
return paths
return paths
def create_directory(self, name, parent_ids):
file_metadata = {
gdu.NAME: name,
gdu.PARENTS: parent_ids,
gdu.MIME_TYPE: gdu.FOLDER
}
file = self.googledrive_create(body=file_metadata)
return gdu.get_id(file)
def stat(self, path):
"""
Get the info about the object at the given path inside the provider's root, or None
if the object doesn't exist
"""
full_path = self.get_full_path(path)
logger.info('stat:path="{}", full_path="{}"'.format(path, full_path))
item = self.session.get_item_from_path(full_path)
if item is None:
return None
return {
DSSConstants.PATH: self.get_normalized_path(path),
DSSConstants.SIZE: gdu.file_size(item),
DSSConstants.LAST_MODIFIED: gdu.get_last_modified(item),
DSSConstants.IS_DIRECTORY: gdu.is_directory(item)
}
def googledrive_download(self, item, stream):
if gdu.is_file_google_doc(item):
document_type = gdu.get_google_doc_type(item)
data = self.drive.files().export_media(
fileId=gdu.get_id(item),
mimeType=gdu.get_google_doc_mime_equivalence(
document_type)).execute()
file_handle = BytesIO()
file_handle.write(data)
file_handle.seek(0)
shutil.copyfileobj(file_handle, stream)
else:
request = self.drive.files().get_media(fileId=gdu.get_id(item))
downloader = MediaIoBaseDownload(stream,
request,
chunksize=1024 * 1024)
done = False
while done is False:
status, done = downloader.next_chunk()
def googledrive_delete(self, item, parent_id=None):
attempts = 0
while attempts < self.max_attempts:
try:
if len(item[gdu.PARENTS]) == 1 or parent_id is None:
self.drive.files().delete(
fileId=gdu.get_id(item)).execute()
else:
self.drive.files().update(
fileId=gdu.get_id(item),
removeParents=parent_id).execute()
except HttpError as err:
if err.resp.status == 404:
return
self.handle_googledrive_errors(err, "delete")
attempts = attempts + 1
logger.info('googledrive_update:attempts={}'.format(attempts))
raise GoogleDriveSessionError(
"Max number of attempts reached in Google Drive directory delete operation"
)
def move(self, from_path, to_path):
"""
Move a file or folder to a new path inside the provider's root. Return false if the moved file didn't exist
"""
full_from_path = self.get_full_path(from_path)
full_to_path = self.get_full_path(to_path)
from_name = os.path.basename(from_path)
to_name = os.path.basename(to_path)
logger.info('move:from "{}" to "{}"'.format(full_from_path, full_to_path))
try:
from_item = self.session.get_item_from_path(full_from_path)
if from_item is None:
return False
if from_name == to_name:
to_item = self.session.get_item_from_path(os.path.split(full_to_path)[0])
prev_parents = ','.join(p for p in from_item.get(gdu.PARENTS))
self.session.drive.files().update(
fileId=gdu.get_id(from_item),
addParents=gdu.get_id(to_item),
removeParents=prev_parents,
fields=gdu.ID_PARENTS_FIELDS,
).execute()
else:
file = self.session.drive.files().get(fileId=gdu.get_id(from_item)).execute()
del file[gdu.ID]
file[gdu.NAME] = to_name
self.session.drive.files().update(
fileId=gdu.get_id(from_item),
body=file,
fields=gdu.ID_PARENTS_FIELDS,
).execute()
except HttpError as err:
raise Exception('Error from Google Drive while moving files: ' + err)
return True
def delete_recursive(self, path):
"""
Delete recursively from path. Return the number of deleted files (optional)
"""
full_path = self.get_full_path(path)
logger.info('delete_recursive:path="{}", full_path="{}"'.format(path, full_path))
self.assert_path_is_not_root(full_path)
deleted_item_count = 0
item = self.session.get_item_from_path(full_path)
if gdu.is_directory(item):
if item is None or gdu.PARENTS not in item:
return deleted_item_count
else:
query = gdu.query_parents_in([gdu.get_id(item)])
children_items = self.session.googledrive_list(query)
for child_item in children_items:
self.session.googledrive_delete(child_item, parent_id=gdu.get_id(item))
deleted_item_count = deleted_item_count + 1
self.session.googledrive_delete(item)
else:
self.session.googledrive_delete(item)
deleted_item_count = deleted_item_count + 1
return deleted_item_count
def browse(self, path):
"""
List the file or directory at the given path, and its children (if directory)
"""
full_path = self.get_full_path(self.get_rel_path(path))
logger.info('browse:path="{}", full_path="{}"'.format(path, full_path))
item = self.session.get_item_from_path(full_path)
if item is None:
return {
DSSConstants.FULL_PATH: None,
DSSConstants.EXISTS: False
}
if gdu.is_file(item):
return {
DSSConstants.FULL_PATH: self.get_normalized_path(path),
DSSConstants.EXISTS: True,
DSSConstants.DIRECTORY: False,
DSSConstants.SIZE: gdu.file_size(item),
DSSConstants.LAST_MODIFIED: gdu.get_last_modified(item)
}
children = []
files = self.session.directory(item, root_path=self.get_rel_path(full_path))
for file in files:
sub_path = self.get_normalized_path(os.path.join(path, gdu.get_name(file)))
children.append({
DSSConstants.FULL_PATH: sub_path,
DSSConstants.EXISTS: True,
DSSConstants.DIRECTORY: gdu.is_directory(file),
DSSConstants.SIZE: gdu.file_size(file),
DSSConstants.LAST_MODIFIED: gdu.get_last_modified(file)
})
return {
DSSConstants.FULL_PATH: self.get_normalized_path(path),
DSSConstants.EXISTS: True,
DSSConstants.DIRECTORY: True,
DSSConstants.CHILDREN: children,
DSSConstants.LAST_MODIFIED: gdu.get_last_modified(item)
}
def enumerate(self, path, first_non_empty):
"""
Enumerate files recursively from prefix. If first_non_empty, stop at the first non-empty file.
If the prefix doesn't denote a file or folder, return None
"""
full_path = self.get_full_path(path)
logger.info('enumerate:path="{}", full_path="{}"'.format(path, full_path))
item = self.session.get_item_from_path(full_path)
if item is None:
no_directory_item = self.session.get_item_from_path(self.get_root_path())
query = gdu.query_parents_in(
[gdu.get_id(no_directory_item)],
name_contains=self.get_rel_path(path),
trashed=False
)
files = self.session.googledrive_list(query)
if len(files) == 0:
return None
paths = []
for file in files:
paths.append({
DSSConstants.PATH: self.get_normalized_path(gdu.get_name(file)),
DSSConstants.SIZE: file[gdu.SIZE],
DSSConstants.LAST_MODIFIED: gdu.get_last_modified(file)
})
return paths
if item is None:
return None
if gdu.is_file(item):
return [{
DSSConstants.PATH: self.get_normalized_path(path),
DSSConstants.SIZE: gdu.file_size(item),
DSSConstants.LAST_MODIFIED: gdu.get_last_modified(item)
}]
paths = []
paths = self.list_recursive(path, item, first_non_empty)
return paths
def test_is_directory(self):
not_a_directory = {'mimeType': "text/csv"}
directory = {'mimeType': "application/vnd.google-apps.folder"}
assert GoogleDriveUtils.is_directory(not_a_directory) == False
assert GoogleDriveUtils.is_directory(directory) == True
def directory(self, item, root_path=None):
query = gdu.query_parents_in([gdu.get_id(item)], trashed=False)
files = self.googledrive_list(query)
return files