def test_09_Files_Update_File(self):
delete_file(self.first_file + "1")
delete_file(self.second_file + "1")
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
filename = "preupdatetestfile"
newfilename = "updatetestfile"
file1["title"] = filename
file1.SetContentFile(self.first_file)
pydrive_retry(file1.Upload) # Files.insert
self.assertEqual(file1.metadata["title"], filename)
pydrive_retry(
file1.FetchContent) # Force download and double check content.
pydrive_retry(lambda: file1.GetContentFile(self.first_file + "1"))
self.assertEqual(filecmp.cmp(self.first_file, self.first_file + "1"),
True)
file1["title"] = newfilename
file1.SetContentFile(self.second_file)
pydrive_retry(file1.Upload) # Files.update
self.assertEqual(file1.metadata["title"], newfilename)
pydrive_retry(lambda: file1.GetContentFile(self.second_file + "1"))
self.assertEqual(filecmp.cmp(self.second_file, self.second_file + "1"),
True)
self.DeleteUploadedFiles(drive, [file1["id"]])
delete_file(self.first_file + "1")
delete_file(self.second_file + "1")
def test_05_Files_Insert_Content_File(self):
delete_file(self.first_file + "1")
delete_file(self.first_file + "2")
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
filename = "filecontent"
file1["title"] = filename
file1.SetContentFile(self.first_file)
pydrive_retry(file1.Upload) # Files.insert
self.assertEqual(file1.metadata["title"], filename)
pydrive_retry(
file1.FetchContent) # Force download and double check content.
pydrive_retry(lambda: file1.GetContentFile(self.first_file + "1"))
self.assertEqual(filecmp.cmp(self.first_file, self.first_file + "1"),
True)
file2 = drive.CreateFile({"id": file1["id"]}) # Download file from id.
pydrive_retry(lambda: file2.GetContentFile(self.first_file + "2"))
self.assertEqual(filecmp.cmp(self.first_file, self.first_file + "2"),
True)
self.DeleteUploadedFiles(drive, [file1["id"]])
delete_file(self.first_file + "1")
delete_file(self.first_file + "2")
def test_08_Files_Update_String(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
filename = 'preupdatetestfile'
newfilename = 'updatetestfile'
content = 'hello world!'
newcontent = 'hello new world!'
file1['title'] = filename
file1.SetContentString(content)
pydrive_retry(file1.Upload) # Files.insert
self.assertEqual(file1.metadata['title'], filename)
self.assertEqual(file1.GetContentString(), content)
pydrive_retry(file1.FetchContent) # Force download and double check content.
self.assertEqual(file1.GetContentString(), content)
file1['title'] = newfilename
file1.SetContentString(newcontent)
pydrive_retry(file1.Upload) # Files.update
self.assertEqual(file1.metadata['title'], newfilename)
self.assertEqual(file1.GetContentString(), newcontent)
self.assertEqual(file1.GetContentString(), newcontent)
self.DeleteUploadedFiles(drive, [file1['id']])
def check_google():
from pydrive2.auth import GoogleAuth
from pydrive2.drive import GoogleDrive
from pydrive2.auth import ServiceAccountCredentials
gauth = GoogleAuth()
scope = ['https://www.googleapis.com/auth/drive']
cred_path = os.path.join(DATA_PATH, 'credentials.json')
gauth.credentials = ServiceAccountCredentials.from_json_keyfile_name(
cred_path, scope)
drive = GoogleDrive(gauth)
file_id = '1603ahBNdt1SnSaYYBE-G8SA6qgRTQ6fF'
file_list = drive.ListFile({
'q':
"'%s' in parents and trashed=false" % file_id
}).GetList()
df = pandas.DataFrame(file_list)
dfclean = df[['createdDate', 'id', 'title']].copy()
dfclean['date'] = pandas.to_datetime(dfclean['createdDate'],
format='%Y-%m-%d',
errors='coerce')
lastupdate = dfclean.loc[dfclean['createdDate'] ==
'2020-09-11T01:53:29.639Z'].iloc[0]['date']
dfnew = dfclean.loc[dfclean['date'] > lastupdate]
all_files = os.listdir(REPORTS_PATH)
new_files = [
item for item in all_files
if item not in dfnew['title'].unique().tolist()
]
reportdf = dfnew.loc[dfnew['title'].isin(new_files)]
return (reportdf)
def test_09_Files_Update_File(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
filename = self.getTempFile("preupdatetestfile")
newfilename = self.getTempFile("updatetestfile")
contentFile = self.getTempFile("actual_content", "some string")
contentFile2 = self.getTempFile("actual_content_2", "some string")
file1["title"] = filename
file1.SetContentFile(contentFile)
pydrive_retry(file1.Upload) # Files.insert
self.assertEqual(file1.metadata["title"], filename)
pydrive_retry(
file1.FetchContent) # Force download and double check content.
fileOut = self.getTempFile()
pydrive_retry(file1.GetContentFile, fileOut)
self.assertEqual(filecmp.cmp(contentFile, fileOut), True)
file1["title"] = newfilename
file1.SetContentFile(contentFile2)
pydrive_retry(file1.Upload) # Files.update
self.assertEqual(file1.metadata["title"], newfilename)
fileOut = self.getTempFile()
pydrive_retry(file1.GetContentFile, fileOut)
self.assertEqual(filecmp.cmp(contentFile2, fileOut), True)
self.DeleteUploadedFiles(drive, [file1["id"]])
def test_08_Files_Update_String(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
filename = self.getTempFile("preupdatetestfile")
newfilename = self.getTempFile("updatetestfile")
content = "hello world!"
newcontent = "hello new world!"
file1["title"] = filename
file1.SetContentString(content)
pydrive_retry(file1.Upload) # Files.insert
self.assertEqual(file1.metadata["title"], filename)
self.assertEqual(file1.GetContentString(), content)
pydrive_retry(
file1.FetchContent) # Force download and double check content.
self.assertEqual(file1.GetContentString(), content)
file1["title"] = newfilename
file1.SetContentString(newcontent)
pydrive_retry(file1.Upload) # Files.update
self.assertEqual(file1.metadata["title"], newfilename)
self.assertEqual(file1.GetContentString(), newcontent)
self.assertEqual(file1.GetContentString(), newcontent)
self.DeleteUploadedFiles(drive, [file1["id"]])
def test_09_Files_Update_File(self):
delete_file(self.first_file+'1')
delete_file(self.second_file+'1')
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
filename = 'preupdatetestfile'
newfilename = 'updatetestfile'
file1['title'] = filename
file1.SetContentFile(self.first_file)
pydrive_retry(file1.Upload) # Files.insert
self.assertEqual(file1.metadata['title'], filename)
pydrive_retry(file1.FetchContent) # Force download and double check content.
file1.GetContentFile(self.first_file+'1')
self.assertEqual(filecmp.cmp(self.first_file, self.first_file+'1'), True)
file1['title'] = newfilename
file1.SetContentFile(self.second_file)
pydrive_retry(file1.Upload) # Files.update
self.assertEqual(file1.metadata['title'], newfilename)
file1.GetContentFile(self.second_file+'1')
self.assertEqual(filecmp.cmp(self.second_file, self.second_file+'1'), True)
self.DeleteUploadedFiles(drive, [file1['id']])
delete_file(self.first_file + '1')
delete_file(self.second_file + '1')
def check_google():
from pydrive2.auth import GoogleAuth
from pydrive2.drive import GoogleDrive
from pydrive2.auth import ServiceAccountCredentials
gauth = GoogleAuth()
scope = ['https://www.googleapis.com/auth/drive']
gauth.credentials = ServiceAccountCredentials.from_json_keyfile_name(
'credentials.json', scope)
drive = GoogleDrive(gauth)
file_id = '1vC8oXhfhogAh7olq9BvEPdwwvyeXsZkk'
file_list = drive.ListFile({
'q':
"'%s' in parents and trashed=false" % file_id
}).GetList()
df = pandas.DataFrame(file_list)
dfclean = df[['createdDate', 'id', 'title']].copy()
dfclean['date'] = pandas.to_datetime(dfclean['createdDate'],
format='%Y-%m-%d',
errors='coerce')
lastupdate = dfclean.loc[dfclean['createdDate'] ==
'2020-09-28T22:28:33.989Z'].iloc[0]['date']
dfnew = dfclean.loc[dfclean['date'] > lastupdate]
all_files = os.listdir('data/tables/')
new_files = [
item for item in dfnew['title'].unique().tolist()
if item not in all_files
]
tabledf = dfnew.loc[dfnew['title'].isin(new_files)]
return (tabledf)
def __init__(self, local_repo_path, model_id):
self.repo_path = local_repo_path
self.model_id = model_id
GoogleAuth = set_secrets_file()
gauth = GoogleAuth()
gauth.LocalWebserverAuth()
self.drive = GoogleDrive(gauth)
def test_Files_Insert_Permission(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
# Verify only one permission before inserting permission.
permissions = pydrive_retry(file1.GetPermissions)
self.assertEqual(len(permissions), 1)
self.assertEqual(len(file1['permissions']), 1)
# Insert the permission.
permission = pydrive_retry(lambda: file1.InsertPermission({'type': 'anyone',
'value': 'anyone',
'role': 'reader'}))
self.assertTrue(permission)
self.assertEqual(len(file1["permissions"]), 2)
self.assertEqual(file1["permissions"][0]["type"], "anyone")
permissions = pydrive_retry(file1.GetPermissions)
self.assertEqual(len(file1["permissions"]), 2)
self.assertEqual(file1["permissions"][0]["type"], "anyone")
self.assertEqual(permissions[0]["type"], "anyone")
# Verify remote changes made.
file2 = drive.CreateFile({'id': file1['id']})
permissions = pydrive_retry(file2.GetPermissions)
self.assertEqual(len(permissions), 2)
self.assertEqual(permissions[0]["type"], "anyone")
pydrive_retry(file1.Delete)
def on_event(self, event, payload):
if event == "plugin_backup_backup_created" and self._settings.get_boolean(
["cert_authorized"]):
self._logger.info(
"{} created, will now attempt to upload to Google Drive".
format(payload["path"]))
from pydrive2.drive import GoogleDrive
from pydrive2.auth import GoogleAuth
credentials_file = "{}/credentials.json".format(
self.get_plugin_data_folder())
gauth = GoogleAuth()
gauth.LoadCredentialsFile(credentials_file)
if gauth.credentials is None:
self._logger.error("not authorized")
self._settings.set(["cert_authorized"], False)
self._settings.save()
return
elif gauth.access_token_expired:
gauth.Refresh()
else:
gauth.Authorize()
gauth.SaveCredentialsFile(credentials_file)
drive = GoogleDrive(gauth)
f = drive.CreateFile({'title': payload["name"]})
f.SetContentFile(payload["path"])
f.Upload()
f = None
def _parallel_downloader(self, file_ids, num_of_workers):
drive = GoogleDrive(self.ga)
thread_pool = ThreadPoolExecutor(max_workers=num_of_workers)
# Create list of gdrive_files.
download_files = []
for file_id in file_ids:
file1 = drive.CreateFile({"id": file_id})
file1["title"] = self.getTempFile()
download_files.append(file1)
# Ensure files don't exist yet.
for file_obj in download_files:
self.assertTrue(not delete_file(file_obj["title"]))
# Submit upload jobs to ThreadPoolExecutor.
futures = []
for file_obj in download_files:
futures.append(
thread_pool.submit(pydrive_retry, file_obj.GetContentFile,
file_obj["title"]))
# Ensure that all threads a) return, and b) encountered no exceptions.
for future in as_completed(futures):
self.assertIsNone(future.exception())
thread_pool.shutdown()
# Ensure all files were downloaded.
for file_obj in download_files:
self.assertTrue(delete_file(file_obj["title"]))
# Remove uploaded files.
self.DeleteUploadedFiles(drive, file_ids)
def test_GFile_Conversion_Lossless_String(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
# Upload a string, and convert into Google Doc format.
test_string = "Generic, non-exhaustive ASCII test string."
file1.SetContentString(test_string)
pydrive_retry(file1.Upload, {"convert": True})
# Download string as plain text.
downloaded_string = file1.GetContentString(mimetype="text/plain")
self.assertEqual(test_string, downloaded_string,
"Strings do not match")
# Download content into file and ensure that file content matches original
# content string.
downloaded_file_name = "_tmp_downloaded_file_name.txt"
pydrive_retry(
file1.GetContentFile,
downloaded_file_name,
mimetype="text/plain",
remove_bom=True,
)
downloaded_string = open(downloaded_file_name).read()
self.assertEqual(test_string, downloaded_string,
"Strings do not match")
# Delete temp file.
delete_file(downloaded_file_name)
def test_Files_Delete_Permission(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
pydrive_retry(
file1.InsertPermission,
{
"type": "anyone",
"value": "anyone",
"role": "reader"
},
)
permissions = pydrive_retry(file1.GetPermissions)
self.assertEqual(len(permissions), 2)
self.assertEqual(len(file1["permissions"]), 2)
pydrive_retry(file1.DeletePermission, permissions[0]["id"])
self.assertEqual(len(file1["permissions"]), 1)
# Verify remote changes made.
file2 = drive.CreateFile({"id": file1["id"]})
permissions = pydrive_retry(file2.GetPermissions)
self.assertEqual(len(permissions), 1)
pydrive_retry(file1.Delete)
def test_Files_Insert_Permission(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
# Verify only one permission before inserting permission.
permissions = pydrive_retry(file1.GetPermissions)
self.assertEqual(len(permissions), 1)
self.assertEqual(len(file1["permissions"]), 1)
# Insert the permission.
permission = pydrive_retry(
file1.InsertPermission,
{
"type": "anyone",
"value": "anyone",
"role": "reader"
},
)
self.assertTrue(permission)
self.assertEqual(len(file1["permissions"]), 2)
self.assertEqual(file1["permissions"][0]["type"], "anyone")
permissions = pydrive_retry(file1.GetPermissions)
self.assertEqual(len(file1["permissions"]), 2)
self.assertEqual(file1["permissions"][0]["type"], "anyone")
self.assertEqual(permissions[0]["type"], "anyone")
# Verify remote changes made.
file2 = drive.CreateFile({"id": file1["id"]})
permissions = pydrive_retry(file2.GetPermissions)
self.assertEqual(len(permissions), 2)
self.assertEqual(permissions[0]["type"], "anyone")
pydrive_retry(file1.Delete)
async def gdrive_upload(filename: str, filebuf: BytesIO = None) -> str:
"""
Upload files to Google Drive using PyDrive2
"""
# a workaround for disabling cache errors
# https://github.com/googleapis/google-api-python-client/issues/299
logging.getLogger('googleapiclient.discovery_cache').setLevel(
logging.CRITICAL)
# Authenticate Google Drive automatically
# https://stackoverflow.com/a/24542604
gauth = GoogleAuth()
# Try to load saved client credentials
gauth.LoadCredentialsFile("secret.json")
if gauth.credentials is None:
return "nosecret"
if gauth.access_token_expired:
gauth.Refresh()
else:
# Initialize the saved credentials
gauth.Authorize()
# Save the current credentials to a file
gauth.SaveCredentialsFile("secret.json")
drive = GoogleDrive(gauth)
if filename.count('/') > 1:
filename = filename.split('/')[-1]
filedata = {
'title': filename,
"parents": [{
"kind": "drive#fileLink",
"id": GDRIVE_FOLDER
}]
}
if filebuf:
mime_type = mimetypes.guess_type(filename)
if mime_type[0] and mime_type[1]:
filedata['mimeType'] = f"{mime_type[0]}/{mime_type[1]}"
else:
filedata['mimeType'] = 'text/plain'
file = drive.CreateFile(filedata)
file.content = filebuf
else:
file = drive.CreateFile(filedata)
file.SetContentFile(filename)
name = filename.split('/')[-1]
file.Upload()
# insert new permission
file.InsertPermission({
'type': 'anyone',
'value': 'anyone',
'role': 'reader'
})
if not filebuf:
os.remove(filename)
reply = f"[{name}]({file['alternateLink']})\n" \
f"__Direct link:__ [Here]({file['downloadUrl']})"
return reply
def search_folder(self, folderid: str):
drive = GoogleDrive(self.gauth)
file_list = drive.ListFile({
'q':
f"'{folderid}' in parents and trashed=false"
}).GetList()
return file_list
def test_Files_FetchAllMetadata_Fields(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
pydrive_retry(file1.FetchMetadata, fetch_all=True)
self.assertTrue("hasThumbnail" in file1)
self.assertTrue("thumbnailVersion" in file1)
self.assertTrue("permissions" in file1)
pydrive_retry(file1.Delete)
def test_Files_FetchMetadata_Fields(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
self.assertFalse('permissions' in file1)
pydrive_retry(lambda: file1.FetchMetadata('permissions'))
self.assertTrue('permissions' in file1)
pydrive_retry(file1.Delete)
def __init__(self):
gauth = GoogleAuth()
gauth.LoadCredentialsFile("google_credentials.txt")
if gauth.credentials is None:
gauth.LocalWebserverAuth()
elif gauth.access_token_expired:
gauth.Refresh()
else:
gauth.Authorize()
gauth.SaveCredentialsFile("google_credentials.txt")
self.drive = GoogleDrive(gauth)
def test_02_Files_Insert_Unicode(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
filename = u'첫번째 파일'
file1['title'] = filename
pydrive_retry(file1.Upload) # Files.insert
self.assertEqual(file1.metadata['title'], filename)
file2 = drive.CreateFile({'id': file1['id']}) # Download file from id.
self.assertEqual(file2['title'], filename)
self.DeleteUploadedFiles(drive, [file1['id']])
def test_02_Files_Insert_Unicode(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
filename = self.getTempFile(u"첫번째 파일")
file1["title"] = filename
pydrive_retry(file1.Upload) # Files.insert
self.assertEqual(file1.metadata["title"], filename)
file2 = drive.CreateFile({"id": file1["id"]}) # Download file from id.
self.assertEqual(file2["title"], filename)
self.DeleteUploadedFiles(drive, [file1["id"]])
def test_Files_Get_Permissions(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
self.assertFalse('permissions' in file1)
permissions = pydrive_retry(file1.GetPermissions)
self.assertTrue(permissions is not None)
self.assertTrue('permissions' in file1)
pydrive_retry(file1.Delete)
def test_Files_Delete_Permission_Invalid(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
try:
pydrive_retry(lambda: file1.DeletePermission('invalid id'))
self.fail("Deleting invalid permission not raising exception.")
except ApiRequestError as e:
pass
pydrive_retry(file1.Delete)
def __init__(self, cred_file, folder_id):
try:
from pydrive2.auth import ServiceAccountCredentials, GoogleAuth
from pydrive2.drive import GoogleDrive
except ImportError:
raise Sorry("Pydrive2 not found. Try:\n$ conda install pydrive2 -c conda-forge")
gauth = GoogleAuth()
scope = ['https://www.googleapis.com/auth/drive']
gauth.credentials = ServiceAccountCredentials.from_json_keyfile_name(
cred_file, scope
)
self.drive = GoogleDrive(gauth)
self.top_folder_id = folder_id
def test_Files_Delete_File_Just_ID(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
file2 = drive.CreateFile({'id': file1['id']})
pydrive_retry(file2.Delete)
try:
pydrive_retry(file1.FetchMetadata)
self.fail("File not deleted correctly.")
except ApiRequestError as e:
pass
def test_12_Upload_Download_Empty_File(self):
filename = os.path.join(self.tmpdir, str(time()))
create_file(filename, "")
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
file1.SetContentFile(filename)
pydrive_retry(file1.Upload)
fileOut1 = self.getTempFile()
pydrive_retry(file1.GetContentFile, fileOut1)
self.assertEqual(os.path.getsize(fileOut1), 0)
self.DeleteUploadedFiles(drive, [file1["id"]])
def setup_gfile_conversion_test(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
# Create a file to upload.
file_name = '_tmp_source_file.txt'
downloaded_file_name = '_tmp_downloaded_file_name.txt'
original_file_content = 'Generic, non-exhaustive\n ASCII test string.'
source_file = open(file_name, mode='w+')
source_file.write(original_file_content)
source_file.close()
original_file_content = test_util.StripNewlines(original_file_content)
return file1, file_name, original_file_content, downloaded_file_name
def test_Files_UnTrash_File_Just_ID(self):
drive = GoogleDrive(self.ga)
file1 = drive.CreateFile()
pydrive_retry(file1.Upload)
pydrive_retry(file1.Trash)
self.assertTrue(file1.metadata[u'labels'][u'trashed'])
file2 = drive.CreateFile({'id': file1['id']})
pydrive_retry(file2.UnTrash) # UnTrash without fetching metadata.
pydrive_retry(file1.FetchMetadata)
self.assertFalse(file1.metadata[u'labels'][u'trashed'])
self.DeleteUploadedFiles(drive, [file1['id']])
class DVCSetup(object):
def __init__(self, local_repo_path, model_id):
self.repo_path = local_repo_path
self.model_id = model_id
GoogleAuth = set_secrets_file()
gauth = GoogleAuth()
gauth.LocalWebserverAuth()
self.drive = GoogleDrive(gauth)
def gdrive_setup(self):
folder = self.drive.CreateFile({
"title":
self.model_id,
"parents": [{
"id": ISAURA_GDRIVE
}],
"mimeType":
"application/vnd.google-apps.folder",
})
folder.Upload()
def gdrive_folder_id(self):
fileList = self.drive.ListFile({
"q": "'" + ISAURA_GDRIVE + "' in parents and trashed=false",
"corpora": "teamDrive",
"teamDriveId": ISAURA_TEAM_GDRIVE,
"includeTeamDriveItems": True,
"supportsTeamDrives": True,
}).GetList()
for file in fileList:
if file["title"] == self.model_id:
return str(file["id"])
def set_dvc_gdrive(self):
terminal.run_command("dvc --cd {0} add data.h5".format(self.repo_path))
terminal.run_command("dvc --cd " + self.repo_path +
" remote add -d public_repo gdrive://" +
self.gdrive_folder_id())
cmd = "dvc --cd " + self.repo_path + " push"
terminal.run_command(cmd, quiet=False)
def git_add_and_commit(self, message="Set to public data repo"):
cwd = os.getcwd()
os.chdir(self.repo_path)
terminal.run_command(
"git add data.h5.dvc"
) # TODO: Be more specific in the files/folders to be added
terminal.run_command("git commit -m '{0}'".format(message))
os.chdir(cwd)