def cleanup(config):
gauth = GoogleDriveActionBase.GoogleDriveActionBase.authenticate(
config)
drive = GoogleDrive(gauth)
retain_from_date = datetime.today() - timedelta(days=int(
config.config_obj.get('GoogleDriveUploadAction',
'file_retention_days')))
file_list_len = 1
logger.debug(drive.GetAbout())
while file_list_len > 0:
file_list = drive.ListFile({
'q':
"properties has { key='source' and value='MotionNotify' and visibility='PRIVATE'} and modifiedDate<'"
+ retain_from_date.strftime("%Y-%m-%d") + "'"
}).GetList()
file_list_len = file_list.__len__()
logger.info("GoogleDriveCleanAction - removing " +
file_list_len.__str__() + " files")
print(file_list.__len__())
for file1 in file_list:
logger.debug(
'GoogleDriveUploadAction Removing: title: %s, id: %s, createdDate: %s, parents: %s'
% (file1['title'], file1['id'], file1['createdDate'],
file1['parents']))
file1.Delete()
def save_file(path,
file_to_save,
env,
min_epoch,
saver_module='torch',
alternative=None):
if env != 'drive':
if saver_module == 'torch':
torch.save(file_to_save, path)
else:
utils.save_obj(file_to_save, path)
return 1
# If platform is Google drive, then do checks
gauth = authenticate_Grive()
drive = GoogleDrive(gauth)
infos = drive.GetAbout()
quotion = int(infos['quotaBytesUsed']) / int(infos['quotaBytesTotal'])
torch.save(file_to_save, path)
if not os.path.exists(path) or quotion >= 0.99:
print('\t [Alert] Maximum storage reached on Drive!', '\n\t',
' Migration of all checkpoints to github ...')
print('\t\t Drive storage : ', quotion * 100, '%')
# Authentification to github
# git_manager=GitManager('5598c0e73e05423e7538fd19cb2d510379e9e588')
# git_manager=GitManager(user='******',pwd='pfemaster2020')
# git_manager.authentification()
# target_repo=git_manager.get_repo('checkpoints')
# # Fetch checkpoints from the directory in order to push them all to github
# files_to_push=[os.path.abspath(el) for el in glob.glob(os.path.join(os.path.split(path)[0],'*.pth'))]
# res=git_manager.push_files(target_repo,files_to_push,'checkpoints migration')
# # If all files were pushed without problem, delete them
# if isinstance(res,int)and res==len(files_to_push):
# print('\t Successfully transfered checkpoints to github')
# for f in glob.glob(os.path.join(os.path.split(path)[0],'*.pth')):
# os.remove(f)
# # Now save the file
# torch.save(file_to_save,path)
# assert os.path.exists(path), 'Error ! File to save couldn\'t be saved !'
# else: raise RuntimeError('Couldn\'t push all files')
return 0
else:
# torch.save(file_to_save,path)
return 1
def _connect() -> GoogleDrive:
gauth = GoogleAuth(settings_file=self._setting_file_name)
# Try to load saved client credentials
gauth.LoadCredentialsFile(self._credential_file_name)
if gauth.credentials is None:
raise GoogleCredentialsNotFoundException(
f"{self._credential_file_name} not found")
elif gauth.access_token_expired:
# Refresh them if expired
gauth.Refresh()
else:
# Initialize the saved creds
gauth.Authorize()
# Save the current credentials to a file
gauth.SaveCredentialsFile(self._credential_file_name)
drive = GoogleDrive(gauth)
# Actually try to connect to the drive
drive.GetAbout()
return drive
class GoogleDriver(RAIDStorage):
FOLDER_ID = '0B3YfnXuRdcz4SXIyaXc2Z0xNQUE'
def __init__(self):
self.client = GoogleDrive(self.authorize())
self.index = None
def authorize(self):
self.connected = False
gauth = GoogleAuth()
gauth.LoadCredentialsFile('credentials.json')
try:
if gauth.credentials is None:
gauth.LocalWebserverAuth()
self.connected = True
elif gauth.access_token_expired:
gauth.Refresh()
self.connected = True
else:
gauth.Authorize()
self.connected = True
except httplib2.ServerNotFoundError:
#logging.critical('Connection could not be made to Google Drive')
self.connected = False
gauth.SaveCredentialsFile("credentials.json")
return gauth
def upload_file(self, file_name):
file_list = self.client.ListFile({
'q':
"'0B3YfnXuRdcz4SXIyaXc2Z0xNQUE' in parents and trashed=false"
}).GetList()
matches = [i for i in file_list if i['title'] == file_name]
if not matches:
file = self.client.CreateFile({
"parents": [{
"kind": "drive#fileLink",
"id": self.FOLDER_ID
}]
})
file.SetContentFile(file_name)
file.Upload()
logging.warning("File Uploaded to Google")
else:
logging.error('Google: File already exists')
def get_data(self, file_name):
name, extention = os.path.splitext(file_name)
file_name = name + self.index + extention
file_list = self.client.ListFile({
'q':
"'0B3YfnXuRdcz4SXIyaXc2Z0xNQUE' in parents and trashed=false"
}).GetList()
matches = [i for i in file_list if i['title'] == file_name]
if not matches:
logging.error('Google: No file found')
return ('Google', self.index)
else:
file = matches[0]
data = file.GetContentString(mimetype='text/csv').replace(
'\r\n', '')
data = [data[i:i + 10] for i in range(0, len(data), 10)]
return [file['title'], data]
return matches[0]
def delete_data(self, file_name):
name, extention = os.path.splitext(file_name)
file_name = name + self.index + '.csv'
file_list = self.client.ListFile({
'q':
"'0B3YfnXuRdcz4SXIyaXc2Z0xNQUE' in parents and trashed=false"
}).GetList()
matches = [i for i in file_list if i['title'] == file_name]
if not matches:
logging.error("Google: No file found")
else:
try:
file = matches[0]
file.Delete()
logging.warning("Google: File deleted")
return True
except:
logging.error("Google: File could not be deleted")
return False
def remaining_storage(self):
info = self.client.GetAbout()
total_bytes = int(info['quotaBytesTotal'])
used_bytes = int(info['quotaBytesUsed'])
remaining_bytes = total_bytes - used_bytes
getcontext().prec = 3
gb_val = Decimal(remaining_bytes) / Decimal(1073741824)
return gb_val
def check_connection(self):
return self.connected
def test_01_About_Request(self):
drive = GoogleDrive(self.ga)
about_object = drive.GetAbout()
self.assertTrue(about_object is not None, "About object not loading.")
class ShareManager:
def __init__(self):
self._drive = None
self._folder_dict = {}
def list(self, output_path=None):
if self._drive is None:
self._get_drive()
email = self._drive.GetAbout()['user']['emailAddress']
items = self._drive.ListFile({
'q':
f"'{email}' in owners and trashed=false and 'me' in owners"
}).GetList()
items = [item for item in items if self._is_shared_by_me(item)]
headers = ['id', 'path', 'type', 'READER', 'COMMENTER', 'EDITOR']
dataset = {header: [] for header in headers}
for item in items:
dataset['id'].append(item['id'])
dataset['path'].append(self._get_path(item))
dataset['type'].append(
'folder' if item['mimeType'] ==
'application/vnd.google-apps.folder' else 'file')
permission = self._get_permission(item)
dataset['READER'].append(','.join(permission[0]))
dataset['COMMENTER'].append(','.join(permission[1]))
dataset['EDITOR'].append(','.join(permission[2]))
df = pd.DataFrame(dataset)
if output_path is None:
output_path = f'shared_{datetime.datetime.now().strftime("%m%d%Y_%H%M%S")}.csv'
df.to_csv(output_path, index=False)
def edit(self, input_path):
if self._drive is None:
self._get_drive()
df = pd.read_csv(input_path)
ids = set(df['id'])
for _id in ids:
row = df.loc[df['id'] == _id]
if len(row) != 1:
print(f"skipping {row['name']}: more than one entry exists")
continue
item = self._drive.CreateFile({'id': _id})
final_permission = {}
for role in list(Roles):
users = row[role.name].values[0]
users = [] if type(users) is not str else users.split(',')
for user in users:
final_permission[user] = role
current_permission = self._get_permission(item)
print(current_permission)
for i, role in enumerate(
[Roles.READER, Roles.COMMENTER, Roles.EDITOR]):
for user in current_permission[i]:
if final_permission.get(user, None) is None:
self._remove_permission(_id, user)
logging.log(10,
f'remove permission from {role}: {user}')
elif final_permission[user].value < role.value:
self._remove_permission(_id, user)
self._insert_permission(_id, user,
final_permission[user])
logging.log(
10,
f'change permission from {role} to {final_permission[user]}: {user}'
)
elif final_permission[user].value > role.value:
logging.log(
10,
f'cannot change permission from {role} to {final_permission.get(user, None)}: {user}'
)
def _remove_permission(self, _id, user):
file = self._drive.CreateFile({"id": _id})
permissions = file.GetPermissions()
permission = list(
filter(
lambda x: True
if (x.get('emailAddress', '') == user or 'anyone' in x.get(
'id', '')) else False, permissions))[0]
file.DeletePermission(permission['id'])
logging.log(10, f'unshared {file["id"]} from {user}')
def _insert_permission(self, _id, user, role):
file = self._drive.CreateFile({"id": _id})
permission = {
'type': 'anyone' if user == 'anyoneWithLink' else 'user',
'value': 'anyone' if user == 'anyoneWithLink' else user,
'role': 'writer' if role == Roles.EDITOR else 'reader'
}
if role == Roles.COMMENTER:
permission['additionalRoles'] = ['commenter']
file.InsertPermission(permission)
def _is_shared_by_me(self, item):
if not item['shared']:
return False
try:
self._get_path(item)
except IndexError:
logging.log(10,
f"{item['title']} is not in My Drive") # 10 = DEBUG
return False
return True
def _get_path(self, item):
path = self._folder_dict.get(item['id'], False)
if path is not False:
return path
elif item['parents'][0]['isRoot']:
path = f'ROOT/{item["title"]}'
if 'folder' in item['mimeType']:
self._folder_dict[item['id']] = path
return path
parent = self._drive.CreateFile({'id': item['parents'][0]['id']})
path = f'{self._get_path(parent)}/{item["id"]}'
if 'folder' in item['mimeType']:
self._folder_dict[item['id']] = path
return path
def _get_drive(self):
if self._drive is not None:
return
token = "token.json"
gauth = GoogleAuth()
gauth.LoadCredentialsFile(token)
if gauth.access_token_expired:
gauth.LocalWebserverAuth()
gauth.SaveCredentialsFile(token)
self._drive = GoogleDrive(gauth)
def _get_permission(self, item):
email = self._drive.GetAbout()['user']['emailAddress']
permission = {Roles.READER: [], Roles.EDITOR: [], Roles.COMMENTER: []}
item.FetchMetadata(fields='permissions')
for user in item['permissions']:
if user.get('emailAddress', None) == email:
continue
permission[self._get_role(user)].append(
user.get('emailAddress', 'anyoneWithLink'))
return permission[Roles.READER], permission[
Roles.COMMENTER], permission[Roles.EDITOR]
@staticmethod
def _get_role(user):
if user['role'] == 'writer':
return Roles.EDITOR
elif user.get('additionalRoles', False):
return Roles.COMMENTER
elif user['role'] == 'reader':
return Roles.READER
return ''
