class Provider(StorageABC):
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh4.yaml"):
super().__init__(service=service, config=config)
self.config = Config()
self.storage_credentials = self.config.credentials("storage", "gdrive")
if self.storage_credentials['maxfiles'] > 1000:
Console.error("Page size must be smaller than 1000")
sys.exit(1)
self.limitFiles = self.storage_credentials['maxfiles']
self.scopes = self.storage_credentials['scopes']
self.clientSecretFile = path_expand(
self.storage_credentials['location_secret'])
self.applicationName = self.storage_credentials['application_name']
self.generate_key_json()
self.flags = self.generate_flags_json()
self.credentials = self.get_credentials()
self.http = self.credentials.authorize(httplib2.Http())
self.driveService = discovery.build('drive', 'v3', http=self.http)
self.cloud = service
self.service = service
self.fields = "nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
def generate_flags_json(self):
#
# TODO: Bug, argparse is not used, we use docopts.
#
args = argparse.Namespace(
auth_host_name=self.storage_credentials["auth_host_name"],
auth_host_port=self.storage_credentials["auth_host_port"],
logging_level='ERROR',
noauth_local_webserver=False)
return args
def generate_key_json(self):
config_path = self.storage_credentials['location_secret']
path = Path(path_expand(config_path)).resolve()
config_folder = os.path.dirname(path)
if not os.path.exists(config_folder):
os.makedirs(config_folder)
data = {
"installed": {
"client_id":
self.storage_credentials["client_id"],
"project_id":
self.storage_credentials["project_id"],
"auth_uri":
self.storage_credentials["auth_uri"],
"token_uri":
self.storage_credentials["token_uri"],
"client_secret":
self.storage_credentials["client_secret"],
"auth_provider_x509_cert_url":
self.storage_credentials["auth_provider_x509_cert_url"],
"redirect_uris":
self.storage_credentials["redirect_uris"]
}
}
with open(self.clientSecretFile, 'w') as fp:
json.dump(data, fp)
def get_credentials(self):
"""
We have stored the credentials in ".credentials"
folder and there is a file named 'google-drive-credentials.json'
that has all the credentials required for our authentication
If there is nothing stored in it this program creates credentials
json file for future authentication
Here the authentication type is OAuth2
:return:
:rtype:
"""
cwd = self.storage_credentials['location_gdrive_credentials']
path = Path(path_expand(cwd)).resolve()
if not os.path.exists(path):
os.makedirs(path)
credentials_path = os.path.join(path, 'google-drive-credentials.json')
credentials_path = Path(path_expand(credentials_path)).resolve()
store = Storage(credentials_path)
print(credentials_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(self.client_secret_file,
self.scopes)
flow.user_agent = self.application_name
if self.flags:
credentials = tools.run_flow(flow, store, self.flags)
return credentials
def put(self,
service=None,
source=None,
destination=None,
recursive=False):
if recursive:
if os.path.isdir(source):
temp_res = []
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
for f in os.listdir(source):
if os.path.isfile(os.path.join(source, f)):
temp_res.append(
self.upload_file(source=source,
filename=f,
parent_it=file_parent_id))
return self.update_dict(tempres)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
res = self.upload_file(source=None,
filename=source,
parent_it=file_parent_id)
return self.update_dict(res)
else:
if os.path.isdir(source):
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
temp_res = []
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
for f in os.listdir(source):
if os.path.isfile(os.path.join(source, f)):
temp_res.append(
self.upload_file(source=source,
filename=f,
parent_it=file_parent_id))
return self.update_dict(temp_res)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
res = self.upload_file(source=None,
filename=source,
parent_it=file_parent_id)
return self.update_dict(res)
def get(self,
service=None,
source=None,
destination=None,
recursive=False):
if not os.path.exists(source):
os.makedirs(source)
if recursive:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
print(sourceid)
if len(sourceid) == 0:
Console.error('No files found')
sys.exit(1)
file_id = sourceid['files'][0]['id']
file_name = sourceid['files'][0]['name']
mime_type = sourceid['files'][0]['mimeType']
tempres = []
if mime_type == 'application/vnd.google-apps.folder':
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
for item in items:
if item['mimeType'] != 'application/vnd.google-apps.folder':
self.download_file(source, item['id'], item['name'],
item['mimeType'])
tempres.append(item)
else:
self.download_file(source, file_id, file_name, mime_type)
tempres.append(sourceid['files'][0])
return self.update_dict(tempres)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
print(sourceid)
if len(sourceid) == 0:
Console.error('No files found')
sys.exit(1)
file_id = sourceid['files'][0]['id']
file_name = sourceid['files'][0]['name']
mime_type = sourceid['files'][0]['mimeType']
tempres = []
if mime_type == 'application/vnd.google-apps.folder':
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
for item in items:
if item['mimeType'] != 'application/vnd.google-apps.folder':
self.download_file(source, item['id'], item['name'],
item['mimeType'])
tempres.append(item)
else:
self.download_file(source, file_id, file_name, mime_type)
tempres.append(sourceid['files'][0])
return self.update_dict(tempres)
def delete(self,
service=None,
filename=None,
recursive=False): # this is working
file_id = ""
file_rec = None
if recursive:
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = items['files']
for i in range(len(items)):
if items[i]['name'] == filename:
file_rec = items[i]
file_id = items[i]['id']
try:
self.driveService.files().delete(fileId=file_id).execute()
except: # errors.HttpError, error:
Console.error('No file found')
return 'No file found'
else:
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = items['files']
for i in range(len(items)):
if items[i]['name'] == filename:
file_rec = items[i]
file_id = items[i]['id']
try:
self.driveService.files().delete(fileId=file_id).execute()
except: # errors.HttpError, error:
Console.error('No file found')
return 'No file found'
return self.update_dict(file_rec)
def create_dir(self, service=None, directory=None):
folders, filename = self.cloud_path(directory)
id = None
files = []
for folder in folders:
if id is None:
file_metadata = {
'name': folder,
'mimeType': 'application/vnd.google-apps.folder'
}
else:
file_metadata = {
'name': folder,
'mimeType': 'application/vnd.google-apps.folder',
'parents': [id]
}
file = self.driveService.files().create(
body=file_metadata,
fields=
'id, name, mimeType, parents, size, modifiedTime, createdTime'
).execute()
files.append(file)
print('Folder ID: %s' % file.get('id'))
id = file.get('id')
return self.update_dict(files)
def list(self, service=None, source=None, recursive=False):
if recursive:
results = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = results.get('files', [])
if not items:
Console.error('No files found')
print('No files found.')
else:
return self.update_dict(items)
else:
query_params = "name='" + source + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
file_id = sourceid['files'][0]['id']
query_params = "'" + file_id + "' in parents"
results = self.driveService.files().list(
q=query_params,
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = results.get('files', [])
print(items)
if not items:
Console.error('No files found')
print('No files found.')
else:
return self.update_dict(items)
def search(self,
service=None,
directory=None,
filename=None,
recursive=False):
if recursive:
found = False
res_file = None
list_of_files = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
for file in list_of_files['files']:
print(file)
if file['name'] == filename:
res_file = file
found = True
break
else:
continue
return self.update_dict(res_file)
else:
found = False
list_of_files = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime, createdTime)"
).execute()
for file in list_of_files['files']:
print(file)
if file['name'] == filename:
res_file = file
found = True
break
else:
continue
return self.update_dict(res_file)
def upload_file(self, source, filename, parent_it):
file_metadata = {'name': filename, 'parents': [parent_it]}
self.driveService = self.driveService
if source is None:
filepath = filename
else:
filepath = source + '/' + filename
media = MediaFileUpload(filepath,
mimetype=mimetypes.guess_type(filename)[0])
file = self.driveService.files().create(
body=file_metadata,
media_body=media,
fields='id, name, mimeType, parents,size,modifiedTime,createdTime'
).execute()
return file
def download_file(self, source, file_id, file_name, mime_type):
filepath = source + '/' + file_name + mimetypes.guess_extension(
mime_type)
request = self.driveService.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
return filepath
def cloud_path(self, srv_path):
# Internal function to determine if the cloud path specified is file or folder or mix
b_folder = []
b_file = None
src_file = srv_path
if srv_path.startswith('/'):
src_file = srv_path[1:]
arr_folders = src_file.split('/')
if '.' in arr_folders[-1]:
return arr_folders[0:-1], arr_folders[-1]
else:
return arr_folders, None
def update_dict(self, elements):
if elements is None:
return None
elif type(elements) is list:
_elements = elements
else:
_elements = [elements]
d = []
for element in _elements:
entry = element
entry["cm"] = {
"kind": "storage",
"cloud": 'gdrive',
"name": element['name'],
}
for c in ['modifiedTime', 'createdTime', 'size']:
if c in entry.keys():
entry['cm'][c] = entry[c]
else:
entry['cm'][c] = None
for p in [
'id', 'name', 'mimeType', 'parents', 'createdTime', 'size',
'modifiedTime'
]:
if p in entry.keys():
del (entry[p])
d.append(entry)
return d
class Provider(StorageABC):
def __init__(self, service='gdrive', config="~/.cloudmesh/cloudmesh.yaml"):
super(Provider, self).__init__(service=service, config=config)
self.limitFiles = 1000
self.scopes = 'https://www.googleapis.com/auth/drive'
# This is max value that can be set according to gdrive api
self.clientSecretFile = path_expand(
'~/.cloudmesh/gdrive/client_secret.json')
self.applicationName = 'Drive API Python Quickstart'
self.config = Config()
print(os.path.abspath('~/.cloudmesh/cloudmesh4.yml'))
self.generate_key_json()
self.flags = self.generate_flags_json()
self.authInst = Authentication(self.scopes,
self.clientSecretFile,
self.applicationName,
flags=self.flags)
self.credentials = self.authInst.get_credentials()
self.http = self.credentials.authorize(httplib2.Http())
self.driveService = discovery.build('drive', 'v3', http=self.http)
self.size = None
self.cloud = service
self.service = service
def generate_flags_json(self):
credentials = self.config.credentials("storage", "gdrive")
args = argparse.Namespace(auth_host_name=credentials["auth_host_name"],
auth_host_port=credentials["auth_host_port"],
logging_level='ERROR',
noauth_local_webserver=False)
return args
def generate_key_json(self):
credentials = self.config.credentials("storage", "gdrive")
config_path = '~/.cloudmesh/gdrive/client_secret.json'
path = Path(path_expand(config_path)).resolve()
config_folder = os.path.dirname(path)
if not os.path.exists(config_folder):
os.makedirs(config_folder)
data = {
"installed": {
"client_id":
credentials["client_id"],
"project_id":
credentials["project_id"],
"auth_uri":
credentials["auth_uri"],
"token_uri":
credentials["token_uri"],
"client_secret":
credentials["client_secret"],
"auth_provider_x509_cert_url":
credentials["auth_provider_x509_cert_url"],
"redirect_uris":
credentials["redirect_uris"]
}
}
with open(self.clientSecretFile, 'w') as fp:
json.dump(data, fp)
def put(self,
service=None,
source=None,
destination=None,
recursive=False):
if recursive:
if os.path.isdir(source):
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
for f in os.listdir(source):
if os.path.isfile(os.path.join(source, f)):
self.upload_file(source=source,
filename=f,
parent_it=file_parent_id)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
return self.upload_file(source=None,
filename=source,
parent_it=file_parent_id)
else:
if os.path.isdir(source):
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
for f in os.listdir(source):
if os.path.isfile(os.path.join(source, f)):
self.upload_file(source=source,
filename=f,
parent_it=file_parent_id)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
return self.upload_file(source=None,
filename=source,
parent_it=file_parent_id)
def get(self,
service=None,
source=None,
destination=None,
recursive=False):
if not os.path.exists(source):
os.makedirs(source)
if recursive:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute()
print(sourceid)
file_id = sourceid['files'][0]['id']
file_name = sourceid['files'][0]['name']
mime_type = sourceid['files'][0]['mimeType']
if mime_type == 'application/vnd.google-apps.folder':
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields="nextPageToken, files(id, name,mimeType)").execute(
)
for item in items:
if item['mimeType'] != 'application/vnd.google-apps.folder':
print("dbsakjdjksa")
print(item['mimeType'])
return self.download(source, item['id'], item['name'],
item['mimeType'])
else:
return self.download(source, file_id, file_name, mime_type)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute()
print(sourceid)
file_id = sourceid['files'][0]['id']
file_name = sourceid['files'][0]['name']
mime_type = sourceid['files'][0]['mimeType']
if mime_type == 'application/vnd.google-apps.folder':
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields="nextPageToken, files(id, name,mimeType)").execute(
)
for item in items:
if item['mimeType'] != 'application/vnd.google-apps.folder':
print("dbsakjdjksa")
print(item['mimeType'])
return self.download_file(source, item['id'],
item['name'],
item['mimeType'])
else:
return self.download_file(source, file_id, file_name,
mime_type)
def delete(self,
service='gdrive',
filename=None,
recursive=False): # this is working
file_id = ""
if recursive:
items = Provider.list(self, recursive=True)
for i in range(len(items)):
if items[i]['name'] == filename:
file_id = items[i]['id']
try:
self.driveService.files().delete(fileId=file_id).execute()
except: # errors.HttpError, error:
return 'An error occurred:' # %s' % error
else:
items = Provider.list(self, recursive=True)
for i in range(len(items)):
if items[i]['name'] == filename:
file_id = items[i]['id']
try:
self.driveService.files().delete(fileId=file_id).execute()
except: # errors.HttpError, error:
return 'An error occurred:' # %s' % error
return "deleted"
def create_dir(self, service='gdrive', directory=None):
file_metadata = {
'name': directory,
'mimeType': 'application/vnd.google-apps.folder'
}
file = self.driveService.files().create(body=file_metadata,
fields='id').execute()
print('Folder ID: %s' % file.get('id'))
return file
def list(self, service='gdrive', source=None, recursive=False):
if recursive:
results = self.driveService.files().list(
pageSize=self.limitFiles,
fields="nextPageToken, files(id, name,mimeType)").execute()
items = results.get('files', [])
if not items:
print('No files found.')
else:
return items
else:
query_params = "name='" + source + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
pageSize=self.limitFiles,
fields="nextPageToken, files(id)").execute()
file_id = sourceid['files'][0]['id']
query_params = "'" + file_id + "' in parents"
results = self.driveService.files().list(
q=query_params,
pageSize=self.limitFiles,
fields="nextPageToken, files(id, name, mimeType)").execute()
items = results.get('files', [])
print(items)
if not items:
print('No files found.')
else:
return items
def search(self,
service='gdrive',
directory=None,
filename=None,
recursive=False):
if recursive:
found = False
list_of_files = self.driveService.files().list(
pageSize=self.limitFiles,
fields="nextPageToken, files(id, name,mimeType)").execute()
print(list_of_files)
for file in list_of_files:
print(file)
if file['name'] == filename:
found = True
break
else:
continue
return found
else:
found = False
list_of_files = self.driveService.files().list(
pageSize=self.limitFiles,
fields="nextPageToken, files(id, name,mimeType)").execute()
print(list_of_files)
for file in list_of_files:
print(file)
if file['name'] == filename:
found = True
break
else:
continue
return found
def upload_file(self, source, filename, parent_it):
file_metadata = {'name': filename, 'parents': [parent_it]}
self.driveService = self.driveService
if source is None:
filepath = filename
else:
filepath = source + '/' + filename
media = MediaFileUpload(filepath,
mimetype=mimetypes.guess_type(filename)[0])
file = self.driveService.files().create(body=file_metadata,
media_body=media,
fields='id').execute()
return file
def download_file(self, source, file_id, file_name, mime_type):
filepath = source + '/' + file_name + mimetypes.guess_extension(
mime_type)
request = self.driveService.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
return filepath
class Provider(object):
def __init__(self):
self.config = Config()
credentials = self.config.credentials("storage", "box")
self.sdk = JWTAuth.from_settings_file(credentials['config_path'])
self.client = Client(self.sdk)
def put(self, filename, folder):
folders = self.client.search().query(folder, type='folder')
if len(folders) > 0:
folder_id = folders[0].id
items = self.client.folder(folder_id).get_items()
if not any(item.name == filename for item in items):
file = self.client.folder(folder_id).upload_file(filename)
else:
file_ind = next((index for (index, item) in enumerate(items)
if item.name == filename), None)
file_id = folders[file_ind].id
file = self.client.file(file_id).update_contents(filename)
return file.__dict__
else:
print("Folder not found.")
def get(self, filename, destination):
items = self.client.search().query(filename, type='file')
if len(items) > 0 & items[0].name == filename:
file_id = items[0].id
file = self.client.file(file_id).get()
with open(destination + "/" + filename, 'wb') as f:
self.client.file(file_id).download_to(f)
return file.__dict__
else:
print("File not found.")
def delete(self, filename):
items = self.client.search().query(filename, type='file')
if len(items) > 0 & items[0].name == filename:
file_id = items[0].id
self.client.file(file_id).delete()
else:
print("File not found.")
def size(self, filename):
items = self.client.search().query(filename,
type='file',
fields=['size'])
if len(items) > 0:
return items[0].__dict__
else:
print("File not found.")
def info(self, filename):
items = self.client.search().query(filename, type='file')
if len(items) > 0:
file_id = items[0].id
file = self.client.file(file_id).get()
return file.__dict__
else:
print("File not found.")
def sync(self, source, dest):
files = []
for file in os.listdir(source):
self.put(file, dest)
files.append(file)
return files
def search(self, filename):
items = self.client.search().query(filename, type='file')
files = []
for item in items:
files.append(item.__dict__)
return files
def add(self, folder, parent):
folders = self.client.search.query(parent, type='folder')
if len(folders) > 0:
parent = folders[0].id
folder = self.client.folder(parent).create_subfolder(folder)
return folder.__dict__
else:
print("Parent directory not found.")