class Provider(StorageQueue):
kind = "parallelgdrive"
sample = "TODO: missing"
status = ['completed', 'waiting', 'inprogress', 'canceled']
output = {} # "TODO: missing"
def __init__(self,
service=None,
config="~/.cloudmesh/cloudmesh.yaml",
parallelism=4):
super().__init__(service=service, parallelism=parallelism)
self.config = Config()
self.storage_credentials = self.config.credentials(
"storage", "parallelgdrive")
self.credentials_json_path = self.storage_credentials[
'credentials_json_path']
self.token_path = self.storage_credentials['token_path']
creds = None
# Temporarily change directory to token_path specified in cloudmesh.yaml
# This is to use the credentials.json file and token.pickle
# Assuming credentials.json file and token.pickle will be in same directory in cloudmesh.yaml
# Get current working directory
cwd = os.getcwd()
os.chdir(self.token_path)
if os.path.exists('token.pickle'):
with open('token.pickle', 'rb') as token:
creds = pickle.load(token)
# If there are no (valid) credentials available, let the user log in.
if not creds or not creds.valid:
if creds and creds.expired and creds.refresh_token:
creds.refresh(Request())
else:
flow = InstalledAppFlow.from_client_secrets_file(
'credentials.json', SCOPES)
creds = flow.run_local_server(port=0)
# Save the credentials for the next run
with open('token.pickle', 'wb') as token:
pickle.dump(creds, token)
# Change directory back to previous working directory
os.chdir(cwd)
service = build('drive', 'v3', credentials=creds)
self.service = service
def list_run(self, specification): # in mongdb, but can't run
source = specification['path']
dir_only = specification['dir_only']
recursive = specification['recursive']
if recursive:
results = self.service.files().list(
# pageSize=self.limitFiles,
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = results.get('files', [])
if not items:
Console.error('No files found')
print('No files found.')
# else:
# return self.update_dict(items)
else:
query_params = "name='" + source + "' and trashed=false"
sourceid = self.service.files().list(
q=query_params,
# pageSize=self.limitFiles,
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
file_id = sourceid['files'][0]['id']
query_params = "'" + file_id + "' in parents"
results = self.service.files().list(
q=query_params,
# pageSize=self.limitFiles,
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = results.get('files', [])
print(items)
if not items:
Console.error('No files found')
print('No files found.')
#else:
# return self.update_dict(items)
specification['status'] = 'completed'
return specification
def put_run(self, specification):
source = specification['source']
destination = specification['destination']
recursive = specification['recursive']
if recursive:
if os.path.isdir(source):
temp_res = []
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.service.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir_helper(directory=destination)
# file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
for f in os.listdir(source):
if os.path.isfile(os.path.join(source, f)):
temp_res.append(
self.upload_file(source=source,
filename=f,
parent_it=file_parent_id))
# return self.update_dict(temp_res)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.service.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir_helper(directory=destination)
# file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
# file_parent_id = sourceid['files'][0]['id']
# res = self.upload_file(source=None, filename=source,
# parent_it=file_parent_id)
# return self.update_dict(res)
else:
if os.path.isdir(source):
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.service.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
temp_res = []
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir_helper(directory=destination)
file_parent_id = parent_file[0].get('id')
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
for f in os.listdir(source):
if os.path.isfile(os.path.join(source, f)):
temp_res.append(
self.upload_file(source=source,
filename=f,
parent_it=file_parent_id))
# return self.update_dict(temp_res)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.service.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
print("len = ", len(sourceid['files']))
if len(sourceid['files']) == 0:
#parent_file = self.create_dir_helper(directory=destination)
#file_parent_id = parent_file['id']
res = self.upload_file(source=None,
filename=source,
parent_it=None)
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
res = self.upload_file(source=None,
filename=source,
parent_it=file_parent_id)
specification['status'] = 'completed'
return specification
# def get(self, service=None, source=None, destination=None, recursive=False):
def get_run(self, specification):
source = specification['source']
destination = specification['destination']
recursive = specification['recursive']
trimmed_source = massage_path(source)
trimed_dest = massage_path(destination)
if not os.path.exists(source):
os.makedirs(source)
if recursive:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.service.files().list(
q=query_params,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
print(sourceid)
if len(sourceid) == 0:
Console.error('No files found')
sys.exit(1)
file_id = sourceid['files'][0]['id']
file_name = sourceid['files'][0]['name']
mime_type = sourceid['files'][0]['mimeType']
tempres = []
if mime_type == 'application/vnd.google-apps.folder':
query_params = "'" + file_id + "' in parents"
results = self.service.files().list(
q=query_params,
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = results.get('files', [])
print("Items in directory to get: ", items)
for item in items:
print("Type of item", type(item))
print("Item is:", item)
if item['mimeType'] != 'application/vnd.google-apps.folder':
self.download_file(source, item['id'], item['name'],
item['mimeType'])
tempres.append(item)
else:
self.download_file(source, file_id, file_name, mime_type)
tempres.append(sourceid['files'][0])
# return self.update_dict(tempres)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.service.files().list(
q=query_params,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
print(sourceid)
if len(sourceid) == 0:
Console.error('No files found')
sys.exit(1)
file_id = sourceid['files'][0]['id']
file_name = sourceid['files'][0]['name']
mime_type = sourceid['files'][0]['mimeType']
tempres = []
if mime_type == 'application/vnd.google-apps.folder':
query_params = "'" + file_id + "' in parents"
results = self.service.files().list(
q=query_params,
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = results.get('files', [])
print("Items in directory to get: ", items)
for item in items:
print("Type of item", type(item))
print("Item is:", item)
if item['mimeType'] != 'application/vnd.google-apps.folder':
self.download_file(source, item['id'], item['name'],
item['mimeType'])
tempres.append(item)
else:
self.download_file(source, file_id, file_name, mime_type)
tempres.append(sourceid['files'][0])
# return self.update_dict(tempres)
specification['status'] = 'completed'
return specification
def delete_run(
self, specification
): # works, deleted dir and sub-dirs in gdrive, seen in mongodb too
source = specification['path']
recursive = specification['recursive']
file_id = ""
file_rec = None
if recursive:
items = self.service.files().list(
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = items['files']
for i in range(len(items)):
if items[i]['name'] == source:
file_rec = items[i]
file_id = items[i]['id']
try:
self.service.files().delete(fileId=file_id).execute()
except: # errors.HttpError, error:
Console.error('No file found')
return 'No file found'
else:
items = self.service.files().list(
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = items['files']
for i in range(len(items)):
if items[i]['name'] == source:
file_rec = items[i]
file_id = items[i]['id']
try:
self.service.files().delete(fileId=file_id).execute()
except: # errors.HttpError, error:
Console.error('No file found')
return 'No file found'
# return self.update_dict(file_rec)
specification['status'] = 'completed'
return specification
def mkdir_run(self, specification):
directory = specification['path']
folders, filename = self.cloud_path(directory)
id = None
files = []
for folder in folders:
if id is None:
file_metadata = {
'name': folder,
'mimeType': 'application/vnd.google-apps.folder'
}
else:
file_metadata = {
'name': folder,
'mimeType': 'application/vnd.google-apps.folder',
'parents': [id]
}
file = self.service.files().create(
body=file_metadata,
fields=
'id, name, mimeType, parents, size, modifiedTime, createdTime'
).execute()
files.append(file)
print('Folder ID: %s' % file.get('id'))
id = file.get('id')
specification['status'] = 'completed'
return specification
def create_dir_helper(self, service=None, directory=None):
folders, filename = self.cloud_path(directory)
id = None
files = []
for folder in folders:
if id is None:
file_metadata = {
'name': folder,
'mimeType': 'application/vnd.google-apps.folder'
}
else:
file_metadata = {
'name': folder,
'mimeType': 'application/vnd.google-apps.folder',
'parents': [id]
}
file = self.service.files().create(
body=file_metadata,
fields=
'id, name, mimeType, parents, size, modifiedTime, createdTime'
).execute()
files.append(file)
print('Folder ID: %s' % file.get('id'))
id = file.get('id')
return files
# def search(self, service=None, directory=None, filename=None,
# recursive=False):
def search_run(self, specification):
directory = specification['path']
filename = specification['filename']
recursive = specification['recursive']
if recursive:
found = False
res_file = None
list_of_files = self.service.files().list(
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
for file in list_of_files['files']:
# print(file)
if file['name'] == filename:
print(file)
res_file = file
found = True
break
else:
continue
# return self.update_dict(res_file)
else:
found = False
list_of_files = self.service.files().list(
pageSize=100,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime, createdTime)"
).execute()
for file in list_of_files['files']:
# print(file)
if file['name'] == filename:
print(file)
res_file = file
found = True
break
else:
continue
# return self.update_dict(res_file)
specification['status'] = 'completed'
return specification
def upload_file(self, source, filename, parent_it):
file_metadata = {'name': filename, 'parents': [parent_it]}
self.service = self.service
if source is None:
filepath = filename
else:
filepath = source + '/' + filename
media = MediaFileUpload(filepath,
mimetype=mimetypes.guess_type(filename)[0])
file = self.service.files().create(
body=file_metadata,
media_body=media,
fields='id, name, mimeType, parents,size,modifiedTime,createdTime'
).execute()
return file
def download_file(self, source, file_id, file_name, mime_type):
filepath = source + '/' + file_name # removed mime_type
request = self.service.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
return filepath
def cloud_path(self, srv_path):
# Internal function to determine if the cloud path specified is file or folder or mix
b_folder = []
b_file = None
src_file = srv_path
if srv_path.startswith('/'):
src_file = srv_path[1:]
arr_folders = src_file.split('/')
if '.' in arr_folders[-1]:
return arr_folders[0:-1], arr_folders[-1]
else:
return arr_folders, None
class Provider(StorageABC):
kind = "gdrive"
sample = "TODO: missing"
output = {} # "TODO: missing"
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml"):
super().__init__(service=service, config=config)
self.config = Config()
self.storage_credentials = self.config.credentials("storage", "gdrive")
if self.storage_credentials['maxfiles'] > 1000:
Console.error("Page size must be smaller than 1000")
sys.exit(1)
self.limitFiles = self.storage_credentials['maxfiles']
self.scopes = self.storage_credentials['scopes']
self.clientSecretFile = path_expand(
self.storage_credentials['location_secret'])
self.applicationName = self.storage_credentials['application_name']
self.generate_key_json()
self.flags = self.generate_flags_json()
self.credentials = self.get_credentials()
self.http = self.credentials.authorize(httplib2.Http())
self.driveService = discovery.build('drive', 'v3', http=self.http)
self.cloud = service
self.service = service
self.fields = "nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
def generate_flags_json(self):
#
# TODO: Bug, argparse is not used, we use docopts.
#
args = argparse.Namespace(
auth_host_name=self.storage_credentials["auth_host_name"],
auth_host_port=self.storage_credentials["auth_host_port"],
logging_level='ERROR',
noauth_local_webserver=False)
return args
def generate_key_json(self):
config_path = self.storage_credentials['location_secret']
path = Path(path_expand(config_path)).resolve()
config_folder = os.path.dirname(path)
if not os.path.exists(config_folder):
os.makedirs(config_folder)
data = {
"installed": {
"client_id":
self.storage_credentials["client_id"],
"project_id":
self.storage_credentials["project_id"],
"auth_uri":
self.storage_credentials["auth_uri"],
"token_uri":
self.storage_credentials["token_uri"],
"client_secret":
self.storage_credentials["client_secret"],
"auth_provider_x509_cert_url":
self.storage_credentials["auth_provider_x509_cert_url"],
"redirect_uris":
self.storage_credentials["redirect_uris"]
}
}
with open(self.clientSecretFile, 'w') as fp:
json.dump(data, fp)
def get_credentials(self):
"""
We have stored the credentials in ".credentials"
folder and there is a file named 'google-drive-credentials.json'
that has all the credentials required for our authentication
If there is nothing stored in it this program creates credentials
json file for future authentication
Here the authentication type is OAuth2
:return:
:rtype:
"""
cwd = self.storage_credentials['location_gdrive_credentials']
path = Path(path_expand(cwd)).resolve()
if not os.path.exists(path):
os.makedirs(path)
credentials_path = os.path.join(path, 'google-drive-credentials.json')
credentials_path = Path(path_expand(credentials_path)).resolve()
store = Storage(credentials_path)
print(credentials_path)
credentials = store.get()
if not credentials or credentials.invalid:
flow = client.flow_from_clientsecrets(self.client_secret_file,
self.scopes)
flow.user_agent = self.application_name
if self.flags:
credentials = tools.run_flow(flow, store, self.flags)
return credentials
def put(self, source=None, destination=None, recursive=False):
if recursive:
if os.path.isdir(source):
temp_res = []
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
for f in os.listdir(source):
if os.path.isfile(os.path.join(source, f)):
temp_res.append(
self.upload_file(source=source,
filename=f,
parent_it=file_parent_id))
return self.update_dict(temp_res)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
res = self.upload_file(source=None,
filename=source,
parent_it=file_parent_id)
return self.update_dict(res)
else:
if os.path.isdir(source):
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
temp_res = []
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
for f in os.listdir(source):
if os.path.isfile(os.path.join(source, f)):
temp_res.append(
self.upload_file(source=source,
filename=f,
parent_it=file_parent_id))
return self.update_dict(temp_res)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields="nextPageToken, files(id, name, mimeType)").execute(
)
file_parent_id = None
print(sourceid)
if len(sourceid['files']) == 0:
parent_file = self.create_dir(directory=destination)
file_parent_id = parent_file['id']
else:
print(sourceid['files'][0]['id'])
file_parent_id = sourceid['files'][0]['id']
res = self.upload_file(source=None,
filename=source,
parent_it=file_parent_id)
return self.update_dict(res)
def get(self, source=None, destination=None, recursive=False):
if not os.path.exists(source):
os.makedirs(source)
if recursive:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
print(sourceid)
if len(sourceid) == 0:
Console.error('No files found')
sys.exit(1)
file_id = sourceid['files'][0]['id']
file_name = sourceid['files'][0]['name']
mime_type = sourceid['files'][0]['mimeType']
tempres = []
if mime_type == 'application/vnd.google-apps.folder':
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
for item in items:
if item['mimeType'] != 'application/vnd.google-apps.folder':
self.download_file(source, item['id'], item['name'],
item['mimeType'])
tempres.append(item)
else:
self.download_file(source, file_id, file_name, mime_type)
tempres.append(sourceid['files'][0])
return self.update_dict(tempres)
else:
query_params = "name='" + destination + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
print(sourceid)
if len(sourceid) == 0:
Console.error('No files found')
sys.exit(1)
file_id = sourceid['files'][0]['id']
file_name = sourceid['files'][0]['name']
mime_type = sourceid['files'][0]['mimeType']
tempres = []
if mime_type == 'application/vnd.google-apps.folder':
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
for item in items:
if item['mimeType'] != 'application/vnd.google-apps.folder':
self.download_file(source, item['id'], item['name'],
item['mimeType'])
tempres.append(item)
else:
self.download_file(source, file_id, file_name, mime_type)
tempres.append(sourceid['files'][0])
return self.update_dict(tempres)
def delete(self, filename=None, recursive=False): # this is working
file_id = ""
file_rec = None
if recursive:
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = items['files']
for i in range(len(items)):
if items[i]['name'] == filename:
file_rec = items[i]
file_id = items[i]['id']
try:
self.driveService.files().delete(fileId=file_id).execute()
except: # errors.HttpError, error:
Console.error('No file found')
return 'No file found'
else:
items = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = items['files']
for i in range(len(items)):
if items[i]['name'] == filename:
file_rec = items[i]
file_id = items[i]['id']
try:
self.driveService.files().delete(fileId=file_id).execute()
except: # errors.HttpError, error:
Console.error('No file found')
return 'No file found'
return self.update_dict(file_rec)
def create_dir(self, directory=None):
folders, filename = self.cloud_path(directory)
id = None
files = []
for folder in folders:
if id is None:
file_metadata = {
'name': folder,
'mimeType': 'application/vnd.google-apps.folder'
}
else:
file_metadata = {
'name': folder,
'mimeType': 'application/vnd.google-apps.folder',
'parents': [id]
}
file = self.driveService.files().create(
body=file_metadata,
fields=
'id, name, mimeType, parents, size, modifiedTime, createdTime'
).execute()
files.append(file)
print('Folder ID: %s' % file.get('id'))
id = file.get('id')
return self.update_dict(files)
def list(self, source=None, recursive=False):
if recursive:
results = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = results.get('files', [])
if not items:
Console.error('No files found')
print('No files found.')
else:
return self.update_dict(items)
else:
query_params = "name='" + source + "' and trashed=false"
sourceid = self.driveService.files().list(
q=query_params,
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
file_id = sourceid['files'][0]['id']
query_params = "'" + file_id + "' in parents"
results = self.driveService.files().list(
q=query_params,
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
items = results.get('files', [])
print(items)
if not items:
Console.error('No files found')
print('No files found.')
else:
return self.update_dict(items)
def search(self, directory=None, filename=None, recursive=False):
if recursive:
found = False
res_file = None
list_of_files = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime,createdTime)"
).execute()
for file in list_of_files['files']:
print(file)
if file['name'] == filename:
res_file = file
found = True
break
else:
continue
return self.update_dict(res_file)
else:
found = False
list_of_files = self.driveService.files().list(
pageSize=self.limitFiles,
fields=
"nextPageToken, files(id, name, mimeType, parents,size,modifiedTime, createdTime)"
).execute()
for file in list_of_files['files']:
print(file)
if file['name'] == filename:
res_file = file
found = True
break
else:
continue
return self.update_dict(res_file)
def upload_file(self, source, filename, parent_it):
file_metadata = {'name': filename, 'parents': [parent_it]}
self.driveService = self.driveService
if source is None:
filepath = filename
else:
filepath = source + '/' + filename
media = MediaFileUpload(filepath,
mimetype=mimetypes.guess_type(filename)[0])
file = self.driveService.files().create(
body=file_metadata,
media_body=media,
fields='id, name, mimeType, parents,size,modifiedTime,createdTime'
).execute()
return file
def download_file(self, source, file_id, file_name, mime_type):
filepath = source + '/' + file_name + mimetypes.guess_extension(
mime_type)
request = self.driveService.files().get_media(fileId=file_id)
fh = io.BytesIO()
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
status, done = downloader.next_chunk()
print("Download %d%%." % int(status.progress() * 100))
with io.open(filepath, 'wb') as f:
fh.seek(0)
f.write(fh.read())
return filepath
def cloud_path(self, srv_path):
# Internal function to determine if the cloud path specified is file or folder or mix
b_folder = []
b_file = None
src_file = srv_path
if srv_path.startswith('/'):
src_file = srv_path[1:]
arr_folders = src_file.split('/')
if '.' in arr_folders[-1]:
return arr_folders[0:-1], arr_folders[-1]
else:
return arr_folders, None
def update_dict(self, elements):
if elements is None:
return None
elif type(elements) is list:
_elements = elements
else:
_elements = [elements]
d = []
for element in _elements:
entry = element
entry["cm"] = {
"kind": "storage",
"cloud": 'gdrive',
"name": element['name'],
}
for c in ['modifiedTime', 'createdTime', 'size']:
if c in entry.keys():
entry['cm'][c] = entry[c]
else:
entry['cm'][c] = None
for p in [
'id', 'name', 'mimeType', 'parents', 'createdTime', 'size',
'modifiedTime'
]:
if p in entry.keys():
del (entry[p])
d.append(entry)
return d
class Provider(StorageABC):
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml"):
super().__init__(service=service, config=config)
self.config = Config()
self.storage_credentials = self.config.credentials("storage", "gdrive")
google_json = self.config["cloudmesh"]["storage"]["gdrive"]["default"][
"service_account"]
self.google_client = storage.Client.from_service_account_json(
google_json)
self.local_dir = self.config["cloudmesh"]["storage"]["local"]["dir"]
self.bucket = self.config["cloudmesh"]["storage"]["gdrive"]["default"][
"bucket"]
def list(self, source=None):
bucket = self.google_client.bucket(self.bucket)
keys = []
match = []
for blob in bucket.list_blobs():
datetimeObj = blob.time_created
timestampStr = datetimeObj.strftime("%d-%b-%Y %H:%M:%S")
keys.append("Name: " + blob.name + " , Created: " + timestampStr)
#pprint(blob.name)
if (blob.name).startswith(source):
match.append("Name: " + blob.name + " , Created: " +
timestampStr)
if match is not None:
pprint(f"Listing from source - {source}")
pprint(match)
else:
Console.ok(
f"{source} cannot be found in Google bucket{self.bucket}")
pprint(f"Listing contents from bucket - {self.bucket}")
pprint(keys)
return keys
def download_blob(self, source_filename, destination_filename):
bucket = self.google_client.get_bucket(self.bucket)
blob = bucket.blob(source_filename)
blob.download_to_filename(destination_filename)
status = "Success"
#Console.ok(f"Blob {source_filename} downloaded to {destination_filename}.")
return status
def uploadfile(self, source_file, destination_file):
bucket = self.google_client.get_bucket(self.bucket)
blob = bucket.blob(destination_file)
blob.upload_from_filename(source_file)
result = "success"
return result
def copy(self,
source=None,
target=None,
source_file_dir=None,
target_fil_dir=None):
print((source_file_dir, target_fil_dir))
status = None
if (source == "local"):
print("Copying file from Local to Google")
source_path = self.getLocalPath(source_file_dir)
target_fil_dir = target_fil_dir.replace("\\", "/")
status = self.uploadfile(source_path, target_fil_dir)
Console.ok(
f"File copied from {source_file_dir} to {target_fil_dir}")
elif (source == "google"):
if (target == "aws"):
pprint(
f"Copying from Google {source_file_dir} to AWS {target_fil_dir}"
)
target_local = self.getLocalPath(
self.local_dir) + "/" + source_file_dir
self.download_blob(source_file_dir, target_local)
target = AWS_Provider(service="aws")
config = Config(config_path="~/.cloudmesh/cloudmesh.yaml")
status = target.put(target_local, target_fil_dir, True)
if status is None:
return Console.error(
f" Cannot Copy {source_file_dir} to {target_fil_dir}")
else:
Console.ok(
f"File copied from {source_file_dir} to {target_fil_dir}"
)
elif (target == "local"):
#To fix bucket, file
target_fil_dir = self.getLocalPath(target_fil_dir)
status = self.download_blob(source_file_dir, target_fil_dir)
Console.ok(
f"File copied from {source_file_dir} to {target_fil_dir}")
else:
NotImplementedError
return status
def getLocalPath(self, fileOrDir):
file_path = Path(fileOrDir).expanduser()
if file_path.is_absolute():
tar_dir = str(file_path)
else:
local_path = Path(self.local_dir).expanduser()
tar_dir = (str(local_path) + "/" + str(file_path)).replace(
"\\", "/")
return tar_dir
def delete(self, source=None):
bucket = self.google_client.bucket(self.bucket)
blob = bucket.blob(source)
blob.delete()
Console.ok(f"{source} has been deleted from Google - {self.bucket}")
keys = []
for blob in bucket.list_blobs():
keys.append(blob.name)
pprint(blob.name)
return keys
