def setup(kind):
config = "~/.cloudmesh/cloudmesh.yaml"
if kind == "local":
from cloudmesh.storage.provider.local.Provider import \
Provider as LocalProvider
provider = LocalProvider(service=kind)
elif kind == "box":
from cloudmesh.storage.provider.box.Provider import \
Provider as BoxProvider
provider = BoxProvider(service=kind)
elif kind == "gdrive":
from cloudmesh.storage.provider.gdrive.Provider import \
Provider as GdriveProvider
provider = GdriveProvider(service=kind)
elif kind == "azureblob":
from cloudmesh.storage.provider.azureblob.Provider import \
Provider as AzureblobProvider
provider = AzureblobProvider(service=kind)
elif kind == "awss3":
from cloudmesh.storage.provider.awss3.Provider import \
Provider as AwsProvider
provider = AwsProvider(service=kind)
else:
Console.error(f"Storage Provider is not supported: {kind}")
return provider
def setup(kind):
config = "~/.cloudmesh/cloudmesh4.yaml"
if kind == "local":
provider = LocalProvider(service=kind, config=config)
elif kind == "box":
provider = BoxProvider(service=kind, config=config)
elif kind == "gdrive":
provider = GdriveProvider(service=kind, config=config)
elif kind == "azureblob":
provider = AzureblobProvider(service=kind, config=config)
elif kind == "awss3":
provider = AwsProvider(service=kind, config=config)
return provider
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh4.yaml"):
super(Provider, self).__init__(service=service, config=config)
if self.kind == "local":
self.provider = LocalProvider(service=service, config=config)
elif self.kind == "box":
self.provider = BoxProvider(service=service, config=config)
elif self.kind == "gdrive":
self.provider = GdriveProvider(service=service, config=config)
elif self.kind == "azureblob":
self.provider = AzureblobProvider(service=service, config=config)
elif self.kind == "awss3":
self.provider = AwsProvider(service=service, config=config)
elif self.kind == "awsobjectstore":
self.provider = AwsobjectstoreProvider(service=service,
config=config)
else:
raise ValueError(
f"Storage provider '{self.kind}' not yet supported")
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh4.yaml"):
super(Provider, self).__init__(service=service, config=config)
Console.error("PLEASE USE cloudmesh.storage.Provider")
warnings.warn("deprecated: use cloudmesh.storage.Provider instead", DeprecationWarning)
if self.kind == "local":
self.provider = LocalProvider(service=service, config=config)
elif self.kind == "box":
self.provider = BoxProvider(service=service, config=config)
elif self.kind == "gdrive":
self.provider = GdriveProvider(service=service, config=config)
elif self.kind == "azureblob":
self.provider = AzureblobProvider(service=service, config=config)
elif self.kind == "awss3":
self.provider = AwsProvider(service=service, config=config)
else:
raise ValueError(f"Storage provider '{self.kind}' not yet supported")
def __init__(self, service=None, config="~/.cloudmesh/cloudmesh.yaml"):
super(Provider, self).__init__(service=service, config=config)
if self.kind == "local":
from cloudmesh.storage.provider.local.Provider import \
Provider as LocalProvider
self.provider = LocalProvider(service=service, config=config)
elif self.kind == "box":
from cloudmesh.storage.provider.box.Provider import \
Provider as BoxProvider
self.provider = BoxProvider(service=service, config=config)
elif self.kind == "gdrive":
from cloudmesh.storage.provider.gdrive.Provider import \
Provider as GdriveProvider
self.provider = GdriveProvider(service=service, config=config)
elif self.kind == "azureblob":
from cloudmesh.storage.provider.azureblob.Provider import \
Provider as AzureblobProvider
self.provider = AzureblobProvider(service=service, config=config)
elif self.kind == "awss3":
from cloudmesh.storage.provider.awss3.Provider import \
Provider as AwsProvider
self.provider = AwsProvider(service=service, config=config)
elif self.kind in ['google']:
from cloudmesh.google.storage.Provider import \
Provider as GoogleStorageProvider
self.provider = GoogleStorageProvider(service=service,
config=config)
elif self.kind in ['oracle']:
from cloudmesh.oracle.storage.Provider import \
Provider as OracleStorageProvider
self.provider = \
OracleStorageProvider(service=service, config=config)
else:
raise ValueError(
f"Storage provider '{self.service}' not yet supported")
def copy(self, source=None, destination=None, recursive=False):
"""
Copies object(s) from source to destination
:param source: "awss3:source.txt" the source is combination of
source CSP name and source object name which either
can be a directory or file
:param destination: "azure:target.txt" the destination is
combination of destination CSP and destination
object name which either can be a directory or file
:param recursive: in case of directory the recursive refers to all
subdirectories in the specified source
:return: dict
"""
# Fetch CSP names and object names
if source:
source, source_obj = source.split(':')
else:
source, source_obj = None, None
if destination:
target, target_obj = destination.split(':')
else:
target, target_obj = None, None
source_obj.replace(":", "")
# oracle provider expects a target name
if target_obj is None or \
len(target_obj.strip()) == 0:
# print("DEBUG:", Path(source_obj).parts)
target_obj = Path(source_obj).parts[-1]
source_obj = str(Path(source_obj).expanduser())
target_obj = str(Path(target_obj).expanduser())
print("DEBUG Provider: values= ", source, source_obj, target,
target_obj)
if source == "local":
print(f"CALL PUT METHOD OF {self.kind} PROVIDER.")
result = self.provider.put(source=source_obj,
destination=target_obj,
recursive=recursive)
return result
elif target == "local":
print(f"CALL GET METHOD OF {self.kind} PROVIDER.")
result = self.provider.get(source=source_obj,
destination=target_obj,
recursive=recursive)
return result
else:
# VERBOSE(f"Copy from {source} to {destination}.")
target_kind = self.kind
target_provider = self.provider
config = "~/.cloudmesh/cloudmesh.yaml"
if source:
super().__init__(service=source, config=config)
source_kind = self.kind
source_provider = self.get_source_provider(
source_kind, source, config)
# get local storage directory
super().__init__(service="local", config=config)
local_storage = self.config[
"cloudmesh.storage.local.default.directory"]
local_target_obj = str(Path(local_storage).expanduser())
source_obj = str(Path(source_obj).expanduser())
try:
result = source_provider.get(source=source_obj,
destination=local_target_obj,
recursive=recursive)
if result and len(result) == 0:
return Console.error(f"{source_obj} could not be found "
f"in {source} CSP. Please check.")
Console.ok(f"Fetched {source_obj} from {source} CSP")
except Exception as e:
return Console.error(f"Error while fetching {source_obj} from "
f"{source} CSP. Please check. {e}")
else:
source_obj = str(Path(local_target_obj) / source_obj)
try:
result = target_provider.put(source=source_obj,
destination=target_obj,
recursive=recursive)
if result is None:
return Console.error(f"{source_obj} couldn't be copied"
f" to {target} CSP.")
Console.ok(f"Copied {source_obj} to {target} CSP")
return result
except Exception as e:
return Console.error(
f"Error while copying {source_obj} to "
f"{target} CSP. Please check,{e}")
finally:
Console.info("\nRemoving local intermediate file.")
from cloudmesh.storage.provider.local.Provider import \
Provider as LocalProvider
local_provider = LocalProvider(service="local",
config=config)
try:
local_provider.delete(source_obj, recursive=recursive)
except Exception as e:
Console.error(
f"{source_obj} could not be deleted from "
f"local storage.")