class TestObjectstorage(object):
def setup(self):
variables = Variables()
self.service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=self.service)
self.sourcedir = path_expand("~/.cloudmesh/storage/test")
print()
def test_list(self):
HEADING()
StopWatch.start("LIST Directory")
contents = self.p.list(self.p.service, "/")
StopWatch.stop("LIST Directory")
for c in contents:
pprint(c)
assert len(contents) > 0
found = False
for entry in contents:
if entry["cm"]["name"] == "a1.txt":
found = True
assert found
def test_results(self):
HEADING()
# storage = self.p.service
service = self.service
banner(f"Benchmark results for {service} Storage")
StopWatch.benchmark()
class TestLocal(object):
def setup_class(self):
# variables = Variables()
# service = Parameter.expand(variables['storage'])[0]
self.service = "local"
self.p = Provider(service=self.service)
def test_00__config(self):
VERBOSE(self.p)
VERBOSE(self.p.kind)
assert self.p.kind == self.service
def test_01_create_source(self):
HEADING()
self.sourcedir = path_expand("~/.cloudmesh/storage/test/")
create_file("~/.cloudmesh/storage/README.md", "content of a")
create_file("~/.cloudmesh/storage/test/a/a.txt", "content of a")
create_file("~/.cloudmesh/storage/test/a/b/b.txt", "content of b")
create_file("~/.cloudmesh/storage/test/a/b/c/c.txt", "content of c")
# test if the files are ok
assert True
def test_02_list(self):
HEADING()
src = '/'
contents = self.p.list(source=src)
VERBOSE(contents, label="c")
for c in contents:
VERBOSE(c)
def test_05_search(self):
HEADING()
src = '/'
filename = 'a.txt'
#
# bug use named arguments
#
files = self.p.search(directory=src, filename=filename, recursive=True)
pprint(files)
assert len(files) > 0
class TestStorageBox(object):
def create_file(self, location, content):
Shell.mkdir(os.dirname(path_expand(location)))
writefile(location, content)
def setup(self):
variables = Variables()
self.service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=self.service)
self.sourcedir = path_expand("~/.cloudmesh/storage/test")
print()
def test_create_source(self):
HEADING()
home = self.sourcedir
# Setup a dir
self.content = []
self.files = [
"a/a1.txt", "a/a2.txt", "a/a3.txt", "a/b/b1.txt", "a/b/b2.txt",
"a/b/b3.txt", "a/b/c/c1.txt", "a/b/c/c2.txt", "a/b/c/c3.txt",
"a/b/d/d1.txt", "a/b/d/d2.txt", "a/b/d/d3.txt", "a/b/d/a1.txt"
]
for f in self.files:
location = f"{home}/{f}"
self.create_file(location, f"content of {f}")
self.content.append(location)
# setup empty dir in a
d1 = Path(path_expand(f"{home}/a/empty"))
d1.mkdir(parents=True, exist_ok=True)
for f in self.files:
assert os.path.isfile(f"{home}/{f}")
assert os.path.isdir(f"{home}/a/empty")
def test_put_and_get(self):
HEADING()
home = self.sourcedir
StopWatch.start("PUT file")
test_file = self.p.put(self.p.service, f"{home}/a/a1.txt", "/")
StopWatch.stop("PUT file")
assert test_file is not None
StopWatch.start("GET file")
test_file = self.p.get(self.p.service, f"/a1.txt", f"{home}/hello.txt")
StopWatch.stop("GET file")
assert test_file is not None
content = readfile(f"{home}/hello.txt")
assert "a1.txt" in content
def test_list(self):
HEADING()
StopWatch.start("LIST Directory")
contents = self.p.list(self.p.service, "/")
StopWatch.stop("LIST Directory")
for c in contents:
pprint(c)
assert len(contents) > 0
found = False
for entry in contents:
if entry["cm"]["name"] == "a1.txt":
found = True
assert found
def test_create_dir(self):
HEADING()
src = '/a/created_dir'
StopWatch.start("CREATE DIR")
directory = self.p.createdir(self.p.service, src)
StopWatch.stop("CREATE DIR")
pprint(directory)
assert dir is not None
assert "a/created_dir" in directory[0]["name"]
def test_search(self):
HEADING()
src = '/'
filename = "a1.txt"
StopWatch.start("SEARCH file")
search_files = self.p.search(self.p.service, src, filename, True)
StopWatch.stop("SEARCH file")
pprint(search_files)
assert len(search_files) > 0
assert search_files[0]["name"] == filename
def test_delete(self):
HEADING()
src = "/a/created_dir"
StopWatch.start("DELETE Directory")
contents = self.p.delete(self.p.service, src)
StopWatch.stop("DELETE Directory")
deleted = False
for entry in contents:
if "created_dir" in entry["cm"]["name"]:
if entry["cm"]["status"] == "deleted":
deleted = True
assert deleted
def test_recursive_put(self):
# must be implemented by student from ~/.cloudmesh/storage/test
# make sure all files are in the list see self.content which contains
# all files
home = self.sourcedir
StopWatch.start("PUT Directory --recursive")
upl_files = self.p.put(self.p.service, f"{home}", "/a", True)
StopWatch.stop("PUT Directory --recursive")
pprint(upl_files)
assert upl_files is not None
def test_recursive_get(self):
# must be implemented by student into ~/.cloudmesh/storage/test/get
# see self.content which contains all files but you must add get/
home = self.sourcedir
d2 = Path(path_expand(f"{home}/get"))
d2.mkdir(parents=True, exist_ok=True)
StopWatch.start("GET Directory --recursive")
dnld_files = self.p.get(self.p.service, "/a", f"{home}/get", True)
StopWatch.stop("GET Directory --recursive")
pprint(dnld_files)
assert dnld_files is not None
def test_recursive_delete(self):
# must be implemented by student into ~/.cloudmesh/storage/test/get
# see self.content which contains all files but you must add get/
src = "/a/a/b/c"
StopWatch.start("DELETE Sub-directory")
del_files = self.p.delete(self.p.service, src)
StopWatch.stop("DELETE Sub-directory")
assert len(del_files) > 0
def test_exhaustive_list(self):
# must be implemented by student into ~/.cloudmesh/storage/test/
# see self.content which contains all files that you can test against
# in the list return. all of them must be in there
StopWatch.start("LIST Directory --recursive")
contents = self.p.list(self.p.service, "/a", True)
StopWatch.stop("LIST Directory --recursive")
assert len(contents) > 0
def test_selective_list(self):
# must be implemented by student into ~/.cloudmesh/storage/test/a/b
# see self.content which contains all files that you can test against
# in the list return. all of them must be in there but not more?
# I am unsure if we implemented a secive list. If not let us know
# full list for now is fine
StopWatch.start("LIST Sub-directory --recursive")
contents = self.p.list(self.p.service, "/a/a/b", True)
StopWatch.stop("LIST Sub-directory --recursive")
assert len(contents) > 0
def test_search_b1(self):
# search for b1.txt
src = '/a'
filename = 'b1.txt'
StopWatch.start("SEARCH file --recursive")
search_files = self.p.search(self.p.service, src, filename, True)
StopWatch.stop("SEARCH file --recursive")
assert search_files is not None
def test_search_b1_dir(self):
# search for b/b2.txt see that this one has even the dir in the search
src = '/a'
filename = '/b/b1.txt'
StopWatch.start("SEARCH file under a sub-dir --r")
search_files = self.p.search(self.p.service, src, filename, True)
StopWatch.stop("SEARCH file under a sub-dir --r")
assert search_files is not None
def test_search_a1(self):
# search for a1.txt which shold return 2 entries
src = '/a'
filename = 'a1.txt'
StopWatch.start("SEARCH file under root dir --r")
search_files = self.p.search(self.p.service, src, filename, True)
StopWatch.stop("SEARCH file under root dir --r")
assert len(search_files) == 2
def test_results(self):
HEADING()
# storage = self.p.service
service = self.service
banner(f"Benchmark results for {service} Storage")
StopWatch.benchmark()
class TestLocal(object):
def setup_class(self):
# variables = Variables()
# service = Parameter.expand(variables['storage'])[0]
self.service = "local"
self.p = Provider(service=self.service)
def test_00__config(self):
VERBOSE(self.p)
VERBOSE(self.p.kind)
assert self.p.kind == self.service
def test_01_create_source(self):
HEADING()
self.sourcedir = path_expand(f"{location}/test/")
create_file(f"{location}/README.md", "content of a")
create_file(f"{location}/test/a/a.txt", "content of a")
create_file(f"{location}/test/a/b/b.txt", "content of b")
create_file(f"{location}/test/a/b/c/c.txt", "content of c")
# test if the files are ok
assert True
def test_02_list(self):
HEADING()
StopWatch.start("list")
src = '/'
contents = self.p.list(source=src)
VERBOSE(contents, label="c")
for c in contents:
VERBOSE(c)
StopWatch.stop("list")
def test_05_search(self):
HEADING()
StopWatch.start("search")
src = '/'
filename = 'a.txt'
#
# bug use named arguments
#
files = self.p.search(directory=src, filename=filename, recursive=True)
# pprint(files)
StopWatch.stop("search")
assert len(files) > 0
def test_02_put(self):
HEADING()
StopWatch.start("put")
src = path_expand("{location}/test/a/a.txt")
dst = f"{location}/destination"
test_file = self.p.put(src, dst)
# pprint(test_file)
StopWatch.stop("put")
assert test_file is not None
def test_03_get(self):
HEADING()
StopWatch.start("get")
src = path_expand(f"{location}/destination/a.txt")
dst = path_expand(f"{location}/destination/test.txt")
file = self.p.get(src, dst)
# pprint(file)
StopWatch.stop("get")
assert file is not None
# assert len(content) > 0
def test_06_create_dir(self):
HEADING()
dst = f"{location}/destination"
src = path_expand("{dst}/created_dir")
StopWatch.start("create_dir")
directory = self.p.create_dir(src)
# pprint(directory)
StopWatch.stop("create_dir")
assert directory is not None
def test_07_delete(self):
HEADING()
dst = f"{location}/destination"
src = path_expand("{dst}/created_dir")
StopWatch.start("delete")
self.p.delete(src)
StopWatch.stop("delete")
def test_benchmark(self):
Benchmark.print(sysinfo=False, csv=True, tag=cloud)
def do_storage(self, args, arguments):
"""
::
Usage:
storage [--storage=SERVICE] create dir DIRECTORY
storage [--storage=SERVICE] get SOURCE DESTINATION [--recursive]
storage [--storage=SERVICE] put SOURCE DESTINATION [--recursive]
storage [--storage=SERVICE] list [SOURCE] [--recursive] [--output=OUTPUT]
storage [--storage=SERVICE] delete SOURCE
storage [--storage=SERVICE] search DIRECTORY FILENAME [--recursive] [--output=OUTPUT]
storage [--storage=SERVICE] sync SOURCE DESTINATION [--name=NAME] [--async]
storage [--storage=SERVICE] sync status [--name=NAME]
storage config list [--output=OUTPUT]
storage copy SOURCE DESTINATION [--recursive]
This command does some useful things.
Arguments:
SOURCE SOURCE can be a directory or file
DESTINATION DESTINATION can be a directory or file
DIRECTORY DIRECTORY refers to a folder on the cloud service
Options:
--storage=SERVICE specify the cloud service name like aws or
azure or box or google
Description:
commands used to upload, download, list files on different
cloud storage services.
storage put [options..]
Uploads the file specified in the filename to specified
cloud from the SOURCEDIR.
storage get [options..]
Downloads the file specified in the filename from the
specified cloud to the DESTDIR.
storage delete [options..]
Deletes the file specified in the filename from the
specified cloud.
storage list [options..]
lists all the files from the container name specified on
the specified cloud.
storage create dir [options..]
creates a folder with the directory name specified on the
specified cloud.
storage search [options..]
searches for the source in all the folders on the specified
cloud.
sync SOURCE DESTINATION
puts the content of source to the destination.
If --recursive is specified this is done recursively from
the source
If --async is specified, this is done asynchronously
If a name is specified, the process can also be monitored
with the status command by name.
If the name is not specified all date is monitored.
sync status
The status for the asynchronous sync can be seen with this
command
config list
Lists the configures storage services in the yaml file
storage copy SOURCE DESTINATION
Copies files from source storage to destination storage.
The syntax of SOURCE and DESTINATION is:
SOURCE - awss3:source.txt
DESTINATION - azure:target.txt
Example:
set storage=azureblob
storage put SOURCE DESTINATION --recursive
is the same as
storage --storage=azureblob put SOURCE DESTINATION --recursive
storage copy azure:source.txt oracle:target.txt
"""
# arguments.CONTAINER = arguments["--container"]
map_parameters(arguments, "recursive", "storage")
VERBOSE(arguments)
if arguments.storage is None:
if arguments.copy is None:
try:
v = Variables()
arguments.storage = v['storage']
except Exception as e:
arguments.storage = None
raise ValueError("Storage provider is not defined")
else:
if arguments.DESTINATION.split(":")[0] == "local":
arguments.storage = arguments.SOURCE.split(":")[0]
else:
arguments.storage = arguments.DESTINATION.split(":")[0]
arguments.storage = Parameter.expand(arguments.storage)
if arguments["get"]:
provider = Provider(arguments.storage[0])
result = provider.get(arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
elif arguments.put:
provider = Provider(arguments.storage[0])
result = provider.put(arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
elif arguments.create and arguments.dir:
provider = Provider(arguments.storage[0])
result = provider.create_dir(arguments.DIRECTORY)
elif arguments.list:
source = arguments.SOURCE or '.'
for storage in arguments.storage:
provider = Provider(storage)
result = provider.list(source, arguments.recursive)
elif arguments.delete:
for storage in arguments.storage:
provider = Provider(storage)
provider.delete(arguments.SOURCE)
elif arguments.search:
for storage in arguments.storage:
provider = Provider(storage)
provider.search(arguments.DIRECTORY, arguments.FILENAME,
arguments.recursive)
elif arguments.rsync:
# TODO: implement
raise NotImplementedError
elif arguments.copy:
VERBOSE(f"COPY: Executing Copy command from {arguments.SOURCE} to "
f"{arguments.DESTINATION} providers")
print(f"DEBUG storage.py: INITIALIZE with {arguments.storage[0]} "
"provider.")
provider = Provider(arguments.storage[0])
result = provider.copy(arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
def do_storage(self, args, arguments):
"""
::
Usage:
storage [--storage=SERVICE] create dir DIRECTORY
storage [--storage=SERVICE] get SOURCE DESTINATION [--recursive]
storage [--storage=SERVICE] put SOURCE DESTINATION [--recursive]
storage [--storage=SERVICE] list SOURCE [--recursive]
storage [--storage=SERVICE] delete SOURCE
storage [--storage=SERVICE] search DIRECTORY FILENAME [--recursive]
This command does some useful things.
Arguments:
SOURCE SOURCE can be a directory or file
DESTINATION DESTINATION can be a directory or file
DIRECTORY DIRECTORY refers to a folder on the cloud service
Options:
--storage=SERVICE specify the cloud service name like aws or azure or box or google
Description:
commands used to upload, download, list files on different cloud storage services.
storage put [options..]
Uploads the file specified in the filename to specified cloud from the SOURCEDIR.
storage get [options..]
Downloads the file specified in the filename from the specified cloud to the DESTDIR.
storage delete [options..]
Deletes the file specified in the filename from the specified cloud.
storage list [options..]
lists all the files from the container name specified on the specified cloud.
storage create dir [options..]
creates a folder with the directory name specified on the specified cloud.
storage search [options..]
searches for the source in all the folders on the specified cloud.
Example:
set storage=azureblob
storage put SOURCE DESTINATION --recursive
is the same as
storage --storage=azureblob put SOURCE DESTINATION --recursive
"""
# arguments.CONTAINER = arguments["--container"]
map_parameters(arguments,
"recursive",
"storage")
arguments.storage = arguments["--storage"]
pprint(arguments)
m = Provider()
service = None
#
# BUG
# services = Parameter.expand(arguments.storage)
# service = services[0]
# if services is None:
# ... do second try
##### BUG
try:
service = arguments["--storage"][0]
except Exception as e:
try:
v = Variables()
service = v['storage']
except Exception as e:
service = None
if service is None:
Console.error("storage service not defined")
return
# bug this is now done twice ....
if arguments.storage is None:
variables = Variables()
arguments.storage = variables['storage']
##### Prvious code needs to be modified
if arguments.get:
m.get(arguments.storage, arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
elif arguments.put:
m.put(arguments.storage, arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
elif arguments.list:
print('in List')
m.list(arguments.storage, arguments.SOURCE, arguments.recursive)
elif arguments.create and arguments.dir.:
m.createdir(arguments.storage, arguments.DIRECTORY)
elif arguments.delete.:
m.delete(arguments.storage, arguments.SOURCE)
elif arguments['search']:
m.search(arguments.storage, arguments.DIRECTORY, arguments.FILENAME,
arguments.recursive)
def do_storage(self, args, arguments):
"""
::
Usage:
storage [--storage=SERVICE] create dir DIRECTORY
storage [--storage=SERVICE] get SOURCE DESTINATION [--recursive]
storage [--storage=SERVICE] put SOURCE DESTINATION [--recursive]
storage [--storage=SERVICE] list SOURCE [--recursive] [--output=OUTPUT]
storage [--storage=SERVICE] delete SOURCE
storage [--storage=SERVICE] search DIRECTORY FILENAME [--recursive] [--output=OUTPUT]
storage [--storage=SERVICE] sync SOURCE DESTINATION [--name=NAME] [--async]
storage [--storage=SERVICE] sync status [--name=NAME]
storage config list [--output=OUTPUT]
This command does some useful things.
Arguments:
SOURCE SOURCE can be a directory or file
DESTINATION DESTINATION can be a directory or file
DIRECTORY DIRECTORY refers to a folder on the cloud service
Options:
--storage=SERVICE specify the cloud service name like aws or
azure or box or google
Description:
commands used to upload, download, list files on different
cloud storage services.
storage put [options..]
Uploads the file specified in the filename to specified
cloud from the SOURCEDIR.
storage get [options..]
Downloads the file specified in the filename from the
specified cloud to the DESTDIR.
storage delete [options..]
Deletes the file specified in the filename from the
specified cloud.
storage list [options..]
lists all the files from the container name specified on
the specified cloud.
storage create dir [options..]
creates a folder with the directory name specified on the
specified cloud.
storage search [options..]
searches for the source in all the folders on the specified
cloud.
sync SOURCE DESTINATION
puts the content of source to the destination.
If --recursive is specified this is done recursively from
the source
If --async is specified, this is done asynchronously
If a name is specified, the process can also be monitored
with the status command by name.
If the name is not specified all date is monitored.
sync status
The status for the asynchronous sync can be seen with this
command
config list
Lists the configures storage services in the yaml file
Example:
set storage=azureblob
storage put SOURCE DESTINATION --recursive
is the same as
storage --storage=azureblob put SOURCE DESTINATION --recursive
"""
# arguments.CONTAINER = arguments["--container"]
map_parameters(arguments, "recursive", "storage")
VERBOSE(arguments)
if arguments.storage is None:
try:
v = Variables()
arguments.storage = v['storage']
except Exception as e:
arguments.storage = None
raise ValueError("Storage provider is not defined")
arguments.storage = Parameter.expand(arguments.storage)
#
# BUG: some commands could be run on more than the first provider,
# such as list
# thus the if condition needs to be reorganized
if arguments["get"]:
provider = Provider(arguments.storage[0])
result = provider.get(arguments.storage, arguments.SOURCE,
arguments.DESTINATION, arguments.recursive)
elif arguments.put:
provider = Provider(arguments.storage[0])
result = provider.put(arguments.storage, arguments.SOURCE,
arguments.DESTINATION, arguments.recursive)
elif arguments.create and arguments.dir:
provider = Provider(arguments.storage[0])
result = provider.createdir(arguments.storage, arguments.DIRECTORY)
elif arguments.list:
#
# BUG: this command is much more complicated
#
for storage in arguments.storage:
provider = Provider(storage)
result = provider.list(arguments.storage, arguments.SOURCE,
arguments.recursive)
elif arguments.delete:
#
# BUG:: this command could be much more complicated
#
for storage in arguments.storage:
provider = Provider(storage)
provider.delete(arguments.storage, arguments.SOURCE)
elif arguments.search:
#
# BUG: this command is much more complicated
#
for storage in arguments.storage:
provider = Provider(storage)
provider.search(arguments.storage, arguments.DIRECTORY,
arguments.FILENAME, arguments.recursive)
elif arguments.rsync:
# TODO: implement
raise NotImplementedError
class StorageQueue:
"""
This class specifies a storage object queue, that allows the queuing of
files to be copied between services.
The queue has a maximal parallelism that can be set to execute the copy in
multiple threads.
Please note that actions only add modify the queue in the db, however,
the run command executes them one by one.
It will be up to the method to guarantee order. For example, in case of a
recursive copy it would make sense to create directories first.
"""
"""
DB object
cm:
id: uuid
collection: storage-queue-{source}-{destination}
...
action: copy
source: the/source/path
destination: the/destination/path
created: date
status:
Actions can be for example
copy
mkdir
delete
cancel
cancel has a specific action allowing all jobs that have not
yet been finished to be canceled.
Each file can be in the state: completed, waiting, inprogress, canceled
here is an example for the status of the queue.
{
"length": 100,
"completed": 10,
"waiting": 80,
"inprogress": 10,
"canceled": 0
}
"""
status = [
'completed',
'waiting',
'inprogress',
'canceled'
]
#def register(self):
# # find the inheritor of StorageQueue and register the methd from
# # self._put = parent._put
def redgister_actions(self,
put=None,
get=None,
delete=None,
mkdir=None,
list=None,
cancel=None):
self._put = put
self._get = get
self._delete = delete
self._mkdir = mkdir
self._list = list # attention list
self._cancel = cancel
def __init__(self,
source,
destination,
name="local",
parallelism=4):
"""
:param name: The name of the queue (used as a collection in mongodb)
:param source: The name of the service in cloudmesh.data from which
to copy
:param destination: The name of the service in cloudmesh.data from
which to copy
:param parallelism: The number of parallel threads
"""
self.source = source
self.destination = destination
self.parallelism = parallelism
config = Config()
self.source_spec = config[f"cloudmesh.storage.{source}"]
self.destination_spec = config[f"cloudmesh.storage.{destination}"]
self.provider_source = Provider(service=source)
self.provider_destination = Provider(service=destination)
self.name = name
self.collection = f"storage-queue-{name}-{source}-{destination}"
self.number = 0
#
# TODO: create collection in mongodb
#
Console.ok(f"Collection: {self.name}")
def _copy_file(self, sourcefile, destinationfile):
"""
adds a copy action to the queue
copies the file from the source service to the destination service using
the file located in the path and storing it into the remote. If remote
is not specified path is used for it.
The copy will not be performed if the files are the same.
:param sourcefile:
:param destinationfile:
:return:
"""
date = DateTime.now()
uuid_str = str(uuid.uuid1())
specification = textwrap.dedent(f"""
cm:
number: {self.number}
name: "{self.source}:{sourcefile}"
kind: storage
id: {uuid_str}
cloud: {self.collection}
collection: {self.collection}
created: {date}
action: copy
source:
service: {self.source}
path: {sourcefile}
destination:
service: {self.destination}
path: {destinationfile}
status: waiting
""")
entries = yaml.load(specification, Loader=yaml.SafeLoader)
self.number = self.number + 1
return entries
@DatabaseUpdate()
def copy_file(self, sourcefile, destinationfile):
"""
adds a copy action to the queue
copies the file from the source service to the destination service using
the file located in the path and storing it into the remote. If remote
is not specified path is used for it.
The copy will not be performed if the files are the same.
:param sourcefile:
:param destinationfile:
:return:
"""
self._copy_file(sourcefile, destinationfile)
@DatabaseUpdate()
def copy_tree(self, sourcetree, destinationtree):
"""
adds a tree to be copied to the queue
it will recursively add all files within the tree
:param sourcetree:
:param destinationtree:
:return:
"""
# goes recursively through the dree and adds_the file
sources = self.provider_source.list(sourcetree, recursive=True)
files = []
dirs = []
for source in sources:
if bool(source['file']):
files.append(source)
else:
dirs.append((source))
# create dirs first
actions = []
for file in dirs:
location = file["cm"]["location"]
actions.append(self.mkdir(self.destination, location))
# now copy files
for file in files:
location = file["cm"]["location"]
actions.append(self._copy_file(location, location))
return actions
def sync(self, sourcetree, destinationtree):
"""
just a more convenient name for copy_tree
:param sourcetree:
:param destinationtree:
:return:
"""
self.copy_tree(sourcetree, destinationtree)
def mkdir(self, service, path):
"""
adds a mkdir action to the queue
create the directory in the storage service
:param service: service must be either source or destination
:param path:
:return:
"""
date = DateTime.now()
uuid_str = str(uuid.uuid1())
specification = textwrap.dedent(f"""
cm:
number: {self.number}
name: "{service}:{path}"
kind: storage
id: {uuid_str}
cloud: {self.collection}
collection: {self.collection}
created: {date}
action: mkdir
source:
service: {service}
path: {path}
status: waiting
""")
entries = yaml.load(specification, Loader=yaml.SafeLoader)
self.number = self.number + 1
return entries
def delete(self, service, path):
"""
adds a deleta action to the queue
:param service: service must be either source or destination
:param path:
:return:
"""
date = DateTime.now()
uuid_str = str(uuid.uuid1())
specification = textwrap.dedent(f"""
cm:
number: {self.number}
name: "{service}:{path}"
kind: storage
id: {uuid_str}
cloud: {self.collection}
collection: {self.collection}
created: {date}
action: delete
source:
service: {service}
path: {path}
status: waiting
""")
entries = yaml.load(specification, Loader=yaml.SafeLoader)
self.number = self.number + 1
return entries
@DatabaseUpdate()
def delete(self, path, recursive=True):
"""
adds a delete action to the queue
:param path:
:return:
"""
date = DateTime.now()
uuid_str = str(uuid.uuid1())
specification = textwrap.dedent(f"""
cm:
number: {self.number}
name: "{path}"
kind: storage
id: {uuid_str}
cloud: {self.name}
collection: {self.collection}
created: {date}
action: delete
source:
service: {service}
path: {path}
recursive: {recursive}
status: waiting
""")
entries = yaml.load(specification, Loader=yaml.SafeLoader)
self.number = self.number + 1
return entries
def status(self):
"""
provides that status of the queue
{
"length": 100,
"completed": 10,
"waiting": 80,
"inprogress": 10
}
:return:
"""
# find all teh values from within the MongoDB
raise NotImplementedError
@DatabaseUpdate()
def cancel(self, id=None):
"""
cancels a job with a specific id
:param id:
:return:
"""
# if None all are canceled
date = DateTime.now()
uuid_str = str(uuid.uuid1())
specification = textwrap.dedent(f"""
cm:
number: {self.number}
name: "{id}"
kind: storage
id: {uuid_str}
cloud: {self.name}
collection: {self.collection}
created: {date}
action: cancel
status: waiting
""")
entries = yaml.load(specification, Loader=yaml.SafeLoader)
self.number = self.number + 1
return entries
@DatabaseUpdate()
def list(self, path, dir_only=False, recursive=False):
"""
adds a list action to the queue
list the directory in the storage service
:param service: service must be either source or destination
:param path:
:return:
"""
date = DateTime.now()
uuid_str = str(uuid.uuid1())
specification = textwrap.dedent(f"""
cm:
number: {self.number}
kind: storage
id: {uuid_str}
cloud: {self.name}
name: {path}
collection: {self.collection}
created: {date}
action: list
path: {path}
dir_only:{dir_only}
recursive:{recursive}
status: waiting
""")
entries = yaml.load(specification, Loader=yaml.SafeLoader)
self.number = self.number + 1
return entries
def action(self, specification):
"""
executes the action identified by the specification. This is used by the
run command.
:param specification:
:return:
"""
action = specification["action"]
if action == "copy":
# print("COPY", specification)
specification = self._put(specification)
# update status
self.update_dict(elements=[specification])
elif action == "delete":
# print("DELETE", specification)
specification = self._delete(specification)
# update status
self.update_dict(elements=[specification])
elif action == "mkdir":
# print("MKDIR", specification)
specification = self._mkdir(specification)
# update status
self.update_dict(elements=[specification])
elif action == "list":
# print("LIST", specification)
specification = self._list(specification)
# update status
self.update_dict(elements=[specification])
elif action == "cancel":
specification = self._cancel(specification)
self.update_dict(elements=[specification])
def get_actions(self):
cm = CmDatabase()
entries = cm.find(cloud=self.name,
kind='storage')
mkdir = []
copy = []
list = []
delete = []
cancel = []
for entry in entries:
pprint(entry)
if entry['action'] == 'mkdir' and entry['status'] == 'waiting':
mkdir.append(entry)
elif entry['action'] == 'copy' and entry['status'] == 'waiting':
copy.append(entry)
elif entry['action'] == 'list' and entry['status'] == 'waiting':
list.append(entry)
elif entry['action'] == 'delete' and entry['status'] == 'waiting':
delete.append(entry)
elif entry['action'] == 'cancel' and entry['status'] == 'waiting':
cancel.append(entry)
return mkdir, copy, list, delete, cancel
def run(self):
"""
runs the copy process for all jobs in the queue and completes when all
actions are completed
:return:
"""
mkdir_action, copy_action, list_action, delete_action, cancel_action = self.get_actions()
# cancel the actions
#
p = Pool(self.parallelism)
#
p.map(self.action, cancel_action)
# delete files/directories
#
p = Pool(self.parallelism)
#
p.map(self.action, delete_action)
# create directories
#
p = Pool(self.parallelism)
#
p.map(self.action, mkdir_action)
# COPY FILES
#
p = Pool(self.parallelism)
#
p.map(self.action, copy_action)
# LIST FILES
p = Pool(self.parallelism)
p.map(self.action, list_action)
def do_storage(self, args, arguments):
"""
::
Usage:
storage run
storage clean
storage monitor [--storage=SERVICES] [--status=all | --status=STATUS] [--output=output] [--clear]
storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N] [--run]
storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] [--run]
storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] [--run]
storage list [--storage=SERVICE] [SOURCE] [--recursive] [--parallel=N] [--output=OUTPUT] [--dryrun] [--run]
storage delete SOURCE [--storage=SERVICE] [--parallel=N] [--dryrun] [--run]
storage search DIRECTORY FILENAME [--recursive] [--storage=SERVICE] [--parallel=N] [--output=OUTPUT] [--run]
storage sync SOURCE DESTINATION [--name=NAME] [--async] [--storage=SERVICE]
storage sync status [--name=NAME] [--storage=SERVICE]
storage config list [--output=OUTPUT]
storage copy --source=SOURCE:SOURCE_FILE_DIR --target=TARGET:TARGET_FILE_DIR [--run]
storage cc --source=SOURCE:SOURCE_FILE_DIR --target=TARGET:TARGET_FILE_DIR
This command does some useful things.
Arguments:
SOURCE SOURCE can be a directory or file
DESTINATION DESTINATION can be a directory or file
DIRECTORY DIRECTORY refers to a folder on the cloud service
SOURCE:SOURCE_FILE_DIR source provider name: file or directory name
TARGET:SOURCE_FILE_DIR destination provider name
Options:
--storage=SERVICE specify the cloud service name like aws or
azure or box or google
Description:
commands used to upload, download, list files on different
cloud storage services.
storage run
Execute the actions in database that are in waiting status.
> storage monitor [--storage=SERVICE]
> [--status=all | --status=STATUS]
> [--output=output]
> [--clear]
Monitor the actions in database and refresh every 5 seconds.
> storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE]
> [--parallel=N]
Uploads the file specified in the filename to specified
cloud from the SOURCEDIR.
> storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE]
> [--parallel=N]
Downloads the file specified in the filename from the
specified cloud to the DESTDIR.
storage delete SOURCE [--parallel=N] [--dryrun]
Deletes the file specified in the filename from the
specified cloud.
> storage list [SOURCE] [--recursive] [--parallel=N]
> [--output=OUTPUT] [--dryrun]
lists all the files from the container name specified on
the specified cloud.
storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N]
creates a folder with the directory name specified on the
specified cloud.
> storage search DIRECTORY FILENAME [--recursive]
> [--storage=SERVICE]
> [--parallel=N]
> [--output=OUTPUT]
searches for the source in all the folders on the specified
cloud.
sync SOURCE DESTINATION
puts the content of source to the destination.
If --recursive is specified this is done recursively from
the source
If --async is specified, this is done asynchronously
If a name is specified, the process can also be monitored
with the status command by name.
If the name is not specified all date is monitored.
sync status
The status for the asynchronous sync can be seen with this
command
config list
Lists the configures storage services in the yaml file
storage copy SOURCE DESTINATION
Copies files from source storage to destination storage.
The syntax of SOURCE and DESTINATION is:
SOURCE - awss3:source.txt
DESTINATION - azure:target.txt
Description of the copy command:
Command enables to Copy files between different cloud service
providers, list and delete them. This command accepts `aws` ,
`google` and `local` as the SOURCE and TARGET provider.
cms storage copy --source=SERVICE:SOURCE --target=DEST:TARGET
Command copies files or directories from Source provider to
Target Provider.
cms storage slist --source=SERVICE:SOURCE
Command lists all the files present in SOURCE provider's in
the given SOURCE_FILE_DIR location This command accepts
`aws` or `google` as the SOURCE provider
cms storage sdelete --source=SERVICE:SOURCE
Command deletes the file or directory from the SOURCE
provider's SOURCE_FILE_DIR location
Examples:
> cms storage_service copy --source=local:test1.txt
> --target=aws:uploadtest1.txt
cms storage_service list --source=google:test
cms storage_service delete --source=aws:uploadtest1.txt
cms storage put test_file1.txt aws_test_file1.txt
cms storage put ./recur_dir recur_dir_aws/ --recursive
cms storage put ./recur_dir recur_dir_aws/
cms storage get aws_test_file1.txt aws_file1.txt
cms storage get recur_dir_aws from_aws_dir
cms storage get recur_dir_aws from_aws_dir --recursive
cms storage list
cms storage list --recursive
cms storage list aws:recur_dir_aws --recursively
cms storage delete aws:aws_test_file1.txt
cms storage search recur_dir_aws recur_file1.txt
Example:
set storage=aws
storage put SOURCE DESTINATION --recursive
is the same as
storage --storage=aws put SOURCE DESTINATION --recursive
storage copy aws:source.txt oracle:target.txt
"""
# arguments.CONTAINER = arguments["--container"]
VERBOSE(arguments)
map_parameters(arguments, "dryrun", "recursive", "storage", "source",
"target", "parallel")
source = arguments.source
target = arguments.target
variables = Variables()
parallelism = arguments.parallel or 1
arguments.storage = Parameter.expand(arguments.storage
or variables['storage'])
run_immediately = arguments['--run']
if arguments.monitor:
provider = Provider(arguments.storage[0], parallelism=parallelism)
status = arguments['--status'] or "all"
output = arguments['--output'] or "table"
result = provider.monitor(status=status, output=output)
elif arguments.clean:
provider = Provider(arguments.storage[0], parallelism=parallelism)
result = provider.clean()
elif arguments.run:
provider = Provider(arguments.storage[0], parallelism=parallelism)
result = provider.run()
elif arguments['get']:
provider = Provider(arguments.storage[0], parallelism=parallelism)
result = provider.get(arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
if run_immediately:
provider.run()
elif arguments.put:
provider = Provider(arguments.storage[0], parallelism=parallelism)
result = provider.put(arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
if run_immediately:
provider.run()
elif arguments.create and arguments.dir:
provider = Provider(arguments.storage[0], parallelism=parallelism)
result = provider.create_dir(arguments.DIRECTORY)
if run_immediately:
provider.run()
elif arguments.list:
"""
storage list SOURCE [--parallel=N]
"""
if variables['storage']:
default_source = f"{variables['storage']}:/"
else:
default_source = "local:/"
sources = arguments.SOURCE or default_source
if not ":" in sources:
sources = f"{variables['storage']}:{sources}"
sources = Parameter.expand(sources)
deletes = []
for source in sources:
storage, entry = Parameter.separate(source)
storage = storage or source or "local"
deletes.append((storage, entry))
_sources = ', '.join(sources)
for delete in deletes:
service, entry = delete
if arguments.dryrun:
print(f"Dryrun: list {service}:{entry}")
else:
provider = Provider(service=service,
parallelism=parallelism)
provider.list(name=entry, recursive=arguments.recursive)
if run_immediately:
provider.run()
return ""
elif arguments.delete:
"""
storage delete SOURCE [--parallel=N]
"""
if variables['storage']:
default_source = f"{variables['storage']}:/"
else:
default_source = "local:/"
sources = arguments.SOURCE or default_source
if not ":" in sources:
sources = f"{variables['storage']}:{sources}"
sources = Parameter.expand(sources)
deletes = []
for source in sources:
storage, entry = Parameter.separate(source)
storage = storage or source or "local"
deletes.append((storage, entry))
_sources = ', '.join(sources)
answer = yn_choice(f"Would you like to delete {_sources}?",
default="no")
if answer:
for delete in deletes:
service, entry = delete
if arguments.dryrun:
print(f"Dryrun: delete {service}:{entry}")
else:
provider = Provider(service=service,
parallelism=parallelism)
provider.delete(name=entry)
else:
Console.error("Deletion canceled")
if run_immediately:
provider.run()
return ""
elif arguments.search:
for storage in arguments.storage:
provider = Provider(storage, parallelism=parallelism)
provider.search(arguments.DIRECTORY, arguments.FILENAME,
arguments.recursive)
if run_immediately:
provider.run()
elif arguments.rsync:
# TODO: implement
raise NotImplementedError
elif arguments['cc']:
scloud, sfileDir = source.split(":", 1) or None
tcloud, tfileDir = target.split(":", 1) or None
print(f" Copying from Source {scloud} : {sfileDir} to Target "
f" {tcloud} : {tfileDir}")
cloudName = ["aws", "google"]
if scloud in cloudName:
provider = Provider(service=scloud, parallelism=parallelism)
provider.copyFiles(scloud, sfileDir, tcloud, tfileDir)
else:
print("Not Implemented")
return ""
elif arguments.copy:
scloud, sbucket = arguments['--source'].split(":", 1) or None
tcloud, tbucket = arguments['--target'].split(":", 1) or None
if scloud == "aws" or scloud == "google":
provider = Provider(service=scloud, parallelism=parallelism)
provider.copy(scloud, tcloud, sbucket, tbucket)
else:
provider = Provider(service=tcloud, parallelism=parallelism)
provider.copy(arguments['--source'], arguments['--target'],
arguments.recursive)
if run_immediately:
provider.run()
return ""
class TestStorage(object):
def create_file(self, location, content):
d = Path(os.path.dirname(path_expand(location)))
print()
print("TESTDIR:", d)
d.mkdir(parents=True, exist_ok=True)
writefile(path_expand(location), content)
def setup(self):
variables = Variables()
service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=service)
def test_create_source(self):
HEADING()
StopWatch.start("create source")
self.sourcedir = path_expand("~/.cloudmesh/storage/test/")
self.create_file("~/.cloudmesh/storage/test/a/a.txt", "content of a")
self.create_file("~/.cloudmesh/storage/test/a/b/b.txt", "content of b")
self.create_file("~/.cloudmesh/storage/test/a/b/c/c.txt",
"content of c")
StopWatch.stop("create source")
# test if the files are ok
assert True
def test_put(self):
HEADING()
src = path_expand("~/.cloudmesh/storage/test/a/a.txt")
dst = "/"
StopWatch.start("put")
test_file = self.p.put(src, dst)
StopWatch.stop("put")
pprint(test_file)
assert test_file is not None
def test_get(self):
HEADING()
src = path_expand("/a.txt")
dst = path_expand("~/test.txt")
StopWatch.start("get")
file = self.p.get(src, dst)
StopWatch.stop("get")
pprint(file)
assert file is not None
def test_list(self):
HEADING()
src = '/'
StopWatch.start("list")
contents = self.p.list(src)
StopWatch.stop("list")
for c in contents:
pprint(c)
assert len(contents) > 0
def test_search(self):
HEADING()
src = '/'
filename = 'test.txt'
#
# bug use named arguments
#
StopWatch.start("search")
search_files = self.p.search(src, filename, True)
StopWatch.stop("serach")
pprint(search_files)
assert len(search_files) > 0
def test_create_dir(self):
HEADING()
src = '/created_dir'
StopWatch.start("create dir")
directory = self.p.create_dir(src)
StopWatch.stop("create dir")
pprint(directory)
assert directory is not None
def test_delete(self):
HEADING()
src = '/created_dir'
StopWatch.start("delete")
self.p.delete(src)
StopWatch.stop("delete")
def test_benchmark(self):
StopWatch.benchmark()
class Test_storage:
def create_file(self, location, content):
d = Path(os.path.dirname(path_expand(location)))
print()
print("TESTDIR:", d)
d.mkdir(parents=True, exist_ok=True)
writefile(path_expand(location), content)
def setup(self):
variables = Variables()
service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=service)
def test_01_create_source(self):
HEADING()
self.sourcedir = path_expand("~/.cloudmesh/storage/test/")
self.create_file("~/.cloudmesh/storage/test/a/a.txt", "content of a")
self.create_file("~/.cloudmesh/storage/test/a/b/b.txt", "content of b")
self.create_file("~/.cloudmesh/storage/test/a/b/c/c.txt",
"content of c")
# test if the files are ok
assert True
def test_01_put(self):
HEADING()
src = path_expand("~/.cloudmesh/storage/test/a/a.txt")
dst = "/"
test_file = self.p.put(src, dst)
pprint(test_file)
assert test_file is not None
def test_02_get(self):
HEADING()
src = path_expand("/a.txt")
dst = path_expand("~/test.txt")
file = self.p.get(src, dst)
pprint(file)
assert file is not None
def test_03_list(self):
HEADING()
src = '/'
contents = self.p.list(src)
for c in contents:
pprint(c)
assert len(contents) > 0
def test_04_search(self):
HEADING()
src = '/'
filename = 'test.txt'
#
# bug use named arguments
#
search_files = self.p.search(src, filename, True)
pprint(search_files)
assert len(search_files) > 0
def test_05_create_dir(self):
HEADING()
src = '/created_dir'
dir = self.p.create_dir(src)
pprint(dir)
assert dir is not None
def test_06_delete(self):
HEADING()
src = '/created_dir'
self.p.delete(src)
def do_storage(self, args, arguments):
"""
::
Usage:
storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N]
storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N]
storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N]
storage list [SOURCE] [--recursive] [--parallel=N] [--output=OUTPUT] [--dryrun]
storage delete SOURCE [--parallel=N] [--dryrun]
storage search DIRECTORY FILENAME [--recursive] [--storage=SERVICE] [--parallel=N] [--output=OUTPUT]
storage sync SOURCE DESTINATION [--name=NAME] [--async] [--storage=SERVICE]
storage sync status [--name=NAME] [--storage=SERVICE]
storage config list [--output=OUTPUT]
storage [--parallel=N] copy SOURCE DESTINATION [--recursive]
storage copy --source=SOURCE:SOURCE_FILE_DIR --target=TARGET:TARGET_FILE_DIR
This command does some useful things.
Arguments:
SOURCE SOURCE can be a directory or file
DESTINATION DESTINATION can be a directory or file
DIRECTORY DIRECTORY refers to a folder on the cloud service
SOURCE:SOURCE_FILE_DIR source provider name: file or directory name
TARGET:SOURCE_FILE_DIR destination provider name
Options:
--storage=SERVICE specify the cloud service name like aws or
azure or box or google
Description:
commands used to upload, download, list files on different
cloud storage services.
storage put [options..]
Uploads the file specified in the filename to specified
cloud from the SOURCEDIR.
storage get [options..]
Downloads the file specified in the filename from the
specified cloud to the DESTDIR.
storage delete [options..]
Deletes the file specified in the filename from the
specified cloud.
storage list [options..]
lists all the files from the container name specified on
the specified cloud.
storage create dir [options..]
creates a folder with the directory name specified on the
specified cloud.
storage search [options..]
searches for the source in all the folders on the specified
cloud.
sync SOURCE DESTINATION
puts the content of source to the destination.
If --recursive is specified this is done recursively from
the source
If --async is specified, this is done asynchronously
If a name is specified, the process can also be monitored
with the status command by name.
If the name is not specified all date is monitored.
sync status
The status for the asynchronous sync can be seen with this
command
config list
Lists the configures storage services in the yaml file
storage copy SOURCE DESTINATION
Copies files from source storage to destination storage.
The syntax of SOURCE and DESTINATION is:
SOURCE - awss3:source.txt
DESTINATION - azure:target.txt
Description of the copy command:
Command enables to Copy files between different cloud service
providers, list and delete them. This command accepts `aws` ,
`google` and `local` as the SOURCE and TARGET provider.
cms storage copy --source=SERVICE:SOURCE --target=DEST:TARGET
Command copies files or directories from Source provider to
Target Provider.
cms storage slist --source=SERVICE:SOURCE
Command lists all the files present in SOURCE provider's in
the given SOURCE_FILE_DIR location This command accepts
`aws` or `google` as the SOURCE provider
cms storage sdelete --source=SERVICE:SOURCE
Command deletes the file or directory from the SOURCE
provider's SOURCE_FILE_DIR location
Examples:
cms storage_service copy --source=local:test1.txt --target=aws:uploadtest1.txt
cms storage_service list --source=google:test
cms storage_service delete --source=aws:uploadtest1.txt
Example:
set storage=azureblob
storage put SOURCE DESTINATION --recursive
is the same as
storage --storage=azureblob put SOURCE DESTINATION --recursive
storage copy azure:source.txt oracle:target.txt
"""
# arguments.CONTAINER = arguments["--container"]
VERBOSE(arguments)
map_parameters(arguments, "dryrun", "recursive", "storage", "source",
"target")
source = arguments.source
target = arguments.target
variables = Variables()
VERBOSE(arguments)
arguments.storage = Parameter.expand(arguments.storage)
if arguments["get"]:
provider = Provider(arguments.storage[0])
result = provider.get(arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
elif arguments.put:
provider = Provider(arguments.storage[0])
result = provider.put(arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
elif arguments.create and arguments.dir:
provider = Provider(arguments.storage[0])
result = provider.create_dir(arguments.DIRECTORY)
elif arguments.list:
"""
storage list SOURCE [--parallel=N]
"""
sources = arguments.SOURCE or variables["storage"] or 'local:.'
sources = Parameter.expand(sources)
deletes = []
for source in sources:
storage, entry = Parameter.separate(source)
storage = storage or "local"
deletes.append((storage, entry))
_sources = ', '.join(sources)
for delete in deletes:
service, entry = delete
if arguments.dryrun:
print(f"Dryrun: list {service}:{entry}")
else:
provider = Provider(service=service)
provider.list(name=entry)
return ""
elif arguments.delete:
"""
storage delete SOURCE [--parallel=N]
"""
sources = arguments.SOURCE or variables["storage"] or 'local:.'
sources = Parameter.expand(sources)
deletes = []
for source in sources:
storage, entry = Parameter.separate(source)
storage = storage or "local"
deletes.append((storage, entry))
_sources = ', '.join(sources)
answer = yn_choice(f"Would you like to delete {_sources}?",
default="no")
if answer:
for delete in deletes:
service, entry = delete
if arguments.dryrun:
print(f"Dryrun: delete {service}:{entry}")
else:
provider = Provider(service=service)
provider.delete(name=entry)
else:
Console.error("Deletion canceled")
return ""
elif arguments.search:
for storage in arguments.storage:
provider = Provider(storage)
provider.search(arguments.DIRECTORY, arguments.FILENAME,
arguments.recursive)
elif arguments.rsync:
# TODO: implement
raise NotImplementedError
elif arguments.copy:
VERBOSE(f"COPY: Executing Copy command from {arguments.SOURCE} to "
f"{arguments.DESTINATION} providers")
print(f"DEBUG storage.py: INITIALIZE with {arguments.storage[0]} "
"provider.")
provider = Provider(arguments.storage[0])
result = provider.copy(arguments.SOURCE, arguments.DESTINATION,
arguments.recursive)
elif arguments.copy:
scloud, sbucket = source.split(":", 1) or None
tcloud, tbucket = target.split(":", 1) or None
# print(scloud + " " + tcloud + " " + sbucket + " " + tbucket)
if scloud == "aws" or scloud == "google":
provider = Provider(service=scloud)
provider.copy(scloud, tcloud, sbucket, tbucket)
elif (scloud == "local"
and tcloud == "aws") or (scloud == "local"
and tcloud == "google"):
provider = Provider(service=tcloud)
provider.copy(scloud, tcloud, sbucket, tbucket)
else:
print("Not Implemented")
return ""
class TestStorage(object):
def create_file(self, location, content):
d = Path(os.path.dirname(path_expand(location)))
print()
print("TESTDIR:", d)
d.mkdir(parents=True, exist_ok=True)
writefile(path_expand(location), content)
def setup(self):
variables = Variables()
service = Parameter.expand(variables['storage'])[0]
self.p = Provider(service=service)
def test_create_source(self):
HEADING()
StopWatch.start("create source")
self.sourcedir = path_expand("~/.cloudmesh/storage/test/")
self.create_file("~/.cloudmesh/storage/test/a/a.txt", "content of a")
self.create_file("~/.cloudmesh/storage/test/a/b/b.txt", "content of b")
self.create_file("~/.cloudmesh/storage/test/a/b/c/c.txt",
"content of c")
StopWatch.stop("create source")
# test if the files are ok
assert True
def test_put(self):
HEADING()
#root="~/.cloudmesh"
#src = "storage/test/a/a.txt"
# source = f"local:{src}"
# destination = f"aws:{src}"
# test_file = self.p.put(src, dst)
#src = "storage_a:test/a/a.txt"
src = "~/.cloudmesh/storage/test/"
dst = '/'
StopWatch.start("put")
test_file = self.p.put(src, dst)
StopWatch.stop("put")
pprint(test_file)
assert test_file is not None
def test_put_recursive(self):
HEADING()
#root="~/.cloudmesh"
#src = "storage/test/a/a.txt"
# source = f"local:{src}"
# destination = f"aws:{src}"
# test_file = self.p.put(src, dst)
#src = "storage_a:test/a/a.txt"
src = "~/.cloudmesh/storage/test/"
dst = '/'
StopWatch.start("put")
test_file = self.p.put(src, dst, True)
StopWatch.stop("put")
pprint(test_file)
assert test_file is not None
def test_get(self):
HEADING()
src = "/a.txt"
dst = "~/.cloudmesh/storage/test"
StopWatch.start("get")
file = self.p.get(src, dst)
StopWatch.stop("get")
pprint(file)
assert file is not None
def test_list(self):
HEADING()
src = '/'
StopWatch.start("list")
contents = self.p.list(src)
StopWatch.stop("list")
for c in contents:
pprint(c)
assert len(contents) > 0
def test_search(self):
HEADING()
src = '/'
filename = "a.txt"
StopWatch.start("search")
search_files = self.p.search(src, filename, True)
StopWatch.stop("search")
pprint(search_files)
assert len(search_files) > 0
#assert filename in search_files[0]['cm']["name"]
def test_create_dir(self):
HEADING()
src = 'created_dir'
StopWatch.start("create dir")
directory = self.p.create_dir(src)
StopWatch.stop("create dir")
pprint(directory)
assert directory is not None
def test_delete(self):
HEADING()
src = '/created_dir'
StopWatch.start("delete")
self.p.delete(src)
StopWatch.stop("delete")
def test_benchmark(self):
Benchmark.print(sysinfo=False, csv=True, tag=cloud)