class TestLocal(object): def setup_class(self): # variables = Variables() # service = Parameter.expand(variables['storage'])[0] self.service = "local" self.p = Provider(service=self.service) def test_00__config(self): VERBOSE(self.p) VERBOSE(self.p.kind) assert self.p.kind == self.service def test_01_create_source(self): HEADING() self.sourcedir = path_expand(f"{location}/test/") create_file(f"{location}/README.md", "content of a") create_file(f"{location}/test/a/a.txt", "content of a") create_file(f"{location}/test/a/b/b.txt", "content of b") create_file(f"{location}/test/a/b/c/c.txt", "content of c") # test if the files are ok assert True def test_02_list(self): HEADING() StopWatch.start("list") src = '/' contents = self.p.list(source=src) VERBOSE(contents, label="c") for c in contents: VERBOSE(c) StopWatch.stop("list") def test_05_search(self): HEADING() StopWatch.start("search") src = '/' filename = 'a.txt' # # bug use named arguments # files = self.p.search(directory=src, filename=filename, recursive=True) # pprint(files) StopWatch.stop("search") assert len(files) > 0 def test_02_put(self): HEADING() StopWatch.start("put") src = path_expand("{location}/test/a/a.txt") dst = f"{location}/destination" test_file = self.p.put(src, dst) # pprint(test_file) StopWatch.stop("put") assert test_file is not None def test_03_get(self): HEADING() StopWatch.start("get") src = path_expand(f"{location}/destination/a.txt") dst = path_expand(f"{location}/destination/test.txt") file = self.p.get(src, dst) # pprint(file) StopWatch.stop("get") assert file is not None # assert len(content) > 0 def test_06_create_dir(self): HEADING() dst = f"{location}/destination" src = path_expand("{dst}/created_dir") StopWatch.start("create_dir") directory = self.p.create_dir(src) # pprint(directory) StopWatch.stop("create_dir") assert directory is not None def test_07_delete(self): HEADING() dst = f"{location}/destination" src = path_expand("{dst}/created_dir") StopWatch.start("delete") self.p.delete(src) StopWatch.stop("delete") def test_benchmark(self): Benchmark.print(sysinfo=False, csv=True, tag=cloud)
class TestStorageBox(object): def create_file(self, location, content): Shell.mkdir(os.dirname(path_expand(location))) writefile(location, content) def setup(self): variables = Variables() self.service = Parameter.expand(variables['storage'])[0] self.p = Provider(service=self.service) self.sourcedir = path_expand("~/.cloudmesh/storage/test") print() def test_create_source(self): HEADING() home = self.sourcedir # Setup a dir self.content = [] self.files = [ "a/a1.txt", "a/a2.txt", "a/a3.txt", "a/b/b1.txt", "a/b/b2.txt", "a/b/b3.txt", "a/b/c/c1.txt", "a/b/c/c2.txt", "a/b/c/c3.txt", "a/b/d/d1.txt", "a/b/d/d2.txt", "a/b/d/d3.txt", "a/b/d/a1.txt" ] for f in self.files: location = f"{home}/{f}" self.create_file(location, f"content of {f}") self.content.append(location) # setup empty dir in a d1 = Path(path_expand(f"{home}/a/empty")) d1.mkdir(parents=True, exist_ok=True) for f in self.files: assert os.path.isfile(f"{home}/{f}") assert os.path.isdir(f"{home}/a/empty") def test_put_and_get(self): HEADING() home = self.sourcedir StopWatch.start("PUT file") test_file = self.p.put(self.p.service, f"{home}/a/a1.txt", "/") StopWatch.stop("PUT file") assert test_file is not None StopWatch.start("GET file") test_file = self.p.get(self.p.service, f"/a1.txt", f"{home}/hello.txt") StopWatch.stop("GET file") assert test_file is not None content = readfile(f"{home}/hello.txt") assert "a1.txt" in content def test_list(self): HEADING() StopWatch.start("LIST Directory") contents = self.p.list(self.p.service, "/") StopWatch.stop("LIST Directory") for c in contents: pprint(c) assert len(contents) > 0 found = False for entry in contents: if entry["cm"]["name"] == "a1.txt": found = True assert found def test_create_dir(self): HEADING() src = '/a/created_dir' StopWatch.start("CREATE DIR") directory = self.p.createdir(self.p.service, src) StopWatch.stop("CREATE DIR") pprint(directory) assert dir is not None assert "a/created_dir" in directory[0]["name"] def test_search(self): HEADING() src = '/' filename = "a1.txt" StopWatch.start("SEARCH file") search_files = self.p.search(self.p.service, src, filename, True) StopWatch.stop("SEARCH file") pprint(search_files) assert len(search_files) > 0 assert search_files[0]["name"] == filename def test_delete(self): HEADING() src = "/a/created_dir" StopWatch.start("DELETE Directory") contents = self.p.delete(self.p.service, src) StopWatch.stop("DELETE Directory") deleted = False for entry in contents: if "created_dir" in entry["cm"]["name"]: if entry["cm"]["status"] == "deleted": deleted = True assert deleted def test_recursive_put(self): # must be implemented by student from ~/.cloudmesh/storage/test # make sure all files are in the list see self.content which contains # all files home = self.sourcedir StopWatch.start("PUT Directory --recursive") upl_files = self.p.put(self.p.service, f"{home}", "/a", True) StopWatch.stop("PUT Directory --recursive") pprint(upl_files) assert upl_files is not None def test_recursive_get(self): # must be implemented by student into ~/.cloudmesh/storage/test/get # see self.content which contains all files but you must add get/ home = self.sourcedir d2 = Path(path_expand(f"{home}/get")) d2.mkdir(parents=True, exist_ok=True) StopWatch.start("GET Directory --recursive") dnld_files = self.p.get(self.p.service, "/a", f"{home}/get", True) StopWatch.stop("GET Directory --recursive") pprint(dnld_files) assert dnld_files is not None def test_recursive_delete(self): # must be implemented by student into ~/.cloudmesh/storage/test/get # see self.content which contains all files but you must add get/ src = "/a/a/b/c" StopWatch.start("DELETE Sub-directory") del_files = self.p.delete(self.p.service, src) StopWatch.stop("DELETE Sub-directory") assert len(del_files) > 0 def test_exhaustive_list(self): # must be implemented by student into ~/.cloudmesh/storage/test/ # see self.content which contains all files that you can test against # in the list return. all of them must be in there StopWatch.start("LIST Directory --recursive") contents = self.p.list(self.p.service, "/a", True) StopWatch.stop("LIST Directory --recursive") assert len(contents) > 0 def test_selective_list(self): # must be implemented by student into ~/.cloudmesh/storage/test/a/b # see self.content which contains all files that you can test against # in the list return. all of them must be in there but not more? # I am unsure if we implemented a secive list. If not let us know # full list for now is fine StopWatch.start("LIST Sub-directory --recursive") contents = self.p.list(self.p.service, "/a/a/b", True) StopWatch.stop("LIST Sub-directory --recursive") assert len(contents) > 0 def test_search_b1(self): # search for b1.txt src = '/a' filename = 'b1.txt' StopWatch.start("SEARCH file --recursive") search_files = self.p.search(self.p.service, src, filename, True) StopWatch.stop("SEARCH file --recursive") assert search_files is not None def test_search_b1_dir(self): # search for b/b2.txt see that this one has even the dir in the search src = '/a' filename = '/b/b1.txt' StopWatch.start("SEARCH file under a sub-dir --r") search_files = self.p.search(self.p.service, src, filename, True) StopWatch.stop("SEARCH file under a sub-dir --r") assert search_files is not None def test_search_a1(self): # search for a1.txt which shold return 2 entries src = '/a' filename = 'a1.txt' StopWatch.start("SEARCH file under root dir --r") search_files = self.p.search(self.p.service, src, filename, True) StopWatch.stop("SEARCH file under root dir --r") assert len(search_files) == 2 def test_results(self): HEADING() # storage = self.p.service service = self.service banner(f"Benchmark results for {service} Storage") StopWatch.benchmark()
class TestVdir(object): def create_file(self, location, content): d = Path(os.path.dirname(path_expand(location))) print() print("TESTDIR:", d) d.mkdir(parents=True, exist_ok=True) writefile(path_expand(location), content) def setup(self): StopWatch.start("vdir setup") self.vdir = Vdir() self.endpoint = 'box:/test.txt' self.directory_and_name = '/testdir/test' self.directory = 'testdir' self.file = 'test' self.create_file('~/.cloudmesh/vdir/test/test.txt', 'test file') self.destination = path_expand("~/.cloudmesh/vdir/test") variables = Variables() service = Parameter.expand(variables['storage'])[0] self.p = Provider(service=service) self.p.put(source='~/.cloudmesh/vdir/test/test.txt', destination='/', recursive=False) StopWatch.stop("vdir setup") def test_collection(self): HEADING() StopWatch.start("vdir collection") col = self.vdir.col StopWatch.stop("vdir collection") assert col.name == 'local-vdir' @pytest.fixture(scope='class') def dummy_file(self): self.endpoint = 'box:/test.txt' self.directory_and_name = 'test' StopWatch.start("vdir add") self.vdir = Vdir() testfile = self.vdir.add(endpoint=self.endpoint, dir_and_name=self.directory_and_name) StopWatch.stop("vdir add") return testfile @pytest.fixture(scope='class') def dummy_dir(self): self.directory = 'testdir' StopWatch.start("vdir mkdir") self.vdir = Vdir() testdir = self.vdir.mkdir(dirname=self.directory) StopWatch.stop("vdir mkdir") return testdir def test_mkdir(self, dummy_dir): HEADING() assert dummy_dir is not None def test_add(self, dummy_file): HEADING() assert dummy_file is not None def test_ls(self): HEADING() StopWatch.start("vdir ls") results = self.vdir.ls(directory=None) StopWatch.stop("vdir ls") assert results is not None def test_get(self): HEADING() StopWatch.start("vdir get") file = self.vdir.get(name=self.file, destination=self.destination) StopWatch.stop("vdir get") print(file) assert file is not None def test_status(self): HEADING() StopWatch.start("vdir status") file = self.vdir.status(dir_or_name=self.file) StopWatch.stop("vdir status") assert file is not None def test_cd(self): HEADING() StopWatch.start("vdir cd") self.vdir.cd(dirname=self.directory) StopWatch.stop("vdir cd") assert self.vdir.directory == self.directory def test_delete(self): HEADING() StopWatch.start("vdir delete") file = self.vdir.delete(dir_or_name=self.file) directory = self.vdir.delete(dir_or_name=self.directory) StopWatch.stop("vdir delete") assert all(obj is not None for obj in [file, directory]) def test_results(self): HEADING() StopWatch.benchmark()
def do_storage(self, args, arguments): """ :: Usage: storage [--storage=SERVICE] create dir DIRECTORY storage [--storage=SERVICE] get SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] put SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] list [SOURCE] [--recursive] [--output=OUTPUT] storage [--storage=SERVICE] delete SOURCE storage [--storage=SERVICE] search DIRECTORY FILENAME [--recursive] [--output=OUTPUT] storage [--storage=SERVICE] sync SOURCE DESTINATION [--name=NAME] [--async] storage [--storage=SERVICE] sync status [--name=NAME] storage config list [--output=OUTPUT] storage copy SOURCE DESTINATION [--recursive] This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete [options..] Deletes the file specified in the filename from the specified cloud. storage list [options..] lists all the files from the container name specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage search [options..] searches for the source in all the folders on the specified cloud. sync SOURCE DESTINATION puts the content of source to the destination. If --recursive is specified this is done recursively from the source If --async is specified, this is done asynchronously If a name is specified, the process can also be monitored with the status command by name. If the name is not specified all date is monitored. sync status The status for the asynchronous sync can be seen with this command config list Lists the configures storage services in the yaml file storage copy SOURCE DESTINATION Copies files from source storage to destination storage. The syntax of SOURCE and DESTINATION is: SOURCE - awss3:source.txt DESTINATION - azure:target.txt Example: set storage=azureblob storage put SOURCE DESTINATION --recursive is the same as storage --storage=azureblob put SOURCE DESTINATION --recursive storage copy azure:source.txt oracle:target.txt """ # arguments.CONTAINER = arguments["--container"] map_parameters(arguments, "recursive", "storage") VERBOSE(arguments) if arguments.storage is None: if arguments.copy is None: try: v = Variables() arguments.storage = v['storage'] except Exception as e: arguments.storage = None raise ValueError("Storage provider is not defined") else: if arguments.DESTINATION.split(":")[0] == "local": arguments.storage = arguments.SOURCE.split(":")[0] else: arguments.storage = arguments.DESTINATION.split(":")[0] arguments.storage = Parameter.expand(arguments.storage) if arguments["get"]: provider = Provider(arguments.storage[0]) result = provider.get(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.put: provider = Provider(arguments.storage[0]) result = provider.put(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.create and arguments.dir: provider = Provider(arguments.storage[0]) result = provider.create_dir(arguments.DIRECTORY) elif arguments.list: source = arguments.SOURCE or '.' for storage in arguments.storage: provider = Provider(storage) result = provider.list(source, arguments.recursive) elif arguments.delete: for storage in arguments.storage: provider = Provider(storage) provider.delete(arguments.SOURCE) elif arguments.search: for storage in arguments.storage: provider = Provider(storage) provider.search(arguments.DIRECTORY, arguments.FILENAME, arguments.recursive) elif arguments.rsync: # TODO: implement raise NotImplementedError elif arguments.copy: VERBOSE(f"COPY: Executing Copy command from {arguments.SOURCE} to " f"{arguments.DESTINATION} providers") print(f"DEBUG storage.py: INITIALIZE with {arguments.storage[0]} " "provider.") provider = Provider(arguments.storage[0]) result = provider.copy(arguments.SOURCE, arguments.DESTINATION, arguments.recursive)
def do_storage(self, args, arguments): """ :: Usage: storage [--storage=SERVICE] create dir DIRECTORY storage [--storage=SERVICE] get SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] put SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] list SOURCE [--recursive] [--output=OUTPUT] storage [--storage=SERVICE] delete SOURCE storage [--storage=SERVICE] search DIRECTORY FILENAME [--recursive] [--output=OUTPUT] storage [--storage=SERVICE] sync SOURCE DESTINATION [--name=NAME] [--async] storage [--storage=SERVICE] sync status [--name=NAME] storage config list [--output=OUTPUT] This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete [options..] Deletes the file specified in the filename from the specified cloud. storage list [options..] lists all the files from the container name specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage search [options..] searches for the source in all the folders on the specified cloud. sync SOURCE DESTINATION puts the content of source to the destination. If --recursive is specified this is done recursively from the source If --async is specified, this is done asynchronously If a name is specified, the process can also be monitored with the status command by name. If the name is not specified all date is monitored. sync status The status for the asynchronous sync can be seen with this command config list Lists the configures storage services in the yaml file Example: set storage=azureblob storage put SOURCE DESTINATION --recursive is the same as storage --storage=azureblob put SOURCE DESTINATION --recursive """ # arguments.CONTAINER = arguments["--container"] map_parameters(arguments, "recursive", "storage") VERBOSE(arguments) if arguments.storage is None: try: v = Variables() arguments.storage = v['storage'] except Exception as e: arguments.storage = None raise ValueError("Storage provider is not defined") arguments.storage = Parameter.expand(arguments.storage) # # BUG: some commands could be run on more than the first provider, # such as list # thus the if condition needs to be reorganized if arguments["get"]: provider = Provider(arguments.storage[0]) result = provider.get(arguments.storage, arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.put: provider = Provider(arguments.storage[0]) result = provider.put(arguments.storage, arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.create and arguments.dir: provider = Provider(arguments.storage[0]) result = provider.createdir(arguments.storage, arguments.DIRECTORY) elif arguments.list: # # BUG: this command is much more complicated # for storage in arguments.storage: provider = Provider(storage) result = provider.list(arguments.storage, arguments.SOURCE, arguments.recursive) elif arguments.delete: # # BUG:: this command could be much more complicated # for storage in arguments.storage: provider = Provider(storage) provider.delete(arguments.storage, arguments.SOURCE) elif arguments.search: # # BUG: this command is much more complicated # for storage in arguments.storage: provider = Provider(storage) provider.search(arguments.storage, arguments.DIRECTORY, arguments.FILENAME, arguments.recursive) elif arguments.rsync: # TODO: implement raise NotImplementedError
def do_storage(self, args, arguments): """ :: Usage: storage [--storage=SERVICE] create dir DIRECTORY storage [--storage=SERVICE] get SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] put SOURCE DESTINATION [--recursive] storage [--storage=SERVICE] list SOURCE [--recursive] storage [--storage=SERVICE] delete SOURCE storage [--storage=SERVICE] search DIRECTORY FILENAME [--recursive] This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete [options..] Deletes the file specified in the filename from the specified cloud. storage list [options..] lists all the files from the container name specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage search [options..] searches for the source in all the folders on the specified cloud. Example: set storage=azureblob storage put SOURCE DESTINATION --recursive is the same as storage --storage=azureblob put SOURCE DESTINATION --recursive """ # arguments.CONTAINER = arguments["--container"] map_parameters(arguments, "recursive", "storage") arguments.storage = arguments["--storage"] pprint(arguments) m = Provider() service = None # # BUG # services = Parameter.expand(arguments.storage) # service = services[0] # if services is None: # ... do second try ##### BUG try: service = arguments["--storage"][0] except Exception as e: try: v = Variables() service = v['storage'] except Exception as e: service = None if service is None: Console.error("storage service not defined") return # bug this is now done twice .... if arguments.storage is None: variables = Variables() arguments.storage = variables['storage'] ##### Prvious code needs to be modified if arguments.get: m.get(arguments.storage, arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.put: m.put(arguments.storage, arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.list: print('in List') m.list(arguments.storage, arguments.SOURCE, arguments.recursive) elif arguments.create and arguments.dir.: m.createdir(arguments.storage, arguments.DIRECTORY) elif arguments.delete.: m.delete(arguments.storage, arguments.SOURCE) elif arguments['search']: m.search(arguments.storage, arguments.DIRECTORY, arguments.FILENAME, arguments.recursive)
def do_storage(self, args, arguments): """ :: Usage: storage run storage clean storage monitor [--storage=SERVICES] [--status=all | --status=STATUS] [--output=output] [--clear] storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N] [--run] storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] [--run] storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] [--run] storage list [--storage=SERVICE] [SOURCE] [--recursive] [--parallel=N] [--output=OUTPUT] [--dryrun] [--run] storage delete SOURCE [--storage=SERVICE] [--parallel=N] [--dryrun] [--run] storage search DIRECTORY FILENAME [--recursive] [--storage=SERVICE] [--parallel=N] [--output=OUTPUT] [--run] storage sync SOURCE DESTINATION [--name=NAME] [--async] [--storage=SERVICE] storage sync status [--name=NAME] [--storage=SERVICE] storage config list [--output=OUTPUT] storage copy --source=SOURCE:SOURCE_FILE_DIR --target=TARGET:TARGET_FILE_DIR [--run] storage cc --source=SOURCE:SOURCE_FILE_DIR --target=TARGET:TARGET_FILE_DIR This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service SOURCE:SOURCE_FILE_DIR source provider name: file or directory name TARGET:SOURCE_FILE_DIR destination provider name Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage run Execute the actions in database that are in waiting status. > storage monitor [--storage=SERVICE] > [--status=all | --status=STATUS] > [--output=output] > [--clear] Monitor the actions in database and refresh every 5 seconds. > storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE] > [--parallel=N] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. > storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE] > [--parallel=N] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete SOURCE [--parallel=N] [--dryrun] Deletes the file specified in the filename from the specified cloud. > storage list [SOURCE] [--recursive] [--parallel=N] > [--output=OUTPUT] [--dryrun] lists all the files from the container name specified on the specified cloud. storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N] creates a folder with the directory name specified on the specified cloud. > storage search DIRECTORY FILENAME [--recursive] > [--storage=SERVICE] > [--parallel=N] > [--output=OUTPUT] searches for the source in all the folders on the specified cloud. sync SOURCE DESTINATION puts the content of source to the destination. If --recursive is specified this is done recursively from the source If --async is specified, this is done asynchronously If a name is specified, the process can also be monitored with the status command by name. If the name is not specified all date is monitored. sync status The status for the asynchronous sync can be seen with this command config list Lists the configures storage services in the yaml file storage copy SOURCE DESTINATION Copies files from source storage to destination storage. The syntax of SOURCE and DESTINATION is: SOURCE - awss3:source.txt DESTINATION - azure:target.txt Description of the copy command: Command enables to Copy files between different cloud service providers, list and delete them. This command accepts `aws` , `google` and `local` as the SOURCE and TARGET provider. cms storage copy --source=SERVICE:SOURCE --target=DEST:TARGET Command copies files or directories from Source provider to Target Provider. cms storage slist --source=SERVICE:SOURCE Command lists all the files present in SOURCE provider's in the given SOURCE_FILE_DIR location This command accepts `aws` or `google` as the SOURCE provider cms storage sdelete --source=SERVICE:SOURCE Command deletes the file or directory from the SOURCE provider's SOURCE_FILE_DIR location Examples: > cms storage_service copy --source=local:test1.txt > --target=aws:uploadtest1.txt cms storage_service list --source=google:test cms storage_service delete --source=aws:uploadtest1.txt cms storage put test_file1.txt aws_test_file1.txt cms storage put ./recur_dir recur_dir_aws/ --recursive cms storage put ./recur_dir recur_dir_aws/ cms storage get aws_test_file1.txt aws_file1.txt cms storage get recur_dir_aws from_aws_dir cms storage get recur_dir_aws from_aws_dir --recursive cms storage list cms storage list --recursive cms storage list aws:recur_dir_aws --recursively cms storage delete aws:aws_test_file1.txt cms storage search recur_dir_aws recur_file1.txt Example: set storage=aws storage put SOURCE DESTINATION --recursive is the same as storage --storage=aws put SOURCE DESTINATION --recursive storage copy aws:source.txt oracle:target.txt """ # arguments.CONTAINER = arguments["--container"] VERBOSE(arguments) map_parameters(arguments, "dryrun", "recursive", "storage", "source", "target", "parallel") source = arguments.source target = arguments.target variables = Variables() parallelism = arguments.parallel or 1 arguments.storage = Parameter.expand(arguments.storage or variables['storage']) run_immediately = arguments['--run'] if arguments.monitor: provider = Provider(arguments.storage[0], parallelism=parallelism) status = arguments['--status'] or "all" output = arguments['--output'] or "table" result = provider.monitor(status=status, output=output) elif arguments.clean: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.clean() elif arguments.run: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.run() elif arguments['get']: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.get(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) if run_immediately: provider.run() elif arguments.put: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.put(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) if run_immediately: provider.run() elif arguments.create and arguments.dir: provider = Provider(arguments.storage[0], parallelism=parallelism) result = provider.create_dir(arguments.DIRECTORY) if run_immediately: provider.run() elif arguments.list: """ storage list SOURCE [--parallel=N] """ if variables['storage']: default_source = f"{variables['storage']}:/" else: default_source = "local:/" sources = arguments.SOURCE or default_source if not ":" in sources: sources = f"{variables['storage']}:{sources}" sources = Parameter.expand(sources) deletes = [] for source in sources: storage, entry = Parameter.separate(source) storage = storage or source or "local" deletes.append((storage, entry)) _sources = ', '.join(sources) for delete in deletes: service, entry = delete if arguments.dryrun: print(f"Dryrun: list {service}:{entry}") else: provider = Provider(service=service, parallelism=parallelism) provider.list(name=entry, recursive=arguments.recursive) if run_immediately: provider.run() return "" elif arguments.delete: """ storage delete SOURCE [--parallel=N] """ if variables['storage']: default_source = f"{variables['storage']}:/" else: default_source = "local:/" sources = arguments.SOURCE or default_source if not ":" in sources: sources = f"{variables['storage']}:{sources}" sources = Parameter.expand(sources) deletes = [] for source in sources: storage, entry = Parameter.separate(source) storage = storage or source or "local" deletes.append((storage, entry)) _sources = ', '.join(sources) answer = yn_choice(f"Would you like to delete {_sources}?", default="no") if answer: for delete in deletes: service, entry = delete if arguments.dryrun: print(f"Dryrun: delete {service}:{entry}") else: provider = Provider(service=service, parallelism=parallelism) provider.delete(name=entry) else: Console.error("Deletion canceled") if run_immediately: provider.run() return "" elif arguments.search: for storage in arguments.storage: provider = Provider(storage, parallelism=parallelism) provider.search(arguments.DIRECTORY, arguments.FILENAME, arguments.recursive) if run_immediately: provider.run() elif arguments.rsync: # TODO: implement raise NotImplementedError elif arguments['cc']: scloud, sfileDir = source.split(":", 1) or None tcloud, tfileDir = target.split(":", 1) or None print(f" Copying from Source {scloud} : {sfileDir} to Target " f" {tcloud} : {tfileDir}") cloudName = ["aws", "google"] if scloud in cloudName: provider = Provider(service=scloud, parallelism=parallelism) provider.copyFiles(scloud, sfileDir, tcloud, tfileDir) else: print("Not Implemented") return "" elif arguments.copy: scloud, sbucket = arguments['--source'].split(":", 1) or None tcloud, tbucket = arguments['--target'].split(":", 1) or None if scloud == "aws" or scloud == "google": provider = Provider(service=scloud, parallelism=parallelism) provider.copy(scloud, tcloud, sbucket, tbucket) else: provider = Provider(service=tcloud, parallelism=parallelism) provider.copy(arguments['--source'], arguments['--target'], arguments.recursive) if run_immediately: provider.run() return ""
class TestStorage(object): def create_file(self, location, content): d = Path(os.path.dirname(path_expand(location))) print() print("TESTDIR:", d) d.mkdir(parents=True, exist_ok=True) writefile(path_expand(location), content) def setup(self): variables = Variables() service = Parameter.expand(variables['storage'])[0] self.p = Provider(service=service) def test_create_source(self): HEADING() StopWatch.start("create source") self.sourcedir = path_expand("~/.cloudmesh/storage/test/") self.create_file("~/.cloudmesh/storage/test/a/a.txt", "content of a") self.create_file("~/.cloudmesh/storage/test/a/b/b.txt", "content of b") self.create_file("~/.cloudmesh/storage/test/a/b/c/c.txt", "content of c") StopWatch.stop("create source") # test if the files are ok assert True def test_put(self): HEADING() src = path_expand("~/.cloudmesh/storage/test/a/a.txt") dst = "/" StopWatch.start("put") test_file = self.p.put(src, dst) StopWatch.stop("put") pprint(test_file) assert test_file is not None def test_get(self): HEADING() src = path_expand("/a.txt") dst = path_expand("~/test.txt") StopWatch.start("get") file = self.p.get(src, dst) StopWatch.stop("get") pprint(file) assert file is not None def test_list(self): HEADING() src = '/' StopWatch.start("list") contents = self.p.list(src) StopWatch.stop("list") for c in contents: pprint(c) assert len(contents) > 0 def test_search(self): HEADING() src = '/' filename = 'test.txt' # # bug use named arguments # StopWatch.start("search") search_files = self.p.search(src, filename, True) StopWatch.stop("serach") pprint(search_files) assert len(search_files) > 0 def test_create_dir(self): HEADING() src = '/created_dir' StopWatch.start("create dir") directory = self.p.create_dir(src) StopWatch.stop("create dir") pprint(directory) assert directory is not None def test_delete(self): HEADING() src = '/created_dir' StopWatch.start("delete") self.p.delete(src) StopWatch.stop("delete") def test_benchmark(self): StopWatch.benchmark()
class Test_storage: def create_file(self, location, content): d = Path(os.path.dirname(path_expand(location))) print() print("TESTDIR:", d) d.mkdir(parents=True, exist_ok=True) writefile(path_expand(location), content) def setup(self): variables = Variables() service = Parameter.expand(variables['storage'])[0] self.p = Provider(service=service) def test_01_create_source(self): HEADING() self.sourcedir = path_expand("~/.cloudmesh/storage/test/") self.create_file("~/.cloudmesh/storage/test/a/a.txt", "content of a") self.create_file("~/.cloudmesh/storage/test/a/b/b.txt", "content of b") self.create_file("~/.cloudmesh/storage/test/a/b/c/c.txt", "content of c") # test if the files are ok assert True def test_01_put(self): HEADING() src = path_expand("~/.cloudmesh/storage/test/a/a.txt") dst = "/" test_file = self.p.put(src, dst) pprint(test_file) assert test_file is not None def test_02_get(self): HEADING() src = path_expand("/a.txt") dst = path_expand("~/test.txt") file = self.p.get(src, dst) pprint(file) assert file is not None def test_03_list(self): HEADING() src = '/' contents = self.p.list(src) for c in contents: pprint(c) assert len(contents) > 0 def test_04_search(self): HEADING() src = '/' filename = 'test.txt' # # bug use named arguments # search_files = self.p.search(src, filename, True) pprint(search_files) assert len(search_files) > 0 def test_05_create_dir(self): HEADING() src = '/created_dir' dir = self.p.create_dir(src) pprint(dir) assert dir is not None def test_06_delete(self): HEADING() src = '/created_dir' self.p.delete(src)
def do_storage(self, args, arguments): """ :: Usage: storage create dir DIRECTORY [--storage=SERVICE] [--parallel=N] storage get SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] storage put SOURCE DESTINATION [--recursive] [--storage=SERVICE] [--parallel=N] storage list [SOURCE] [--recursive] [--parallel=N] [--output=OUTPUT] [--dryrun] storage delete SOURCE [--parallel=N] [--dryrun] storage search DIRECTORY FILENAME [--recursive] [--storage=SERVICE] [--parallel=N] [--output=OUTPUT] storage sync SOURCE DESTINATION [--name=NAME] [--async] [--storage=SERVICE] storage sync status [--name=NAME] [--storage=SERVICE] storage config list [--output=OUTPUT] storage [--parallel=N] copy SOURCE DESTINATION [--recursive] storage copy --source=SOURCE:SOURCE_FILE_DIR --target=TARGET:TARGET_FILE_DIR This command does some useful things. Arguments: SOURCE SOURCE can be a directory or file DESTINATION DESTINATION can be a directory or file DIRECTORY DIRECTORY refers to a folder on the cloud service SOURCE:SOURCE_FILE_DIR source provider name: file or directory name TARGET:SOURCE_FILE_DIR destination provider name Options: --storage=SERVICE specify the cloud service name like aws or azure or box or google Description: commands used to upload, download, list files on different cloud storage services. storage put [options..] Uploads the file specified in the filename to specified cloud from the SOURCEDIR. storage get [options..] Downloads the file specified in the filename from the specified cloud to the DESTDIR. storage delete [options..] Deletes the file specified in the filename from the specified cloud. storage list [options..] lists all the files from the container name specified on the specified cloud. storage create dir [options..] creates a folder with the directory name specified on the specified cloud. storage search [options..] searches for the source in all the folders on the specified cloud. sync SOURCE DESTINATION puts the content of source to the destination. If --recursive is specified this is done recursively from the source If --async is specified, this is done asynchronously If a name is specified, the process can also be monitored with the status command by name. If the name is not specified all date is monitored. sync status The status for the asynchronous sync can be seen with this command config list Lists the configures storage services in the yaml file storage copy SOURCE DESTINATION Copies files from source storage to destination storage. The syntax of SOURCE and DESTINATION is: SOURCE - awss3:source.txt DESTINATION - azure:target.txt Description of the copy command: Command enables to Copy files between different cloud service providers, list and delete them. This command accepts `aws` , `google` and `local` as the SOURCE and TARGET provider. cms storage copy --source=SERVICE:SOURCE --target=DEST:TARGET Command copies files or directories from Source provider to Target Provider. cms storage slist --source=SERVICE:SOURCE Command lists all the files present in SOURCE provider's in the given SOURCE_FILE_DIR location This command accepts `aws` or `google` as the SOURCE provider cms storage sdelete --source=SERVICE:SOURCE Command deletes the file or directory from the SOURCE provider's SOURCE_FILE_DIR location Examples: cms storage_service copy --source=local:test1.txt --target=aws:uploadtest1.txt cms storage_service list --source=google:test cms storage_service delete --source=aws:uploadtest1.txt Example: set storage=azureblob storage put SOURCE DESTINATION --recursive is the same as storage --storage=azureblob put SOURCE DESTINATION --recursive storage copy azure:source.txt oracle:target.txt """ # arguments.CONTAINER = arguments["--container"] VERBOSE(arguments) map_parameters(arguments, "dryrun", "recursive", "storage", "source", "target") source = arguments.source target = arguments.target variables = Variables() VERBOSE(arguments) arguments.storage = Parameter.expand(arguments.storage) if arguments["get"]: provider = Provider(arguments.storage[0]) result = provider.get(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.put: provider = Provider(arguments.storage[0]) result = provider.put(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.create and arguments.dir: provider = Provider(arguments.storage[0]) result = provider.create_dir(arguments.DIRECTORY) elif arguments.list: """ storage list SOURCE [--parallel=N] """ sources = arguments.SOURCE or variables["storage"] or 'local:.' sources = Parameter.expand(sources) deletes = [] for source in sources: storage, entry = Parameter.separate(source) storage = storage or "local" deletes.append((storage, entry)) _sources = ', '.join(sources) for delete in deletes: service, entry = delete if arguments.dryrun: print(f"Dryrun: list {service}:{entry}") else: provider = Provider(service=service) provider.list(name=entry) return "" elif arguments.delete: """ storage delete SOURCE [--parallel=N] """ sources = arguments.SOURCE or variables["storage"] or 'local:.' sources = Parameter.expand(sources) deletes = [] for source in sources: storage, entry = Parameter.separate(source) storage = storage or "local" deletes.append((storage, entry)) _sources = ', '.join(sources) answer = yn_choice(f"Would you like to delete {_sources}?", default="no") if answer: for delete in deletes: service, entry = delete if arguments.dryrun: print(f"Dryrun: delete {service}:{entry}") else: provider = Provider(service=service) provider.delete(name=entry) else: Console.error("Deletion canceled") return "" elif arguments.search: for storage in arguments.storage: provider = Provider(storage) provider.search(arguments.DIRECTORY, arguments.FILENAME, arguments.recursive) elif arguments.rsync: # TODO: implement raise NotImplementedError elif arguments.copy: VERBOSE(f"COPY: Executing Copy command from {arguments.SOURCE} to " f"{arguments.DESTINATION} providers") print(f"DEBUG storage.py: INITIALIZE with {arguments.storage[0]} " "provider.") provider = Provider(arguments.storage[0]) result = provider.copy(arguments.SOURCE, arguments.DESTINATION, arguments.recursive) elif arguments.copy: scloud, sbucket = source.split(":", 1) or None tcloud, tbucket = target.split(":", 1) or None # print(scloud + " " + tcloud + " " + sbucket + " " + tbucket) if scloud == "aws" or scloud == "google": provider = Provider(service=scloud) provider.copy(scloud, tcloud, sbucket, tbucket) elif (scloud == "local" and tcloud == "aws") or (scloud == "local" and tcloud == "google"): provider = Provider(service=tcloud) provider.copy(scloud, tcloud, sbucket, tbucket) else: print("Not Implemented") return ""
class TestStorage(object): def create_file(self, location, content): d = Path(os.path.dirname(path_expand(location))) print() print("TESTDIR:", d) d.mkdir(parents=True, exist_ok=True) writefile(path_expand(location), content) def setup(self): variables = Variables() service = Parameter.expand(variables['storage'])[0] self.p = Provider(service=service) def test_create_source(self): HEADING() StopWatch.start("create source") self.sourcedir = path_expand("~/.cloudmesh/storage/test/") self.create_file("~/.cloudmesh/storage/test/a/a.txt", "content of a") self.create_file("~/.cloudmesh/storage/test/a/b/b.txt", "content of b") self.create_file("~/.cloudmesh/storage/test/a/b/c/c.txt", "content of c") StopWatch.stop("create source") # test if the files are ok assert True def test_put(self): HEADING() #root="~/.cloudmesh" #src = "storage/test/a/a.txt" # source = f"local:{src}" # destination = f"aws:{src}" # test_file = self.p.put(src, dst) #src = "storage_a:test/a/a.txt" src = "~/.cloudmesh/storage/test/" dst = '/' StopWatch.start("put") test_file = self.p.put(src, dst) StopWatch.stop("put") pprint(test_file) assert test_file is not None def test_put_recursive(self): HEADING() #root="~/.cloudmesh" #src = "storage/test/a/a.txt" # source = f"local:{src}" # destination = f"aws:{src}" # test_file = self.p.put(src, dst) #src = "storage_a:test/a/a.txt" src = "~/.cloudmesh/storage/test/" dst = '/' StopWatch.start("put") test_file = self.p.put(src, dst, True) StopWatch.stop("put") pprint(test_file) assert test_file is not None def test_get(self): HEADING() src = "/a.txt" dst = "~/.cloudmesh/storage/test" StopWatch.start("get") file = self.p.get(src, dst) StopWatch.stop("get") pprint(file) assert file is not None def test_list(self): HEADING() src = '/' StopWatch.start("list") contents = self.p.list(src) StopWatch.stop("list") for c in contents: pprint(c) assert len(contents) > 0 def test_search(self): HEADING() src = '/' filename = "a.txt" StopWatch.start("search") search_files = self.p.search(src, filename, True) StopWatch.stop("search") pprint(search_files) assert len(search_files) > 0 #assert filename in search_files[0]['cm']["name"] def test_create_dir(self): HEADING() src = 'created_dir' StopWatch.start("create dir") directory = self.p.create_dir(src) StopWatch.stop("create dir") pprint(directory) assert directory is not None def test_delete(self): HEADING() src = '/created_dir' StopWatch.start("delete") self.p.delete(src) StopWatch.stop("delete") def test_benchmark(self): Benchmark.print(sysinfo=False, csv=True, tag=cloud)