def main(): if len(sys.argv) == 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help": printUsage() sys.exit(1) chunk_id = sys.argv[-1] if not isValidChunkId(chunk_id): print("Invalid chunk id") sys.exit(1) # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() session = get_session(loop=loop) app = {} app["session"] = session app['bucket_name'] = config.get("bucket_name") app['node_count'] = 1 app['node_number'] = 0 app['deleted_ids'] = set() app['meta_cache'] = {} app['pending_s3_read'] = {} app['meta_cache'] = LruCache(mem_target=1024*1024, chunk_cache=False) app['chunk_cache'] = LruCache(mem_target=64*1024*1024, chunk_cache=True) domain = config.get("domain") if not domain: printUsage() sys.exit(-1) print("got domain:", domain) loop.run_until_complete(printChunkValues(app, domain, chunk_id)) loop.close()
def main(): do_update = False if len(sys.argv) < 4: printUsage() rootid = sys.argv[1] prefix_old = sys.argv[2] prefix_new = sys.argv[3] if len(sys.argv) > 4 and sys.argv[4] == "-update": do_update = True if not isValidUuid(rootid): print("Invalid root id!") sys.exit(1) if not isSchema2Id(rootid): print("This tool can only be used with Schema v2 ids") sys.exit(1) if prefix_old == prefix_new: print("prefix_old and prefix_new or the same") sys.exit(1) # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["bucket_name"] = config.get("bucket_name") app["prefix_old"] = prefix_old app["prefix_new"] = prefix_new app["do_update"] = do_update app["dataset_count"] = 0 app["matched_dset_uri"] = 0 app["indirect_dataset_keys"] = [] app["loop"] = loop session = get_session() app["session"] = session app["filter_map"] = {} # need the metadata cache since we will be calling into some SN methods metadata_mem_cache_size = int(config.get("metadata_mem_cache_size")) app['meta_cache'] = LruCache(mem_target=metadata_mem_cache_size, name="MetaCache") loop.run_until_complete(run_scan(app, rootid=rootid, update=do_update)) loop.close() print("datsets scanned:", app["dataset_count"]) print( "datasets with matching uri ('H5D_CONTIGUOUS_REF', 'H5D_CHUNKED_REF' layouts):", app["matched_dset_uri"]) print("done!")
def main(): if len(sys.argv) > 1 and (sys.argv[1] == "-h" or sys.argv[1] == "--help"): printUsage() if len(sys.argv) > 1 and sys.argv[1] == "--update": do_update = True else: do_update = False # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["bucket_name"] = config.get("bucket_name") app["loop"] = loop session = get_session() app["session"] = session loop.run_until_complete(run_scan(app, update=do_update)) loop.close() results = app["bucket_scan"] print("root_count:", results["root_count"]) print("info_count:", results["info_count"]) print("group_count", results["group_count"]) print("dataset_count:", results["dataset_count"]) print("datatype_count", results["datatype_count"]) print("chunk_count:", results["chunk_count"]) print('allocated_bytes:', results["allocated_bytes"]) print("metadata_bytes:", results["metadata_bytes"]) print("updated_count:", results["updated_count"]) print("done!")
def main(): if len(sys.argv) == 1 or len(sys.argv) > 1 and (sys.argv[1] == "-h" or sys.argv[1] == "--help"): printUsage() rootid = sys.argv[1] if not isValidUuid(rootid): print("Invalid root id!") sys.exit(1) if not isSchema2Id(rootid): print("This tool can only be used with Schema v2 ids") sys.exit(1) # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["bucket_name"] = config.get("bucket_name") app["loop"] = loop session = get_session(loop=loop) app["session"] = session loop.run_until_complete(run_delete(app, rootid)) loop.close() print("done!")
def main(): if len(sys.argv) == 1 or len(sys.argv) > 1 and (sys.argv[1] == "-h" or sys.argv[1] == "--help"): printUsage() rootid = sys.argv[1] if len(sys.argv) > 2 and sys.argv[2] == "-update": do_update = True else: do_update = False if not isValidUuid(rootid): print("Invalid root id!") sys.exit(1) if not isSchema2Id(rootid): print("This tool can only be used with Schema v2 ids") sys.exit(1) # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["bucket_name"] = config.get("bucket_name") app["loop"] = loop session = get_session(loop=loop) app["session"] = session loop.run_until_complete(run_scan(app, rootid=rootid, update=do_update)) loop.close() results = app["scanRoot_results"] datasets = results["datasets"] lastModified = datetime.fromtimestamp(results["lastModified"]) total_size = results["metadata_bytes"] + results["allocated_bytes"] print(f"lastModified: {lastModified}") print(f"size: {total_size}") print(f"num chunks: {results['num_chunks']}") print(f"num_groups: {results['num_groups']}") print(f"num_datatypes: {results['num_datatypes']}") print(f"num_datasets: {len(datasets)}") for dsetid in datasets: dataset_info = datasets[dsetid] print(f" {dsetid}: {dataset_info['lastModified']}, {dataset_info['num_chunks']}, {dataset_info['allocated_bytes']}") scan_start = datetime.fromtimestamp(results["scan_start"]) print(f"scan_start: {scan_start}") scan_complete = datetime.fromtimestamp(results["scan_complete"]) print(f"scan_complete: {scan_complete}") print("done!")
def main(): if len(sys.argv) > 1 and (sys.argv[1] == "-h" or sys.argv[1] == "--help"): printUsage() prefix = '' deliminator = '' suffix = '' showstats = False argn = 1 while argn < len(sys.argv): arg = sys.argv[argn] val = None if len(sys.argv) > argn + 1: val = sys.argv[argn + 1] if arg == "--prefix": prefix = val argn += 2 elif arg == "--deliminator": deliminator = val argn += 2 elif arg == "--suffix": suffix = val argn += 2 elif arg == "--showstats": showstats = True argn += 1 else: printUsage() print("prefix:", prefix) print("deliminator:", deliminator) print("suffix:", suffix) print("showstats:", showstats) # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["bucket_name"] = config.get("bucket_name") app["loop"] = loop session = get_session() app["session"] = session loop.run_until_complete( listObjects(app, prefix=prefix, deliminator=deliminator, suffix=suffix, showstats=showstats)) loop.close() print("done!")
def testStorUtil(self): cors_domain = config.get("cors_domain") print(f"cors_domain: [{cors_domain}]") bucket = config.get("hsds_unit_test_bucket") if not bucket: print( "No bucket configured, create bucket and export HSDS_UNIT_TEST_BUCKET=<bucket_name> to enable test" ) return # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() session = get_session(loop=loop) app = {} app["session"] = session app["bucket_name"] = bucket app["loop"] = loop loop.run_until_complete(self.stor_util_test(app)) loop.close()
def main(): if len(sys.argv) == 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help": printUsage() sys.exit(1) obj_id = sys.argv[-1] if not isValidUuid(obj_id): print("Invalid obj id") # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() session = get_session(loop=loop) app = {} app["session"] = session app['bucket_name'] = config.get("bucket_name") loop.run_until_complete(printS3Obj(app, obj_id)) loop.close()
def main(): if len(sys.argv) > 1 and (sys.argv[1] == "-h" or sys.argv[1] == "--help"): printUsage() sys.exit(1) # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() #loop.run_until_complete(init(loop)) session = get_session(loop=loop) app = {} app['bucket_name'] = config.get("bucket_name") app["session"] = session app["loop"] = loop loop.run_until_complete(deleteAll(app)) #releaseClient(app) loop.close() print("done!")
def main(): default_public_perm = { 'create': False, 'read': True, 'update': False, 'delete': False, 'readACL': False, 'updateACL': False } default_private_perm = { 'create': False, 'read': False, 'update': False, 'delete': False, 'readACL': False, 'updateACL': False } if len(sys.argv) == 1 or sys.argv[1] == "-h" or sys.argv[1] == "--help": printUsage() sys.exit(1) default_perm = default_public_perm # will switch if private is specified userarg = None domain = None for arg in sys.argv[1:]: if arg.startswith('--user='******'--user='******'--private': default_perm = default_private_perm elif arg.startswith('--domain='): domain = arg[len('--domain='):] else: print("Unexpected argument:", arg) printUsage() sys.exit(1) if not userarg: print("No user supplied") printUsage() sys.exit(1) usernames = [] if userarg[0] == '[' and userarg[-1] == ']': names = userarg[1:-1].split(',') for name in names: usernames.append(name) else: usernames.append(userarg) for username in usernames: if username != username.lower(): raise ValueError("username must be lowercase") if not username[0].isalpha(): raise ValueError( "first character of username must be character a-z") for c in username: if c != '_' and not c.isalnum(): raise ValueError( "username must consist of the characters a-z, numeric or underscore" ) if len(username) < 3: raise ValueError("username must have at least three characters") # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["loop"] = loop app["bucket_name"] = config.get("bucket_name") loop.run_until_complete( createDomains(app, usernames, default_perm, domain_name=domain)) loop.run_until_complete(shutdown(app)) loop.close() print("done!")
# Main # if __name__ == '__main__': base_folder = "/home" if len(sys.argv) > 1: last_arg = sys.argv[-1] if last_arg in ("-h", "--help"): print("Usage: python bucket_check.py <base_domain>") sys.exit(0) if not last_arg.startswith("-"): base_folder = last_arg print("base_folder:", base_folder) # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["bucket_name"] = config.get("bucket_name") app["s3objs"] = {} app["domains"] = {} # domain to root map app["roots"] = {} # root obj to domain map app["deleted_ids"] = set() app["bytes_in_bucket"] = 0 app["loop"] = loop loop.run_until_complete(bucketCheck(app, base_folder)) loop.close() print("done!")
print("total storage: {}".format(app["bytes_in_bucket"])) print("Num objects: {}".format(len(app["s3objs"]))) print("Num domains: {}".format(len(app["domains"]))) print("Num root groups: {}".format(len(app["roots"]))) print("Unlinked objects: {}".format(unlinked_count)) # # Main # if __name__ == '__main__': # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["bucket_name"] = config.get("bucket_name") app["anonymous_ttl"] = config.get("anonymous_ttl") app["s3objs"] = {} app["domains"] = {} # domain to root map app["roots"] = {} # root obj to domain map app["deleted_ids"] = set() app["bytes_in_bucket"] = 0 app["loop"] = loop session = get_session(loop=loop) app["session"] = session loop.run_until_complete(bucketCheck(app)) releaseClient(app) loop.close() print("done!")
def main(): if len(sys.argv) == 1 or len(sys.argv) > 1 and (sys.argv[1] == "-h" or sys.argv[1] == "--help"): printUsage() rootid = sys.argv[1] if len(sys.argv) > 2 and sys.argv[2] == "-update": do_update = True else: do_update = False if not isValidUuid(rootid): print("Invalid root id!") sys.exit(1) if not isSchema2Id(rootid): print("This tool can only be used with Schema v2 ids") sys.exit(1) # we need to setup a asyncio loop to query s3 loop = asyncio.get_event_loop() app = {} app["bucket_name"] = config.get("bucket_name") app["loop"] = loop session = get_session() app["session"] = session app["filter_map"] = {} # need the metadata cache since we will be calling into some SN methods metadata_mem_cache_size = int(config.get("metadata_mem_cache_size")) app['meta_cache'] = LruCache(mem_target=metadata_mem_cache_size, name="MetaCache") loop.run_until_complete(run_scan(app, rootid=rootid, update=do_update)) loop.close() results = app["scanRoot_results"] datasets = results["datasets"] lastModified = datetime.fromtimestamp(results["lastModified"]) print(f"lastModified: {lastModified}") if "md5_sum" in results: checksum = results["md5_sum"] print(f"md5_sum: {checksum}") print(f"metadata bytes: {results['metadata_bytes']}") print(f"allocated bytes: {results['allocated_bytes']}") print(f"logical bytes: {results['logical_bytes']}") print(f"num chunks: {results['num_chunks']}") print(f"linked chunks: {results['num_linked_chunks']}") print(f"linked bytes: {results['linked_bytes']}") print(f"num_groups: {results['num_groups']}") print(f"num_datatypes: {results['num_datatypes']}") print(f"num_datasets: {len(datasets)}") if datasets: print( " dataset_id\tlast_modified\tnum_chunks\tallocated_bytes\tlogical_bytes\tlinked_bytes\tnum_link_chunks" ) for dsetid in datasets: dataset_info = datasets[dsetid] lm = dataset_info['lastModified'] nc = dataset_info['num_chunks'] ab = dataset_info['allocated_bytes'] lb = dataset_info['logical_bytes'] ln = dataset_info['linked_bytes'] nl = dataset_info['num_linked_chunks'] print(f" {dsetid}: {lm}, {nc}, {ab}, {lb}, {ln}, {nl}") scan_start = datetime.fromtimestamp(results["scan_start"]) print(f"scan_start: {scan_start}") scan_complete = datetime.fromtimestamp(results["scan_complete"]) print(f"scan_complete: {scan_complete}") print("done!")