async def verifyDomain(domain): """ create domain if it doesn't already exist """ params = {"host": domain} headers = getRequestHeaders() client = globals["client"] req = getEndpoint() + '/' root_id = None log.info("GET " + req) timeout = config.get("timeout") async with client.get(req, headers=headers, timeout=timeout, params=params) as rsp: if rsp.status == 200: domain_json = await rsp.json() else: log.info("got status: {}".format(rsp.status)) if rsp.status == 200: root_id = domain_json["root"] elif rsp.status == 404: # create the domain setupDomain(domain) async with client.get(req, headers=headers, timeout=timeout, params=params) as rsp: if rsp.status == 200: domain_json = await rsp.json() root_id = domain_json["root"] else: log.error("got status: {} for GET req: {}".format(rsp.status, req)) raise HttpProcessingError(code=rsp.status, message="Service error") globals["root"] = root_id
async def scanRootKeys(app, update=False): # iterate through all s3 root keys in the bucket. # # Note: not re-entrant! Only one scanRoot an be run at a time per app. log.info("scanRootKeys") app["scanRootKeys_update"] = update await getStorKeys(app, prefix="db/", deliminator='/', include_stats=False, callback=getS3RootKeysCallback)
async def bucketCheck(app): """ Verify that contents of bucket are self-consistent """ now = int(time.time()) log.info("bucket check {}".format(unixTimeToUTC(now))) # do initial listKeys await listKeys(app) # clear used flags clearUsedFlags(app) # mark objs await markObjs(app) unlinked_count = 0 s3objs = app["s3objs"] for objid in s3objs: if isValidUuid(objid) and not isValidChunkId(objid): try: s3obj = await getS3Obj(app, objid) if s3obj.used is False: unlinked_count += 1 except HTTPInternalServerError as hpe: log.warn("got error retreiving {}: {}".format(objid, hpe.code)) domains = app["domains"] for domain in domains: print("domain:", domain) roots = app["roots"] for root in roots: print("root:", root) top_level_domains = [] for domain in domains: if domain[0] != '/': log.error("unexpected domain: {}".format(domain)) continue if domain[1:].find('/') == -1: top_level_domains.append(domain) print("top-level-domains:") for domain in top_level_domains: print(domain) print("=" * 80) print("total storage: {}".format(app["bytes_in_bucket"])) print("Num objects: {}".format(len(app["s3objs"]))) print("Num domains: {}".format(len(app["domains"]))) print("Num root groups: {}".format(len(app["roots"]))) print("Unlinked objects: {}".format(unlinked_count))
async def getKeysCallback(app, s3keys): log.info(f"getKeysCallback, {len(s3keys)} items") if not isinstance(s3keys, list): log.error("expected list result for objDeleteCallback") raise ValueError("unexpected callback format") if "root_prefix" not in app or not app["root_prefix"]: log.error("Unexpected getKeysCallback") raise ValueError("Invalid getKeysCallback") prefix = app["root_prefix"] prefix_len = len(prefix) for s3key in s3keys: if not s3key.startswith(prefix): log.error(f"Unexpected key {s3key} for prefix: {prefix}") raise ValueError("invalid s3key for getKeysCallback") if not s3key.endswith(".dataset.json"): log.info(f"got unexpected key {s3key}, ignoring") continue dset_key = prefix + s3key[prefix_len:] log.info(f"getKeys - :{dset_key}") await checkDataset(app, dset_key) log.info("getKeysCallback complete")
async def createDomain(app, domain, domain_json): try: s3_key = getS3Key(domain) domain_exists = await isS3Obj(app, s3_key) if domain_exists: raise ValueError("Domain already exists") parent_domain = getParentDomain(domain) if parent_domain is None: raise ValueError("Domain must have a parent") log.info("writing domain") await putS3JSONObj(app, s3_key, domain_json) print("domain created! s3_key: {} domain_json: {}".format( s3_key, domain_json)) except ValueError as ve: print("Got ValueError exception: {}".format(str(ve))) except ClientOSError as coe: print("Got S3 error: {}".format(str(coe)))
async def createDomain(app, domain, domain_json): try: domain = app["bucket_name"] + domain print("domain:", domain) s3_key = getS3Key(domain) print("s3_key: ", s3_key) domain_exists = await isStorObj(app, s3_key) if domain_exists: raise ValueError("Domain already exists") parent_domain = getParentDomain(domain) if parent_domain is None: raise ValueError("Domain must have a parent") log.info("writing domain") await putStorJSONObj(app, s3_key, domain_json) print("domain created! s3_key: {} domain_json: {}".format( s3_key, domain_json)) except ValueError as ve: print("Got ValueError exception: {}".format(str(ve))) raise
async def getS3RootKeysCallback(app, s3keys): log.info(f"getS3RootKeysCallback, {len(s3keys)} items") if not isinstance(s3keys, list): log.error("expected list result for s3keys callback") raise ValueError("unexpected callback format") results = app["bucket_scan"] for s3key in s3keys: log.info(f"got key: {s3key}") if not s3key.startswith("db/") or s3key[-1] != '/': log.error(f"unexpected key for getS3RootKeysCallback: {s3key}") continue root_id = getObjId(s3key + ".group.json") log.info(f"root_id: {root_id}") results["root_count"] += 1 info_key = s3key + ".info.json" if app["scanRootKeys_update"]: log.info("updating...") await scanRoot(app, root_id, update=True) info_obj = None try: info_obj = await getStorJSONObj(app, info_key) except HTTPNotFound: pass # info.json not created yet except HTTPInternalServerError as ie: log.warn(f"error getting s3obj: {ie}") continue if info_obj: log.info(f"got obj: {info_obj}") results["info_count"] += 1 results["group_count"] += info_obj["num_groups"] results["dataset_count"] += len(info_obj["datasets"]) results["datatype_count"] += info_obj["num_datatypes"] results["chunk_count"] += info_obj["num_chunks"] results["allocated_bytes"] += info_obj["allocated_bytes"] results["metadata_bytes"] += info_obj["metadata_bytes"]
async def run_scan(app, rootid, update=False): root_key = getS3Key(rootid) if not root_key.endswith("/.group.json"): raise ValueError("unexpected root key") root_prefix = root_key[:-(len(".group.json"))] app["root_prefix"] = root_prefix try: await getStorKeys(app, prefix=root_prefix, suffix=".dataset.json", include_stats=False, callback=getKeysCallback) except ClientError as ce: log.error(f"removeKeys - getS3Keys faiiled: {ce}") except HTTPNotFound: log.warn( f"getStorKeys - HTTPNotFound error for getStorKeys with prefix: {root_prefix}" ) except HTTPInternalServerError: log.error( f"getStorKeys - HTTPInternalServerError for getStorKeys with prefix: {root_prefix}" ) except Exception as e: log.error( f"getStorKeys - Unexpected Exception for getStorKeys with prefix: {root_prefix}: {e}" ) # update all chunks for datasets with H5D_CHUNKED_REF_INDIRECT layout indirect_dataset_keys = app["indirect_dataset_keys"] for prefix in indirect_dataset_keys: log.info(f"got inidirect prefix: {prefix}") # TBD... await releaseStorageClient(app)
def main(): domain = getTestDomainName("mkgroups_perf") print("domain: {}".format(domain) ) log.info("initializing") signal.signal(signal.SIGTERM, sig_handler) # add handlers for early exit signal.signal(signal.SIGINT, sig_handler) loop = asyncio.get_event_loop() globals["loop"] = loop #domain = helper.getTestDomainName() # create a client Session here so that all client requests # will share the same connection pool max_tcp_connections = int(config.get("max_tcp_connections")) client = ClientSession(loop=loop, connector=TCPConnector(limit=max_tcp_connections)) globals["client"] = client globals["group_count"] = 0 globals["grp_request_count"] = 0 globals["lnk_request_count"] = 0 globals["grp_failed_posts"] = 0 globals["lnk_failed_posts"] = 0 globals["group_target"] = config.get("group_target") max_concurrent_tasks = config.get("max_concurrent_tasks") loop.run_until_complete(getEndpoints()) if len(globals["sn_endpoints"]) == 0: log.error("no SN endpoints found!") loop.close() client.close() sys.exit() for endpoint in globals["sn_endpoints"]: log.info("got endpoint: {}".format(endpoint)) loop.run_until_complete(verifyDomain(domain)) globals["domain"] = domain # save the domain globals["start_time"] = time.time() # start making groups! while globals["grp_request_count"] < globals["group_target"]: tasks = [] count = max_concurrent_tasks if globals["group_target"] - globals["grp_request_count"] < count: count = globals["group_target"] - globals["grp_request_count"] log.info("adding {} tasks".format(count)) for i in range(count): tasks.append(asyncio.ensure_future(createGroup())) # got a batch, move them out! loop.run_until_complete(asyncio.gather(*tasks)) tasks = [] loop.close() client.close() globals["stop_time"] = time.time() print_results()
async def getEndpoints(): docker_machine_ip = config.get("docker_machine_ip") req = "{}/nodestate/sn".format(config.get("head_endpoint")) client = globals["client"] async with client.get(req) as rsp: if rsp.status == 200: rsp_json = await rsp.json() nodes = rsp_json["nodes"] sn_endpoints = [] docker_links = checkDockerLink() for node in nodes: if not node["host"]: continue if docker_links: # when running in docker, use the machine address as host host = "hsds_sn_{}".format(node["node_number"]) elif docker_machine_ip: host = docker_machine_ip else: host = node["host"] url = "http://{}:{}".format(host, node["port"]) sn_endpoints.append(url) log.info("{} endpoints".format(len(sn_endpoints))) globals["sn_endpoints"] = sn_endpoints
async def createGroup(): """ create a new group and link it to the parent group with link name of group name """ client = globals["client"] domain = globals["domain"] params = {"host": domain} base_req = getEndpoint() headers = getRequestHeaders() # create a new group req = base_req + "/groups" log.info("POST:" + req) globals["grp_request_count"] += 1 group_name = globals["grp_request_count"] timeout = config.get("timeout") async with client.post(req, headers=headers, timeout=timeout, params=params) as rsp: if rsp.status != 201: log.error("POST {} failed with status: {}, rsp: {}".format(req, rsp.status, str(rsp))) globals["grp_failed_posts"] += 1 raise HttpProcessingError(code=rsp.status, message="Unexpected error") else: globals["group_count"] += 1 log.info("group_count: {}".format(globals["group_count"])) group_json = await rsp.json() group_id = group_json["id"] # link group to parent root_id = globals["root"] group_name = "group_{}".format(group_name) req = base_req + "/groups/" + root_id + "/links/" + group_name data = {"id": group_id } log.info("PUT " + req) globals["lnk_request_count"] += 1 async with client.put(req, data=json.dumps(data), headers=headers, timeout=timeout, params=params) as rsp: if rsp.status == 409: # another task has created this link already log.warn("got 409 in request: " + req) elif rsp.status != 201: globals["lnk_failed_posts"] += 1 log.error("got http error: {} for request: {}, rsp: {}".format(rsp.status, req, rsp)) raise HttpProcessingError(code=rsp.status, message="Unexpected error") else: link_created = True return group_id
async def shutdown(app): log.info("closing storage connections") await releaseStorageClient(app)
async def bucketCheck(app, base_folder): """ Verify that contents of bucket are self-consistent """ now = int(time.time()) log.info("bucket check {}".format(unixTimeToUTC(now))) bucket = app["bucket_name"] if base_folder.startswith('/'): # slash is not part of the storage key prefix = base_folder[1:] else: prefix = base_folder keys = await getStorKeys(app, prefix=prefix, suffix='domain.json') root_count = 0 group_count = 0 dataset_count = 0 datatype_count = 0 chunk_count = 0 total_chunk_bytes = 0 total_metadata_bytes = 0 if not keys: print("no storage keys were found!") return log.info(f"got {len(keys)} keys") print( "name, num_groups, num_datasets, num_datatypes, num chunks, metadata bytes, chunk bytes" ) for key in keys: log.info(f"got key: {key}") domain_json = await getStorJSONObj(app, key, bucket=bucket) #print("domain_json:", domain_json) if "root" not in domain_json: log.info(f"skipping folder object: {key}") continue root_id = domain_json["root"] scan = await scanRoot(app, root_id, bucket=bucket) log.debug(f"got scan_results: {scan}") num_groups = scan["num_groups"] datasets = scan["datasets"] num_datasets = len(datasets) num_datatypes = scan["num_datatypes"] num_chunks = scan["num_chunks"] chunk_bytes = scan["allocated_bytes"] metadata_bytes = scan["metadata_bytes"] print( f"{key},{num_groups},{num_datasets},{num_datatypes},{num_chunks},{metadata_bytes},{chunk_bytes}" ) # TBD - get service scan results from .info.json and compare to ones just calculated root_count += 1 group_count += num_groups dataset_count += num_datasets datatype_count += num_datatypes chunk_count += num_chunks total_chunk_bytes += chunk_bytes total_metadata_bytes += metadata_bytes await releaseStorageClient(app) print("") print("Totals") print("=" * 40) print(f"folders: {len(keys) - root_count}") print(f"domains: {root_count}") print(f"groups: {group_count}") print(f"datasets: {dataset_count}") print(f"chunk count {chunk_count}") print(f"metadata bytes: {total_metadata_bytes}") print(f"chunk bytes: {total_chunk_bytes}") print("")
async def checkDataset(app, dset_key): log.info(f"checkDataset for key: {dset_key}") dset_json = await getStorJSONObj(app, dset_key) dset_id = dset_json["id"] prefix_old = app["prefix_old"] prefix_new = app["prefix_new"] do_update = app["do_update"] indirect_dataset_keys = app["indirect_dataset_keys"] app["dataset_count"] += 1 log.info(f"checkDataset for id: {dset_id}") if "layout" not in dset_json: log.info("no layout found") return layout_json = dset_json["layout"] if "class" not in layout_json: log.warn(f"no class found in layout for id: {dset_id}") return layout_class = layout_json["class"] log.info(f"got layout_class: {layout_class}") if layout_class in ('H5D_CONTIGUOUS_REF', 'H5D_CHUNKED_REF'): if "file_uri" not in layout_json: log.warn( f"Expected to find key 'file_uri' in layout_json for id: {dset_id}" ) return file_uri = layout_json["file_uri"] if file_uri.startswith(prefix_old): new_file_uri = prefix_new + file_uri[len(prefix_old):] log.info(f"replacing uri: {file_uri} with {new_file_uri}") app["matched_dset_uri"] += 1 if do_update: # update the dataset json layout_json["file_uri"] = new_file_uri dset_json["layout"] = layout_json # write back to storage try: await putStorJSONObj(app, dset_key, dset_json) log.info(f"dataset {dset_id} updated") except Exception as e: log.error(f"get exception writing dataset json: {e}") elif layout_class == 'H5D_CHUNKED_REF_INDIRECT': # add to list to be scanned later indirect_dataset_keys += dset_key[:-len(".dataset.json")] else: log.info(f"skipping check for layout_class: {layout_class}")
async def shutdown(app): log.info("closing S3 connections") await releaseClient(app)