def walk_tree(QHOST, QUSER, QPASS, start_path): global gvars if start_path[-1] != "/": start_path = start_path + "/" log("Tree walk on Qumulo cluster %s starting at path %s" % (QHOST, start_path)) gvars = Gvars(QHOST, QUSER, QPASS) the_pool = multiprocessing.Pool(16, worker_main) rc = RestClient(gvars.QHOST, 8000) rc.login(gvars.QUSER, gvars.QPASS) root = rc.fs.read_dir_aggregates(path=start_path, max_depth=0) log("Directories to walk: %12s" % "{:,}".format(int(root["total_directories"]))) log(" Files to walk: %12s" % "{:,}".format(int(root["total_files"]))) add_to_queue({"path": start_path, "max_depth": 5}) time.sleep(0.1) # wait a bit for the queue to get build up. wait_count = 0 while gvars.the_queue_len.value > 0: wait_count += 1 if (wait_count % 50) == 0: # show status every ~5 seconds log("Processed %s entries. Queue length: %s" % (gvars.done_queue_len.value, gvars.the_queue_len.value)) time.sleep(0.1) the_pool.terminate() log("Processed %s entries." % gvars.done_queue_len.value) log("Done with tree walk. Combining results") fw = open("file-list.txt", "w") for f in glob.glob('out-*.txt'): fr = open(f, "r") fw.write(fr.read()) fr.close() os.remove(f) del gvars log("Results save to file: file-list.txt")
def has_user(self, username): logger.debug("has_user(%s)" % username) local_rc = RestClient(API_HOST, API_PORT) local_rc.login(API_USER, API_PASS) response = local_rc.users.list_users() name_list = [user['name'] for user in response] return username in name_list
def main(): parser = argparse.ArgumentParser( description= 'Walk levels of a known Qumulo filesystem tree and gather capacity metrics' ) parser.add_argument('-s', help='Qumulo hostname', required=True) parser.add_argument('-u', help='Qumulo api user', required=True) parser.add_argument('-p', help='Qumulo api passord') parser.add_argument('-d', help='Starting directory path', required=True) parser.add_argument('-o', help='Output file name', default='qumulo-data-by-directory.csv') parser.add_argument('-l', nargs='+', help='Level names', required=True) args, other_args = parser.parse_known_args() if not args.p: args.p = getpass() creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p} level_names = [] # initialize the REST client rc = RestClient(creds["QHOST"], 8000) rc.login(creds["QUSER"], creds["QPASS"]) get_capacity_aggregates(rc, args.d, args.l, args.o) print("Created file: %s" % args.o)
def process_snap_diff(creds, path, snap_before, snap_after): q = multiprocessing.Queue() q_len = multiprocessing.Value("i", 0) q_lock = multiprocessing.Lock() pool = multiprocessing.Pool(MAX_WORKER_COUNT, snap_worker, (creds, q, q_lock, q_len)) rc = RestClient(creds["QHOST"], 8000) rc.login(creds["QUSER"], creds["QPASS"]) results = rc.snapshot.get_all_snapshot_tree_diff(older_snap=snap_before['id'], newer_snap=snap_after['id']) ent_list = [] for res in results: for ent in res['entries']: print(ent) add_to_q(q, q_lock, q_len, {"type":"diff_item", "value": ent}) # ent_list.append(ent) # if len(ent_list) > 1: # add_to_q(q, q_lock, q_len, ent_list) # ent_list = [] # add_to_q(q, q_lock, q_len, ent_list) while True: log_it("Queue length: %s" % q_len.value) time.sleep(WAIT_SECONDS) if q_len.value <= 0: break
def rc_get_ips(creds: Creds) -> Sequence[str]: rc = RestClient(creds["QHOST"], 8000) rc.login(creds["QUSER"], creds["QPASS"]) ips = [] for d in rc.network.list_network_status_v2(1): ips.append(d["network_statuses"][0]["address"]) return ips
def main(): parser = argparse.ArgumentParser(description='') parser.add_argument('-s', help='Qumulo hostname', required=True) parser.add_argument('-u', help='Qumulo API user', default=os.getenv('QUSER') or 'admin') parser.add_argument('-p', help='Qumulo API password', default=os.getenv('QPASS') or 'admin') parser.add_argument('-d', help='Root Directory', default='/') try: args, other_args = parser.parse_known_args() except: print("-" * 80) parser.print_help() print("-" * 80) sys.exit(0) if args.d != '/': args.d = re.sub('/$', '', args.d) + '/' creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p} log_it("Log in to: %s" % (args.s)) rc = RestClient(creds["QHOST"], 8000) rc.login(creds["QUSER"], creds["QPASS"]) res = rc.snapshot.list_snapshot_statuses() snap_before = None snap_after = None for snap in sorted(res['entries'], key=lambda d: int(d['id'])): if snap['name'] == SNAP_NAME and snap['source_file_path'] == args.d: if snap_before is None: snap_before = snap elif snap_after is None: snap_after = snap if snap_before and not snap_after: snap_after = rc.snapshot.create_snapshot(path=args.d, name=SNAP_NAME) log_it("Created new snapshot %s on %s" % (snap_after['id'], args.d)) if snap_before and snap_after: log_it("Diffing snaps on %s between %s and %s" % (args.d, snap_before['timestamp'][0:16], snap_after['timestamp'][0:16])) process_snap_diff(creds, args.d, snap_before['id'], snap_after['id']) rc.snapshot.delete_snapshot(snapshot_id=snap_before['id']) else: # initial tree walk snap_before = rc.snapshot.create_snapshot(path=args.d, name=SNAP_NAME) log_it("Initial tree walk for: %s+snap:%s" % (args.d, snap_before['id'])) log_file = "output-qumulo-fs-index-%s-tree.txt" % (re.sub( "[^a-z0-9]+", "_", args.d)) w = QWalkWorker( creds, Search([ '--re', '.', '--cols', 'dir_id,id,type,path,name,size,blocks,owner,change_time,link_target,NEW' ]), args.d, str(snap_before['id']), None, log_file, None) w.run()
def impersonate_user(self, username, password): """This should probably return a RestClient assuming it gets called after login """ logger.debug("impersonate_user() returning RestClient") local_rc = RestClient(API_HOST, API_PORT) local_rc.login(username, password) return local_rc
def worker_main(): global gvars rc = RestClient(gvars.QHOST, 8000) rc.login(gvars.QUSER, gvars.QPASS) while True: item = gvars.the_queue.get(True) list_dir(rc, item) with gvars.the_queue_len.get_lock(): gvars.the_queue_len.value -= 1
def login(configdict): '''Obtain credentials from the REST server''' try: rc = RestClient(configdict['host'], configdict['port']) rc.login(configdict['user'], configdict['password']) except Exception, excpt: print "Error connecting to the REST server: %s" % excpt print __doc__ sys.exit(1)
def read_time_series_from_cluster(host: str, user: str, password: str, port: int) -> List[Mapping[str, Any]]: """ Communicates with the cluster to grab the analytics in time series format """ rest_client = RestClient(host, port) rest_client.login(user, password) return rest_client.analytics.time_series_get( begin_time=calculate_begin_time(CSV_FILENAME))
def api_perf_counters(): rc = get_rc() pc_data = {} for n in rc.network.list_network_status_v2(1): rc = RestClient(n['network_statuses'][0]['address'], 8000) rc.login('admin', app.config['API_PASSWORD']) resp = rc.request("GET", "/v1/metrics/") pc_data[n['node_id']] = resp return json.dumps(pc_data)
def ids_to_attrs(cluster, id_attr_list): inode_types = {} qumulo_client = RestClient(cluster['host'], 8000) qumulo_client.login(cluster['user'], cluster['password']) for inode_id in id_attr_list: try: attrs = qumulo_client.fs.get_attr(id_=inode_id) if inode_id not in inode_types: inode_types[inode_id] = attrs["type"] except: pass return inode_types
def run(self) -> None: if not os.path.exists("old-queue.txt"): self.run_task.work_start(self) rc = RestClient(self.creds["QHOST"], 8000) rc.login(self.creds["QUSER"], self.creds["QPASS"]) if self.snap: d_attr = rc.fs.read_dir_aggregates(path=self.start_path, snapshot=self.snap, max_entries=0) else: d_attr = rc.fs.read_dir_aggregates(path=self.start_path, max_entries=0) d_attr["total_directories"] = 1 + int(d_attr["total_directories"]) d_attr["total_inodes"] = d_attr["total_directories"] + int( d_attr["total_files"]) log_it( "Walking - %(total_directories)9s dir|%(total_inodes)10s inod" % d_attr) self.add_to_queue({ "type": "list_dir", "path_id": d_attr["id"], "snapshot": self.snap }) self.wait_for_complete() else: with open("old-queue.txt", "r") as fr: last_time = time.time() for line in fr: # back off because we have a lot of directories now while self.queue_len.value > MAX_QUEUE_LENGTH: if time.time() - last_time >= WAIT_SECONDS: self.print_status() last_time = time.time() time.sleep(1) self.add_to_queue({ "type": "list_dir", "path_id": line.strip(), "snapshot": self.snap, }) if time.time() - last_time >= WAIT_SECONDS: self.print_status() last_time = time.time() os.remove("old-queue.txt") self.wait_for_complete() self.pool.close() self.pool.join() self.queue.close() self.queue.join_thread() if self.rc: rc.close() del rc del self.pool del self.queue
def worker_main(): global gvars proc = multiprocessing.current_process() rc = RestClient(gvars.QHOST, 8000) rc.login(gvars.QUSER, gvars.QPASS) out_file = open("out-%s.txt" % proc.pid, "w") while True: item = gvars.the_queue.get(True) list_dir(rc, item, out_file) out_file.flush() with gvars.the_queue_len.get_lock(): gvars.the_queue_len.value -= 1
def main(): parser = argparse.ArgumentParser(description='Example command usage usage:\npython timeseries-to-csv.py --host product --user admin --pass admin' , epilog='.' , formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("--host", required=True, help="Required: Specify host (Qumulo cluster)") parser.add_argument("--user", required=True, help="Specify api user") parser.add_argument("--pass", required=True, dest="passwd", help="Specify api password ") args = parser.parse_args() rc = RestClient(args.host, 8000) rc.login(args.user, args.passwd); last_line = None columns = ["iops.read.rate", "iops.write.rate", "throughput.read.rate", "throughput.write.rate", "reclaim.deferred.rate", "reclaim.snapshot.rate"] csv_file_name = "qumulo-timeseries-data.csv" if os.path.exists(csv_file_name): # read to the last line in the file with open(csv_file_name, "rb") as csvfile: reader = csv.reader(csvfile) for row in reader: last_line = row # at most we'll have 1 day of data begin_time = int(time.time()) - 60 * 60 * 24 # otherwise, pull only the latest data, if we already have a file if last_line is not None: begin_time = int(last_line[0]) + 5 results = rc.analytics.time_series_get(begin_time = begin_time) print("Appending %s results to '%s'." % (len(results[0]['values']), csv_file_name)) data = {} for i in range(0,len(results[0]['times'])-1): ts = results[0]['times'][i] data[ts] = [None] * len(columns) for series in results: if series['id'] not in columns: continue for i in range(0,len(series['values'])): ts = series['times'][i] data[ts][columns.index(series['id'])] = series['values'][i] fw = open(csv_file_name, "a") if last_line is None: fw.write("unix.timestamp,gmtime," + ",".join(columns) + "\r\n") for ts in sorted(data): gmt = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(ts)) fw.write("%s,%s," % (ts, gmt) + ",".join([str(d) for d in data[ts]]) + "\r\n") fw.close()
def get_many_clients(self, creds, client_count): rc = RestClient(creds["QHOST"], 8000) rc.login(creds["QUSER"], creds["QPASS"]) ips = [] rcs = [] for d in rc.network.list_network_status_v2(1): ips.append(d['network_statuses'][0]['address']) for i in range(0, client_count): rcc = RestClient(ips[i % len(ips)], 8000) rcc.login(creds["QUSER"], creds["QPASS"]) rcs.append(rcc) return rcs
def main(): # Initial setup rc1 = RestClient(cluster1, 8000) rc1.login(api_user, api_password) rc2 = RestClient(cluster2, 8000) rc2.login(api_user, api_password) relationship_id = get_relationship_id(rc2, "/keyva_demo/") rc2.replication.make_target_writable(relationship_id=relationship_id) rc1.replication.modify_source_relationship(relationship_id=relationship_id, source_root_read_only=True) time.sleep(15) rc2.replication.reverse_target_relationship(relationship_id=relationship_id, source_address=cluster1) rc1.replication.reconnect_target_relationship(relationship_id=relationship_id)
def get_file_attrs(x): credentials, paths = x client = RestClient(credentials["cluster"], credentials["port"]) client.login(credentials["user"], credentials["password"]) result = [] for path in paths: if seen.has_key(path): result += [seen[path]] continue str_owner = enumerate_owner(client, path) seen[path] = str_owner result.append(str_owner) return result
def do_it(opts, args): credentials = { "user": opts.user, "password": opts.password, "cluster": opts.cluster } # Qumulo API login client = RestClient(opts.cluster, opts.port) client.login(opts.user, opts.password) total_capacity_used = int( client.fs.read_dir_aggregates(args[0])['total_capacity']) pool = Pool(opts.concurrency) # First build a vector of all samples... samples = get_samples(pool, args[0], credentials, opts) # Then get a corresponding vector of owner strings owner_vec = get_owner_vec(pool, credentials, samples, opts) owners = {} directories = {} # Create a mapping of user to tree... for s, owner in zip(samples, owner_vec): owners.setdefault(owner, SampleTreeNode("")) owners[owner].insert(s["name"], 1) def format_capacity(samples): bytes_per_terabyte = 1000.**4 if opts.dollars_per_terabyte != None: return "$%0.02f/month" % (samples * total_capacity_used / opts.samples / bytes_per_terabyte * opts.dollars_per_terabyte) else: return pretty_print_capacity(samples * total_capacity_used / opts.samples) print "Total: %s" % (format_capacity(opts.samples)) sorted_owners = sorted( owners.items(), lambda x, y: cmp(y[1].sum_samples, x[1].sum_samples)) # For each owner, print total used, then refine the tree and dump it. for name, tree in sorted_owners: print "Owner %s (~%0.1f%%/%s)" % (name, tree.sum_samples / float( opts.samples) * 100, format_capacity(tree.sum_samples)) tree.prune_until(max_leaves=opts.max_leaves, min_samples=opts.min_samples) print tree.__str__(" ", lambda x: format_capacity(x))
def get_file_attrs(x): credentials, paths = x client = RestClient(credentials["cluster"], 8000) client.login(credentials["user"], credentials["password"]) result = [] for path in paths: if seen.has_key(path): result += [seen[path]] continue owner_id = client.fs.get_attr(path)["owner"] str_owner = translate_owner_to_owner_string(client, owner_id) seen[path] = str_owner result.append(str_owner) return result
def timestamps(): ''' Get Qumulo Timestamps ''' # Get user informaation creds = Creds.objects.first() user = creds.user password = creds.password ipaddress = creds.ipaddress columns = [ "iops.read.rate", "iops.write.rate", "throughput.read.rate", "throughput.write.rate", "reclaim.deferred.rate", "reclaim.snapshot.rate" ] # feed = [] rc = RestClient(ipaddress, 8000) rc.login(user, password) # begin_time = int(time.time()) - 60 * 60 * 24 results = rc.analytics.time_series_get(begin_time=begin_time) data = {} # for i in range(0, len(results[0]['times']) - 1): ts = results[0]['times'][i] data[ts] = [None] * len(columns) for series in results: if series['id'] not in columns: continue for i in range(0, len(series['values'])): ts = series['times'][i] data[ts][columns.index(series['id'])] = series['values'][i] for key in data.items(): tmp = [ key[0], key[1][0], key[1][1], key[1][2], key[1][3], key[1][4], key[1][5] ] if key[1][0] == 0.0 and key[1][1] == 0.0 and key[1][2] == 0.0 and key[ 1][3] == 0.0 and key[1][4] == 0.0 and key[1][5] == 0.0: continue feed.append(tmp) return render_template('main/index.sm.html', feed=feed)
def main(): parser = argparse.ArgumentParser(description='') parser.add_argument('-s', help='Qumulo hostname', required=True) parser.add_argument('-u', help='Qumulo API user', default=os.getenv('QUSER') or 'admin') parser.add_argument('-p', help='Qumulo API password', default=os.getenv('QPASS') or 'admin') parser.add_argument('-d', help='Root Directory', default='/') try: args, other_args = parser.parse_known_args() except: print("-"*80) parser.print_help() print("-"*80) sys.exit(0) if args.d != '/': args.d = re.sub('/$', '', args.d) + '/' creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p} log_it("Log in to: %s" % (args.s)) rc = RestClient(creds["QHOST"], 8000) rc.login(creds["QUSER"], creds["QPASS"]) res = rc.snapshot.list_snapshot_statuses() existing_snap = None for snap in res['entries']: if snap['name'] == SNAP_NAME and snap['source_file_path'] == args.d: existing_snap = snap break # snap = rcs[len(rcs)-1].snapshot.create_snapshot(path=path, name=SNAP_NAME) snap_before = rc.snapshot.get_snapshot(746219) snap_after = rc.snapshot.get_snapshot(748466) if snap_before: process_snap_diff(creds, args.d, snap_before, snap_after ) else: w = QWalkWorker(creds, Search(['--re', '.', '--cols', 'dir_id,id,type,name,size,blocks,owner,change_time']), args.d, None, "qumulo-fs-index.txt", None)
def cluster_login(config_file) -> RestInfo: """ Log into cluster via Qumulo Rest API. """ api_hostname = config_file['cluster_settings']['cluster_address'] api_username = config_file['cluster_settings']['username'] api_password = config_file['cluster_settings']['password'] try: rest_client = RestClient(api_hostname, 8000) rest_client.login(api_username, api_password) return rest_client except TimeoutError as err: sys.exit(f'{err}\nExiting...') except RequestError as err: print('ERROR: Invalid credentials. Please check config file & try again.') sys.exit('Exiting...')
def get_file_attrs(x): credentials, file_ids = x client = RestClient(credentials["cluster"], credentials["port"]) client.login(credentials["user"], credentials["password"]) result = [] for file_id in file_ids: if file_id in seen: result += [seen[file_id]] continue attrs = client.fs.get_file_attr(file_id) str_owner = translate_owner_to_owner_string(client , attrs['owner'] , attrs['owner_details']['id_type'] , attrs['owner_details']['id_value']) seen[file_id] = str_owner result.append(str_owner) return result
def qumulo_upload(user, password, cluster_address, cluster_port, src, dest): """ Function to read file and upload it to the Qumulo file system """ # Connect to Qumulo REST API rc = RestClient(cluster_address, cluster_port) rc.login(user, password) # Split filename and directory dir_path, filename = os.path.split(dest) # Make sure the directory we are uploading to exists qumulo_mkdir_p(rc, dir_path) # Open local file in binary read-only mode with open(src, 'rb') as fh: # Initialize file rc.fs.create_file(name=filename, dir_path=dir_path) # Write data to file rc.fs.write_file(data_file=fh, path=dest)
def process_dirs(work_queue, i): print("Thread{0}: starting".format(i)) rc1 = RestClient(cluster1, 8000) rc1.login(api_user, api_password) rc2 = RestClient(cluster2, 8000) rc2.login(api_user, api_password) while not work_queue.empty(): directory = work_queue.get() if int(directory[-5:]) % 100 == 0: print("Thread{0:02d}: removing SMB share {1}".format(i, directory)) delete_smb_share(rc1, share_name="test_{0}".format(directory), fs_path="/keyva_demo/{0}".format(directory)) delete_smb_share(rc2, share_name="test_{0}".format(directory), fs_path="/keyva_demo/{0}".format(directory)) work_queue.task_done()
def qumulo_connect_api(qumulo_cluster_ips, qumulo_api_user, qumulo_api_password): ip = random.choice(qumulo_cluster_ips) rc = RestClient(ip, 8000) creds = rc.login(qumulo_api_user, qumulo_api_password) ses = requests.Session() headers = {"Authorization": "Bearer %s" % str(creds.bearer_token)} ses.headers.update(headers) return ip, ses
def process_snap_diff(creds, path, snap_before_id, snap_after_id): q = multiprocessing.Queue() q_len = multiprocessing.Value("i", 0) q_lock = multiprocessing.Lock() w_lock = multiprocessing.Lock() w_file = "output-qumulo-fs-index-%s-change-log-%s-%s.txt" % (re.sub( "[^a-z0-9]+", "_", path), snap_before_id, snap_after_id) fw = open(w_file, mode="w", encoding='utf-8') fw.close() rc = RestClient(creds["QHOST"], 8000) rc.login(creds["QUSER"], creds["QPASS"]) log_it("Run get_all_snapshot_tree_diff") results = rc.snapshot.get_all_snapshot_tree_diff(older_snap=snap_before_id, newer_snap=snap_after_id) log_it("Done get_all_snapshot_tree_diff.") log_it("Creating worker pool.") pool = multiprocessing.Pool(MAX_WORKER_COUNT, snap_worker, (creds, q, q_lock, q_len, w_lock, w_file)) log_it("Add items to queue.") ent_list = [] for res in results: for ent in res['entries']: ent["dir_id"] = None ent_list.append(ent) if len(ent_list) > 5: add_to_q( q, q_lock, q_len, { "list": ent_list, "snap_before_id": snap_before_id, "snap_after_id": snap_after_id }) ent_list = [] add_to_q( q, q_lock, q_len, { "list": ent_list, "snap_before_id": snap_before_id, "snap_after_id": snap_after_id }) log_it("Done adding items to queue.") while True: log_it("Queue length: %s" % q_len.value) time.sleep(WAIT_SECONDS) if q_len.value <= 0: break
def qumulo_connect_api(qumulo_cluster_ips, qumulo_api_user, qumulo_api_password): ip = random.choice(qumulo_cluster_ips) rc = RestClient(ip, 8000) creds = rc.login(qumulo_api_user, qumulo_api_password) ses = requests.Session() adapter = requests.adapters.HTTPAdapter(pool_maxsize=100) ses.mount('https://', adapter) headers = {"Authorization": "Bearer %s" % str(creds.bearer_token)} ses.headers.update(headers) return ip, ses
def get_qumulo_cluster_ips(qumulo_cluster, qumulo_api_user, qumulo_api_password): qumulo_cluster_ips = [] rc = RestClient(qumulo_cluster, 8000) creds = rc.login(qumulo_api_user, qumulo_api_password) for d in rc.cluster.list_nodes(): c = rc.network.get_network_status_v2(1, d['id']) if len(c['network_statuses'][0]['floating_addresses']) > 0: qumulo_cluster_ips.append(c['network_statuses'][0]['floating_addresses'][0]) else: qumulo_cluster_ips.append(c['network_statuses'][0]['address']) return qumulo_cluster_ips
def process_dirs(work_queue1, work_queue2, i): global replication_sync print("Thread{0}: starting".format(i)) rc1 = RestClient(cluster1, 8000) rc1.login(api_user, api_password) rc2 = RestClient(cluster2, 8000) rc2.login(api_user, api_password) while not work_queue1.empty(): try: directory = work_queue1.get(timeout=30) except queue.Empty: break # if int(directory[-5:]) % 100 == 0: # print("Thread{0:02d}: making dir {1}".format(i, directory)) create_dir(rc1, name=directory, dir_path='/keyva_demo') work_queue1.task_done() counter = 0 while replication_sync != "success": if counter == 600: raise RuntimeError("Timed out waiting on replication sync.") time.sleep(1) counter += 1 while not work_queue2.empty(): try: directory = work_queue2.get(timeout=30) except queue.Empty: break # if int(directory[-5:]) % 100 == 0: # print("Thread{0:02d}: making smb share test_{1}".format(i, directory)) create_smb_share(rc1, share_name="test_{0}".format(directory), fs_path="/keyva_demo/{0}".format(directory)) create_smb_share(rc2, share_name="test_{0}".format(directory), fs_path="/keyva_demo/{0}".format(directory)) create_test_file(rc1, filename="testfile", dir_path="/keyva_demo/{0}".format(directory)) work_queue2.task_done() print("Thread{0}: complete".format(i))
def walk_tree(QHOST, QUSER, QPASS, start_path, min_size): global gvars gvars = Gvars(QHOST, QUSER, QPASS) the_pool = multiprocessing.Pool(16, worker_main) rc = RestClient(gvars.QHOST, 8000) rc.login(gvars.QUSER, gvars.QPASS) root = rc.fs.read_dir_aggregates(path=start_path, max_depth=0) min_sz = int(root["total_capacity"]) / int(min_size) add_to_queue({"path": start_path, "min_sz": min_sz, "max_depth": 5}) time.sleep(0.1) wait_count = 0 while gvars.the_queue_len.value > 0: wait_count += 1 # every 5 seconds report status # if (wait_count % 50) == 0: # print("Queue: %s - Items looked at: %s" % (gvars.the_queue_len.value, gvars.item_count.value)) time.sleep(0.1) # print("Queue: %s - Items looked at: %s" % (gvars.the_queue_len.value, gvars.item_count.value)) r = process_results_for_treemap() the_pool.terminate() del gvars return r
"QFS_ACE_FLAG_OBJECT_INHERIT", "QFS_ACE_FLAG_CONTAINER_INHERIT" ] RO = [ "QFS_ACCESS_READ", "QFS_ACCESS_READ_EA", "QFS_ACCESS_READ_ATTR", "QFS_ACCESS_READ_ACL", "QFS_ACCESS_EXECUTE", "QFS_ACCESS_SYNCHRONIZE" ] from qumulo.rest_client import RestClient RC = RestClient(address=API['host'], port=API['port']) RC.login(username=API['user'], password=API['pass']) class TestQacls(unittest.TestCase): def test_uid_to_qid_root(self): qid_target = QID_UID_BASE uid = 0 self.assertEqual(qid_target, qacls.uid_to_qid(uid)) def test_uid_to_qid_20(self): uid = 20 qid_target = QID_UID_BASE + uid self.assertEqual(qid_target, qacls.uid_to_qid(uid)) def test_gid_to_qid_wheel(self): gid = 0
def get_rc(): """return a working instance of RestClient""" rc = RestClient(API_HOST, API_PORT) rc.login(API_USER, API_PASS) return rc
def get_rc(): rc = RestClient(API_HOST, API_PORT) rc.login(API_USER, API_PASS) return rc
class QSFSAuthorizer(DummyAuthorizer): """We need a wedge that attempts authentication with the Qumulo API and allows a user in based on their credentials """ def __init__(self): super(QSFSAuthorizer, self).__init__() self.rc = RestClient(API_HOST, API_PORT) def add_user(self, username, password, homedir, perm='elr', msg_login="******", msg_quit="Goodbye."): logger.debug("NOT IMPLEMENTED add_user()") super(QSFSAuthorizer, self).add_user( username, password, homedir, perm, msg_login, msg_quit ) def add_anonymous(self, homedir, **kwargs): logger.debug("NOT IMPLEMENTED add_anonymous()") super(QSFSAuthorizer, self).add_anonymous(homedir, **kwargs) def remove_user(self, username): logger.debug("NOT IMPLEMENTED remove_user()") super(QSFSAuthorizer, self).remove_user(username) def override_perm(self, username, directory, perm, recursive=False): logger.debug("NOT IMPLEMENTED override_perm()") super(QSFSAuthorizer, self).override_perm( username, directory, perm, recursive ) def validate_authentication(self, username, password, handler): """Attempt to login using RestClient, raise AuthenticationFailed if we don't login successfully """ logger.debug("validate_authentication(%s, %s, handler)" % (username, password)) # attempt login with restclient try: self.rc.login(username, password) except RequestError: raise AuthenticationFailed def get_home_dir(self, username): logger.debug("get_home_dir() will return '/' for all users") return u'/' def impersonate_user(self, username, password): """This should probably return a RestClient assuming it gets called after login """ logger.debug("impersonate_user() returning RestClient") local_rc = RestClient(API_HOST, API_PORT) local_rc.login(username, password) return local_rc def terminate_impersonation(self, username): """This should kill off the restclient created when impersonating the user """ logger.debug("terminate_impersonation() called, doing nothing") def has_user(self, username): logger.debug("has_user(%s)" % username) local_rc = RestClient(API_HOST, API_PORT) local_rc.login(API_USER, API_PASS) response = local_rc.users.list_users() name_list = [user['name'] for user in response] return username in name_list def has_perm(self, username, perm, path=None): logger.debug("has_perm() will always return True") return True def get_perms(self, username): logger.debug("NOT IMPLEMENTED get_perms()") super(QSFSAuthorizer, self).get_perms(username) def get_msg_login(self, username): logger.debug("get_msg_login() returns the same message for everyone") admin_rc = get_rc() response = admin_rc.config.cluster_config_get() cluster_name = response[u'bootstrap'][u'cluster_name'] version = self.rc.version.version()['revision_id'] return u"Welcome to qftpd on %s (%s)" % (cluster_name, version) def get_msg_quit(self, username): logger.debug("get_msg_quit() will return the same message for everyone") return u"Goodbye." def _check_permissions(self, username, perm): logger.debug("NOT IMPLEMENTED _check_permissions()") super(QSFSAuthorizer, self)._check_permissions(username, perm) def _issubpath(self, a, b): logger.debug("NOT IMPLEMENTED _issubpath()") super(QSFSAuthorizer, self)._issubpath(a, b)
import time import pprint from qumulo.rest_client import RestClient # NOTE change these credentials to matc your environment cluster = "product.eng.qumulo.com" username = "******" password = "******" api_cli = RestClient(cluster, 8000) api_cli.login(username=username, password=password) def show_data(title, data, kv=False): print "\n+++++++++++++++ " + title if not kv: for d in data: print d else: for k in data: print k + " -> " + str(data[k]) print "-------------------------------------------------" def show_data_custom(title): print "\n+++++++++++++++ " + title if title == "Read Dir Aggregates": res = api_cli.fs.read_dir_aggregates("/", max_entries=10, recursive=True) for e in res: pprint.pprint(e) break print "-------------------------------------------------" # Cluster total filesystem stats