def main():
parser = argparse.ArgumentParser(
description=
'Walk levels of a known Qumulo filesystem tree and gather capacity metrics'
)
parser.add_argument('-s', help='Qumulo hostname', required=True)
parser.add_argument('-u', help='Qumulo api user', required=True)
parser.add_argument('-p', help='Qumulo api passord')
parser.add_argument('-d', help='Starting directory path', required=True)
parser.add_argument('-o',
help='Output file name',
default='qumulo-data-by-directory.csv')
parser.add_argument('-l', nargs='+', help='Level names', required=True)
args, other_args = parser.parse_known_args()
if not args.p:
args.p = getpass()
creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
level_names = []
# initialize the REST client
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
get_capacity_aggregates(rc, args.d, args.l, args.o)
print("Created file: %s" % args.o)
def process_snap_diff(creds, path, snap_before, snap_after):
q = multiprocessing.Queue()
q_len = multiprocessing.Value("i", 0)
q_lock = multiprocessing.Lock()
pool = multiprocessing.Pool(MAX_WORKER_COUNT,
snap_worker,
(creds, q, q_lock, q_len))
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
results = rc.snapshot.get_all_snapshot_tree_diff(older_snap=snap_before['id'], newer_snap=snap_after['id'])
ent_list = []
for res in results:
for ent in res['entries']:
print(ent)
add_to_q(q, q_lock, q_len, {"type":"diff_item", "value": ent})
# ent_list.append(ent)
# if len(ent_list) > 1:
# add_to_q(q, q_lock, q_len, ent_list)
# ent_list = []
# add_to_q(q, q_lock, q_len, ent_list)
while True:
log_it("Queue length: %s" % q_len.value)
time.sleep(WAIT_SECONDS)
if q_len.value <= 0:
break
def __init__(self, hostname, user, password):
self.hostname = hostname
self.user = user
self.password = password
self.rc = RestClient(self.hostname, 8000)
self.rc.login(self.user, self.password)
self.date = date.today()
def has_user(self, username):
logger.debug("has_user(%s)" % username)
local_rc = RestClient(API_HOST, API_PORT)
local_rc.login(API_USER, API_PASS)
response = local_rc.users.list_users()
name_list = [user['name'] for user in response]
return username in name_list
def rc_get_ips(creds: Creds) -> Sequence[str]:
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
ips = []
for d in rc.network.list_network_status_v2(1):
ips.append(d["network_statuses"][0]["address"])
return ips
def walk_tree(QHOST, QUSER, QPASS, start_path):
global gvars
if start_path[-1] != "/":
start_path = start_path + "/"
log("Tree walk on Qumulo cluster %s starting at path %s" %
(QHOST, start_path))
gvars = Gvars(QHOST, QUSER, QPASS)
the_pool = multiprocessing.Pool(16, worker_main)
rc = RestClient(gvars.QHOST, 8000)
rc.login(gvars.QUSER, gvars.QPASS)
root = rc.fs.read_dir_aggregates(path=start_path, max_depth=0)
log("Directories to walk: %12s" %
"{:,}".format(int(root["total_directories"])))
log(" Files to walk: %12s" % "{:,}".format(int(root["total_files"])))
add_to_queue({"path": start_path, "max_depth": 5})
time.sleep(0.1) # wait a bit for the queue to get build up.
wait_count = 0
while gvars.the_queue_len.value > 0:
wait_count += 1
if (wait_count % 50) == 0: # show status every ~5 seconds
log("Processed %s entries. Queue length: %s" %
(gvars.done_queue_len.value, gvars.the_queue_len.value))
time.sleep(0.1)
the_pool.terminate()
log("Processed %s entries." % gvars.done_queue_len.value)
log("Done with tree walk. Combining results")
fw = open("file-list.txt", "w")
for f in glob.glob('out-*.txt'):
fr = open(f, "r")
fw.write(fr.read())
fr.close()
os.remove(f)
del gvars
log("Results save to file: file-list.txt")
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('-s', help='Qumulo hostname', required=True)
parser.add_argument('-u',
help='Qumulo API user',
default=os.getenv('QUSER') or 'admin')
parser.add_argument('-p',
help='Qumulo API password',
default=os.getenv('QPASS') or 'admin')
parser.add_argument('-d', help='Root Directory', default='/')
try:
args, other_args = parser.parse_known_args()
except:
print("-" * 80)
parser.print_help()
print("-" * 80)
sys.exit(0)
if args.d != '/':
args.d = re.sub('/$', '', args.d) + '/'
creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
log_it("Log in to: %s" % (args.s))
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
res = rc.snapshot.list_snapshot_statuses()
snap_before = None
snap_after = None
for snap in sorted(res['entries'], key=lambda d: int(d['id'])):
if snap['name'] == SNAP_NAME and snap['source_file_path'] == args.d:
if snap_before is None:
snap_before = snap
elif snap_after is None:
snap_after = snap
if snap_before and not snap_after:
snap_after = rc.snapshot.create_snapshot(path=args.d, name=SNAP_NAME)
log_it("Created new snapshot %s on %s" % (snap_after['id'], args.d))
if snap_before and snap_after:
log_it("Diffing snaps on %s between %s and %s" %
(args.d, snap_before['timestamp'][0:16],
snap_after['timestamp'][0:16]))
process_snap_diff(creds, args.d, snap_before['id'], snap_after['id'])
rc.snapshot.delete_snapshot(snapshot_id=snap_before['id'])
else:
# initial tree walk
snap_before = rc.snapshot.create_snapshot(path=args.d, name=SNAP_NAME)
log_it("Initial tree walk for: %s+snap:%s" %
(args.d, snap_before['id']))
log_file = "output-qumulo-fs-index-%s-tree.txt" % (re.sub(
"[^a-z0-9]+", "_", args.d))
w = QWalkWorker(
creds,
Search([
'--re', '.', '--cols',
'dir_id,id,type,path,name,size,blocks,owner,change_time,link_target,NEW'
]), args.d, str(snap_before['id']), None, log_file, None)
w.run()
def impersonate_user(self, username, password):
"""This should probably return a RestClient assuming it gets called
after login
"""
logger.debug("impersonate_user() returning RestClient")
local_rc = RestClient(API_HOST, API_PORT)
local_rc.login(username, password)
return local_rc
def worker_main():
global gvars
rc = RestClient(gvars.QHOST, 8000)
rc.login(gvars.QUSER, gvars.QPASS)
while True:
item = gvars.the_queue.get(True)
list_dir(rc, item)
with gvars.the_queue_len.get_lock():
gvars.the_queue_len.value -= 1
def read_time_series_from_cluster(host: str, user: str, password: str,
port: int) -> List[Mapping[str, Any]]:
"""
Communicates with the cluster to grab the analytics in time series format
"""
rest_client = RestClient(host, port)
rest_client.login(user, password)
return rest_client.analytics.time_series_get(
begin_time=calculate_begin_time(CSV_FILENAME))
def login(configdict):
'''Obtain credentials from the REST server'''
try:
rc = RestClient(configdict['host'], configdict['port'])
rc.login(configdict['user'], configdict['password'])
except Exception, excpt:
print "Error connecting to the REST server: %s" % excpt
print __doc__
sys.exit(1)
def api_perf_counters():
rc = get_rc()
pc_data = {}
for n in rc.network.list_network_status_v2(1):
rc = RestClient(n['network_statuses'][0]['address'], 8000)
rc.login('admin', app.config['API_PASSWORD'])
resp = rc.request("GET", "/v1/metrics/")
pc_data[n['node_id']] = resp
return json.dumps(pc_data)
def qumulo_connect_api(qumulo_cluster_ips, qumulo_api_user,
qumulo_api_password):
ip = random.choice(qumulo_cluster_ips)
rc = RestClient(ip, 8000)
creds = rc.login(qumulo_api_user, qumulo_api_password)
ses = requests.Session()
headers = {"Authorization": "Bearer %s" % str(creds.bearer_token)}
ses.headers.update(headers)
return ip, ses
def qumulo_connect_api(qumulo_cluster_ips, qumulo_api_user, qumulo_api_password):
ip = random.choice(qumulo_cluster_ips)
rc = RestClient(ip, 8000)
creds = rc.login(qumulo_api_user, qumulo_api_password)
ses = requests.Session()
adapter = requests.adapters.HTTPAdapter(pool_maxsize=100)
ses.mount('https://', adapter)
headers = {"Authorization": "Bearer %s" % str(creds.bearer_token)}
ses.headers.update(headers)
return ip, ses
def get_qumulo_cluster_ips(qumulo_cluster, qumulo_api_user, qumulo_api_password):
qumulo_cluster_ips = []
rc = RestClient(qumulo_cluster, 8000)
creds = rc.login(qumulo_api_user, qumulo_api_password)
for d in rc.cluster.list_nodes():
c = rc.network.get_network_status_v2(1, d['id'])
if len(c['network_statuses'][0]['floating_addresses']) > 0:
qumulo_cluster_ips.append(c['network_statuses'][0]['floating_addresses'][0])
else:
qumulo_cluster_ips.append(c['network_statuses'][0]['address'])
return qumulo_cluster_ips
def worker_main():
global gvars
proc = multiprocessing.current_process()
rc = RestClient(gvars.QHOST, 8000)
rc.login(gvars.QUSER, gvars.QPASS)
out_file = open("out-%s.txt" % proc.pid, "w")
while True:
item = gvars.the_queue.get(True)
list_dir(rc, item, out_file)
out_file.flush()
with gvars.the_queue_len.get_lock():
gvars.the_queue_len.value -= 1
def main():
parser = argparse.ArgumentParser(description='Example command usage usage:\npython timeseries-to-csv.py --host product --user admin --pass admin'
, epilog='.'
, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("--host", required=True, help="Required: Specify host (Qumulo cluster)")
parser.add_argument("--user", required=True, help="Specify api user")
parser.add_argument("--pass", required=True, dest="passwd", help="Specify api password ")
args = parser.parse_args()
rc = RestClient(args.host, 8000)
rc.login(args.user, args.passwd);
last_line = None
columns = ["iops.read.rate", "iops.write.rate",
"throughput.read.rate", "throughput.write.rate",
"reclaim.deferred.rate", "reclaim.snapshot.rate"]
csv_file_name = "qumulo-timeseries-data.csv"
if os.path.exists(csv_file_name):
# read to the last line in the file
with open(csv_file_name, "rb") as csvfile:
reader = csv.reader(csvfile)
for row in reader:
last_line = row
# at most we'll have 1 day of data
begin_time = int(time.time()) - 60 * 60 * 24
# otherwise, pull only the latest data, if we already have a file
if last_line is not None:
begin_time = int(last_line[0]) + 5
results = rc.analytics.time_series_get(begin_time = begin_time)
print("Appending %s results to '%s'." % (len(results[0]['values']), csv_file_name))
data = {}
for i in range(0,len(results[0]['times'])-1):
ts = results[0]['times'][i]
data[ts] = [None] * len(columns)
for series in results:
if series['id'] not in columns:
continue
for i in range(0,len(series['values'])):
ts = series['times'][i]
data[ts][columns.index(series['id'])] = series['values'][i]
fw = open(csv_file_name, "a")
if last_line is None:
fw.write("unix.timestamp,gmtime," + ",".join(columns) + "\r\n")
for ts in sorted(data):
gmt = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(ts))
fw.write("%s,%s," % (ts, gmt) + ",".join([str(d) for d in data[ts]]) + "\r\n")
fw.close()
def ids_to_attrs(cluster, id_attr_list):
inode_types = {}
qumulo_client = RestClient(cluster['host'], 8000)
qumulo_client.login(cluster['user'], cluster['password'])
for inode_id in id_attr_list:
try:
attrs = qumulo_client.fs.get_attr(id_=inode_id)
if inode_id not in inode_types:
inode_types[inode_id] = attrs["type"]
except:
pass
return inode_types
def get_file_attrs(x):
credentials, paths = x
client = RestClient(credentials["cluster"], credentials["port"])
client.login(credentials["user"], credentials["password"])
result = []
for path in paths:
if seen.has_key(path):
result += [seen[path]]
continue
str_owner = enumerate_owner(client, path)
seen[path] = str_owner
result.append(str_owner)
return result
def do_it(opts, args):
credentials = {
"user": opts.user,
"password": opts.password,
"cluster": opts.cluster
}
# Qumulo API login
client = RestClient(opts.cluster, opts.port)
client.login(opts.user, opts.password)
total_capacity_used = int(
client.fs.read_dir_aggregates(args[0])['total_capacity'])
pool = Pool(opts.concurrency)
# First build a vector of all samples...
samples = get_samples(pool, args[0], credentials, opts)
# Then get a corresponding vector of owner strings
owner_vec = get_owner_vec(pool, credentials, samples, opts)
owners = {}
directories = {}
# Create a mapping of user to tree...
for s, owner in zip(samples, owner_vec):
owners.setdefault(owner, SampleTreeNode(""))
owners[owner].insert(s["name"], 1)
def format_capacity(samples):
bytes_per_terabyte = 1000.**4
if opts.dollars_per_terabyte != None:
return "$%0.02f/month" % (samples * total_capacity_used /
opts.samples / bytes_per_terabyte *
opts.dollars_per_terabyte)
else:
return pretty_print_capacity(samples * total_capacity_used /
opts.samples)
print "Total: %s" % (format_capacity(opts.samples))
sorted_owners = sorted(
owners.items(), lambda x, y: cmp(y[1].sum_samples, x[1].sum_samples))
# For each owner, print total used, then refine the tree and dump it.
for name, tree in sorted_owners:
print "Owner %s (~%0.1f%%/%s)" % (name, tree.sum_samples / float(
opts.samples) * 100, format_capacity(tree.sum_samples))
tree.prune_until(max_leaves=opts.max_leaves,
min_samples=opts.min_samples)
print tree.__str__(" ", lambda x: format_capacity(x))
def __init__(self, cluster, conf):
self.DIRECTORY_DEPTH_LIMIT = conf['DIRECTORY_DEPTH_LIMIT']
self.IOPS_THRESHOLD = conf['IOPS_THRESHOLD']
self.THROUGHPUT_THRESHOLD = conf['THROUGHPUT_THRESHOLD']
self.DIRECTORIES_ONLY = conf['DIRECTORIES_ONLY']
# initial fields in the data dict
# "qumulo_host"
# "client_ip"
# "client_host_name"
# "path"
# "path_levels"
# "timestamp"
# default dict for the metrics
self.EMPTY_DATA = OrderedDict([
('file-throughput-write', 0.0),
('file-throughput-read', 0.0),
('file-iops-write', 0.0),
('file-iops-read', 0.0),
('metadata-iops-write', 0.0),
('metadata-iops-read', 0.0),
('iops-total', 0.0),
('throughput-total', 0.0),
])
self.ids_to_paths = {}
self.ips_to_hostnames = {}
self.combined_data = {}
self.new_db_entries = []
self.pool = Pool(6)
self.cluster = cluster
log("Connect to Qumulo API for cluster: %s" % self.cluster['host'])
self.qumulo_client = RestClient(self.cluster['host'], 8000)
self.qumulo_client.login(self.cluster['user'],
self.cluster['password'])
log("Get current activity data from Qumulo API for %s" %
self.cluster['host'])
activity_data = self.qumulo_client.analytics.current_activity_get()
self.current_timestamp = datetime.datetime.utcnow()
self.current_epoch = int(time.time())
# Filter out client IPs based on regex in per-cluster config
self.entries = list()
client_ip_regex = self.cluster.get('client_ip_regex', '.*')
for entry in activity_data['entries']:
if re.match(client_ip_regex, entry['ip']):
self.entries.append(entry)
log("Successfully recieved %s activity entries." % len(self.entries))
def get_file_attrs(x):
credentials, paths = x
client = RestClient(credentials["cluster"], 8000)
client.login(credentials["user"], credentials["password"])
result = []
for path in paths:
if seen.has_key(path):
result += [seen[path]]
continue
owner_id = client.fs.get_attr(path)["owner"]
str_owner = translate_owner_to_owner_string(client, owner_id)
seen[path] = str_owner
result.append(str_owner)
return result
def __init__(self, cluster):
# default dictionary for activity data
self.EMPTY_DATA = OrderedDict([
('file-throughput-write', 0.0),
('file-throughput-read', 0.0),
('file-iops-write', 0.0),
('file-iops-read', 0.0),
('metadata-iops-write', 0.0),
('metadata-iops-read', 0.0),
('iops-total', 0.0),
('throughput-total', 0.0),
])
PG_SETUP_SQL = """CREATE TABLE qumulo_activity(
qumulo_host VARCHAR(128),
client_ip VARCHAR(20),
client_hostname VARCHAR(256),
path VARCHAR(2048),
ts TIMESTAMP,
file_throughput_write FLOAT,
file_throughput_read FLOAT,
file_iops_write FLOAT,
file_iops_read FLOAT,
metadata_iops_write FLOAT,
metadata_iops_read FLOAT,
iops_total FLOAT,
throughput_total FLOAT
);"""
self.ids_to_paths = {}
self.ips_to_hostnames = {}
self.combined_data = {}
self.new_db_entries = []
self.pool = Pool(6)
self.cluster = cluster
log("Connect to Qumulo API for cluster: %s" % self.cluster['host'])
self.qumulo_client = RestClient(self.cluster['host'], 8000)
self.qumulo_client.login(self.cluster['user'],
self.cluster['password'])
log("Get current activity data from Qumulo API for %s" %
self.cluster['host'])
activity_data = self.qumulo_client.analytics.current_activity_get()
self.current_timestamp = datetime.datetime.utcnow()
self.current_epoch = int(time.time())
self.entries = activity_data['entries']
log("Successfully recieved %s activity entries." % len(self.entries))
def timestamps():
'''
Get Qumulo Timestamps
'''
# Get user informaation
creds = Creds.objects.first()
user = creds.user
password = creds.password
ipaddress = creds.ipaddress
columns = [
"iops.read.rate", "iops.write.rate", "throughput.read.rate",
"throughput.write.rate", "reclaim.deferred.rate",
"reclaim.snapshot.rate"
]
#
feed = []
rc = RestClient(ipaddress, 8000)
rc.login(user, password)
#
begin_time = int(time.time()) - 60 * 60 * 24
results = rc.analytics.time_series_get(begin_time=begin_time)
data = {}
#
for i in range(0, len(results[0]['times']) - 1):
ts = results[0]['times'][i]
data[ts] = [None] * len(columns)
for series in results:
if series['id'] not in columns:
continue
for i in range(0, len(series['values'])):
ts = series['times'][i]
data[ts][columns.index(series['id'])] = series['values'][i]
for key in data.items():
tmp = [
key[0], key[1][0], key[1][1], key[1][2], key[1][3], key[1][4],
key[1][5]
]
if key[1][0] == 0.0 and key[1][1] == 0.0 and key[1][2] == 0.0 and key[
1][3] == 0.0 and key[1][4] == 0.0 and key[1][5] == 0.0:
continue
feed.append(tmp)
return render_template('main/index.sm.html', feed=feed)
def __init__(self, cluster, username, password, data_dir):
self.username = username
self.password = password
self.api_cli = RestClient(cluster, 8000)
self.qumulo_api_call(self.api_cli.login, username=username, password=password)
self.data_dir = data_dir
self.datestamp = self.timestamp[:10]
def qumulo_upload(user, password, cluster_address, cluster_port, src, dest):
"""
Function to read file and upload it to the Qumulo file system
"""
# Connect to Qumulo REST API
rc = RestClient(cluster_address, cluster_port)
rc.login(user, password)
# Split filename and directory
dir_path, filename = os.path.split(dest)
# Make sure the directory we are uploading to exists
qumulo_mkdir_p(rc, dir_path)
# Open local file in binary read-only mode
with open(src, 'rb') as fh:
# Initialize file
rc.fs.create_file(name=filename, dir_path=dir_path)
# Write data to file
rc.fs.write_file(data_file=fh, path=dest)
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('-s', help='Qumulo hostname', required=True)
parser.add_argument('-u', help='Qumulo API user',
default=os.getenv('QUSER') or 'admin')
parser.add_argument('-p', help='Qumulo API password',
default=os.getenv('QPASS') or 'admin')
parser.add_argument('-d', help='Root Directory', default='/')
try:
args, other_args = parser.parse_known_args()
except:
print("-"*80)
parser.print_help()
print("-"*80)
sys.exit(0)
if args.d != '/':
args.d = re.sub('/$', '', args.d) + '/'
creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
log_it("Log in to: %s" % (args.s))
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
res = rc.snapshot.list_snapshot_statuses()
existing_snap = None
for snap in res['entries']:
if snap['name'] == SNAP_NAME and snap['source_file_path'] == args.d:
existing_snap = snap
break
# snap = rcs[len(rcs)-1].snapshot.create_snapshot(path=path, name=SNAP_NAME)
snap_before = rc.snapshot.get_snapshot(746219)
snap_after = rc.snapshot.get_snapshot(748466)
if snap_before:
process_snap_diff(creds,
args.d,
snap_before,
snap_after
)
else:
w = QWalkWorker(creds,
Search(['--re', '.',
'--cols', 'dir_id,id,type,name,size,blocks,owner,change_time']),
args.d,
None, "qumulo-fs-index.txt", None)
def get_file_attrs(x):
credentials, file_ids = x
client = RestClient(credentials["cluster"], credentials["port"])
client.login(credentials["user"], credentials["password"])
result = []
for file_id in file_ids:
if file_id in seen:
result += [seen[file_id]]
continue
attrs = client.fs.get_file_attr(file_id)
str_owner = translate_owner_to_owner_string(client
, attrs['owner']
, attrs['owner_details']['id_type']
, attrs['owner_details']['id_value'])
seen[file_id] = str_owner
result.append(str_owner)
return result
def cluster_login(config_file) -> RestInfo:
"""
Log into cluster via Qumulo Rest API.
"""
api_hostname = config_file['cluster_settings']['cluster_address']
api_username = config_file['cluster_settings']['username']
api_password = config_file['cluster_settings']['password']
try:
rest_client = RestClient(api_hostname, 8000)
rest_client.login(api_username, api_password)
return rest_client
except TimeoutError as err:
sys.exit(f'{err}\nExiting...')
except RequestError as err:
print('ERROR: Invalid credentials. Please check config file & try again.')
sys.exit('Exiting...')
def process_snap_diff(creds, path, snap_before_id, snap_after_id):
q = multiprocessing.Queue()
q_len = multiprocessing.Value("i", 0)
q_lock = multiprocessing.Lock()
w_lock = multiprocessing.Lock()
w_file = "output-qumulo-fs-index-%s-change-log-%s-%s.txt" % (re.sub(
"[^a-z0-9]+", "_", path), snap_before_id, snap_after_id)
fw = open(w_file, mode="w", encoding='utf-8')
fw.close()
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
log_it("Run get_all_snapshot_tree_diff")
results = rc.snapshot.get_all_snapshot_tree_diff(older_snap=snap_before_id,
newer_snap=snap_after_id)
log_it("Done get_all_snapshot_tree_diff.")
log_it("Creating worker pool.")
pool = multiprocessing.Pool(MAX_WORKER_COUNT, snap_worker,
(creds, q, q_lock, q_len, w_lock, w_file))
log_it("Add items to queue.")
ent_list = []
for res in results:
for ent in res['entries']:
ent["dir_id"] = None
ent_list.append(ent)
if len(ent_list) > 5:
add_to_q(
q, q_lock, q_len, {
"list": ent_list,
"snap_before_id": snap_before_id,
"snap_after_id": snap_after_id
})
ent_list = []
add_to_q(
q, q_lock, q_len, {
"list": ent_list,
"snap_before_id": snap_before_id,
"snap_after_id": snap_after_id
})
log_it("Done adding items to queue.")
while True:
log_it("Queue length: %s" % q_len.value)
time.sleep(WAIT_SECONDS)
if q_len.value <= 0:
break
def get_many_clients(self, creds, client_count):
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
ips = []
rcs = []
for d in rc.network.list_network_status_v2(1):
ips.append(d['network_statuses'][0]['address'])
for i in range(0, client_count):
rcc = RestClient(ips[i % len(ips)], 8000)
rcc.login(creds["QUSER"], creds["QPASS"])
rcs.append(rcc)
return rcs
def main():
# Initial setup
rc1 = RestClient(cluster1, 8000)
rc1.login(api_user, api_password)
rc2 = RestClient(cluster2, 8000)
rc2.login(api_user, api_password)
relationship_id = get_relationship_id(rc2, "/keyva_demo/")
rc2.replication.make_target_writable(relationship_id=relationship_id)
rc1.replication.modify_source_relationship(relationship_id=relationship_id, source_root_read_only=True)
time.sleep(15)
rc2.replication.reverse_target_relationship(relationship_id=relationship_id, source_address=cluster1)
rc1.replication.reconnect_target_relationship(relationship_id=relationship_id)
import time
import pprint
from qumulo.rest_client import RestClient
# NOTE change these credentials to matc your environment
cluster = "product.eng.qumulo.com"
username = "******"
password = "******"
api_cli = RestClient(cluster, 8000)
api_cli.login(username=username, password=password)
def show_data(title, data, kv=False):
print "\n+++++++++++++++ " + title
if not kv:
for d in data:
print d
else:
for k in data:
print k + " -> " + str(data[k])
print "-------------------------------------------------"
def show_data_custom(title):
print "\n+++++++++++++++ " + title
if title == "Read Dir Aggregates":
res = api_cli.fs.read_dir_aggregates("/", max_entries=10, recursive=True)
for e in res:
pprint.pprint(e)
break
print "-------------------------------------------------"
# Cluster total filesystem stats
def get_rc():
"""return a working instance of RestClient"""
rc = RestClient(API_HOST, API_PORT)
rc.login(API_USER, API_PASS)
return rc
INHERIT_ALL = [
"QFS_ACE_FLAG_OBJECT_INHERIT",
"QFS_ACE_FLAG_CONTAINER_INHERIT"
]
RO = [
"QFS_ACCESS_READ",
"QFS_ACCESS_READ_EA",
"QFS_ACCESS_READ_ATTR",
"QFS_ACCESS_READ_ACL",
"QFS_ACCESS_EXECUTE",
"QFS_ACCESS_SYNCHRONIZE"
]
from qumulo.rest_client import RestClient
RC = RestClient(address=API['host'], port=API['port'])
RC.login(username=API['user'], password=API['pass'])
class TestQacls(unittest.TestCase):
def test_uid_to_qid_root(self):
qid_target = QID_UID_BASE
uid = 0
self.assertEqual(qid_target, qacls.uid_to_qid(uid))
def test_uid_to_qid_20(self):
uid = 20
qid_target = QID_UID_BASE + uid
self.assertEqual(qid_target, qacls.uid_to_qid(uid))
def test_gid_to_qid_wheel(self):
def get_rc():
rc = RestClient(API_HOST, API_PORT)
rc.login(API_USER, API_PASS)
return rc
class QSFSAuthorizer(DummyAuthorizer):
"""We need a wedge that attempts authentication with the Qumulo API and
allows a user in based on their credentials
"""
def __init__(self):
super(QSFSAuthorizer, self).__init__()
self.rc = RestClient(API_HOST, API_PORT)
def add_user(self, username, password, homedir, perm='elr',
msg_login="******", msg_quit="Goodbye."):
logger.debug("NOT IMPLEMENTED add_user()")
super(QSFSAuthorizer, self).add_user(
username, password, homedir, perm, msg_login, msg_quit
)
def add_anonymous(self, homedir, **kwargs):
logger.debug("NOT IMPLEMENTED add_anonymous()")
super(QSFSAuthorizer, self).add_anonymous(homedir, **kwargs)
def remove_user(self, username):
logger.debug("NOT IMPLEMENTED remove_user()")
super(QSFSAuthorizer, self).remove_user(username)
def override_perm(self, username, directory, perm, recursive=False):
logger.debug("NOT IMPLEMENTED override_perm()")
super(QSFSAuthorizer, self).override_perm(
username, directory, perm, recursive
)
def validate_authentication(self, username, password, handler):
"""Attempt to login using RestClient, raise AuthenticationFailed if we
don't login successfully
"""
logger.debug("validate_authentication(%s, %s, handler)" %
(username, password))
# attempt login with restclient
try:
self.rc.login(username, password)
except RequestError:
raise AuthenticationFailed
def get_home_dir(self, username):
logger.debug("get_home_dir() will return '/' for all users")
return u'/'
def impersonate_user(self, username, password):
"""This should probably return a RestClient assuming it gets called
after login
"""
logger.debug("impersonate_user() returning RestClient")
local_rc = RestClient(API_HOST, API_PORT)
local_rc.login(username, password)
return local_rc
def terminate_impersonation(self, username):
"""This should kill off the restclient created when impersonating the
user
"""
logger.debug("terminate_impersonation() called, doing nothing")
def has_user(self, username):
logger.debug("has_user(%s)" % username)
local_rc = RestClient(API_HOST, API_PORT)
local_rc.login(API_USER, API_PASS)
response = local_rc.users.list_users()
name_list = [user['name'] for user in response]
return username in name_list
def has_perm(self, username, perm, path=None):
logger.debug("has_perm() will always return True")
return True
def get_perms(self, username):
logger.debug("NOT IMPLEMENTED get_perms()")
super(QSFSAuthorizer, self).get_perms(username)
def get_msg_login(self, username):
logger.debug("get_msg_login() returns the same message for everyone")
admin_rc = get_rc()
response = admin_rc.config.cluster_config_get()
cluster_name = response[u'bootstrap'][u'cluster_name']
version = self.rc.version.version()['revision_id']
return u"Welcome to qftpd on %s (%s)" % (cluster_name, version)
def get_msg_quit(self, username):
logger.debug("get_msg_quit() will return the same message for everyone")
return u"Goodbye."
def _check_permissions(self, username, perm):
logger.debug("NOT IMPLEMENTED _check_permissions()")
super(QSFSAuthorizer, self)._check_permissions(username, perm)
def _issubpath(self, a, b):
logger.debug("NOT IMPLEMENTED _issubpath()")
super(QSFSAuthorizer, self)._issubpath(a, b)
def __init__(self):
super(QSFSAuthorizer, self).__init__()
self.rc = RestClient(API_HOST, API_PORT)
