def main():
parser = argparse.ArgumentParser(
description='Walk Qumulo filesystem and do that thing.')
parser.add_argument('-s', help='Qumulo hostname', required=True)
parser.add_argument('-u',
help='Qumulo API user',
default=os.getenv('QUSER') or 'admin')
parser.add_argument('-p',
help='Qumulo API password',
default=os.getenv('QPASS') or 'admin')
parser.add_argument('-d', help='Starting directory', required=True)
parser.add_argument('-g',
help='Run with filesystem changes',
action='store_true')
parser.add_argument('-l', help='Log file', default='output-walk-log.txt')
parser.add_argument('-c', help='Class to run. Options include: ' + \
'ChangeExtension, SummarizeOwners, DataReductionTest, ' + \
'ModeBitsChecker'
, required=True)
try:
# Will fail with missing args, but unknown args will all fall through.
args, other_args = parser.parse_known_args()
except:
print("-" * 80)
parser.print_help()
print("-" * 80)
sys.exit(0)
QWalkWorker.run_all(args, other_args)
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('-s', help='Qumulo hostname', required=True)
parser.add_argument('-u',
help='Qumulo API user',
default=os.getenv('QUSER') or 'admin')
parser.add_argument('-p',
help='Qumulo API password',
default=os.getenv('QPASS') or 'admin')
parser.add_argument('-d', help='Root Directory', default='/')
try:
args, other_args = parser.parse_known_args()
except:
print("-" * 80)
parser.print_help()
print("-" * 80)
sys.exit(0)
if args.d != '/':
args.d = re.sub('/$', '', args.d) + '/'
creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
log_it("Log in to: %s" % (args.s))
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
res = rc.snapshot.list_snapshot_statuses()
snap_before = None
snap_after = None
for snap in sorted(res['entries'], key=lambda d: int(d['id'])):
if snap['name'] == SNAP_NAME and snap['source_file_path'] == args.d:
if snap_before is None:
snap_before = snap
elif snap_after is None:
snap_after = snap
if snap_before and not snap_after:
snap_after = rc.snapshot.create_snapshot(path=args.d, name=SNAP_NAME)
log_it("Created new snapshot %s on %s" % (snap_after['id'], args.d))
if snap_before and snap_after:
log_it("Diffing snaps on %s between %s and %s" %
(args.d, snap_before['timestamp'][0:16],
snap_after['timestamp'][0:16]))
process_snap_diff(creds, args.d, snap_before['id'], snap_after['id'])
rc.snapshot.delete_snapshot(snapshot_id=snap_before['id'])
else:
# initial tree walk
snap_before = rc.snapshot.create_snapshot(path=args.d, name=SNAP_NAME)
log_it("Initial tree walk for: %s+snap:%s" %
(args.d, snap_before['id']))
log_file = "output-qumulo-fs-index-%s-tree.txt" % (re.sub(
"[^a-z0-9]+", "_", args.d))
w = QWalkWorker(
creds,
Search([
'--re', '.', '--cols',
'dir_id,id,type,path,name,size,blocks,owner,change_time,link_target,NEW'
]), args.d, str(snap_before['id']), None, log_file, None)
w.run()
def test_search(creds, args, search):
log_it("Search: %s" % search)
w = QWalkWorker(creds, Search(search), args.d, None, LOG_FILE_NAME, None)
w.run()
if os.path.exists(LOG_FILE_NAME):
content = re.sub(r'[\r\n]+', ' ', open(LOG_FILE_NAME).read())
log_it("FOUND! : Search found - %s" % content)
os.remove(LOG_FILE_NAME)
else:
log_it("NOTFOUND: Search failure: %s" % search)
def test_search(
creds: Creds, start_path: str, search: Sequence[str], snapshot: Optional[str] = None
) -> None:
log_it("Search: %s" % search)
w = QWalkWorker(
creds, Search(search), start_path, snapshot, False, LOG_FILE_NAME, None
)
w.run()
if os.path.exists(LOG_FILE_NAME):
content = re.sub(r"[\r\n]+", " ", open(LOG_FILE_NAME).read())
log_it("FOUND! : Search found - %s" % content)
os.remove(LOG_FILE_NAME)
else:
log_it("NOTFOUND: Search failure: %s" % search)
def main() -> None:
parser = argparse.ArgumentParser(
description="Walk Qumulo filesystem and do that thing."
)
parser.add_argument("-s", help="Qumulo hostname", required=True)
parser.add_argument(
"-u", help="Qumulo API user", default=os.getenv("QUSER") or "admin"
)
parser.add_argument(
"-p", help="Qumulo API password", default=os.getenv("QPASS") or "admin"
)
parser.add_argument("-d", help="Starting directory", required=True)
parser.add_argument("-g", help="Run with filesystem changes", action="store_true")
parser.add_argument("-l", help="Log file", default="output-walk-log.txt")
parser.add_argument(
"-c",
help="Class to run.",
choices=list(QTASKS.keys()),
required=True,
)
parser.add_argument("--snap", help="Snapshot id")
try:
# Will fail with missing args, but unknown args will all fall through.
args, other_args = parser.parse_known_args()
except:
print("-" * 80)
parser.print_help()
print("-" * 80)
sys.exit(0)
QWalkWorker.run_all(
args.s,
args.u,
args.p,
args.d,
args.g,
args.l,
args.c,
args.snap,
other_args,
)
def main():
parser = argparse.ArgumentParser(description='')
parser.add_argument('-s', help='Qumulo hostname', required=True)
parser.add_argument('-u', help='Qumulo API user',
default=os.getenv('QUSER') or 'admin')
parser.add_argument('-p', help='Qumulo API password',
default=os.getenv('QPASS') or 'admin')
parser.add_argument('-d', help='Root Directory', default='/')
try:
args, other_args = parser.parse_known_args()
except:
print("-"*80)
parser.print_help()
print("-"*80)
sys.exit(0)
if args.d != '/':
args.d = re.sub('/$', '', args.d) + '/'
creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
log_it("Log in to: %s" % (args.s))
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
res = rc.snapshot.list_snapshot_statuses()
existing_snap = None
for snap in res['entries']:
if snap['name'] == SNAP_NAME and snap['source_file_path'] == args.d:
existing_snap = snap
break
# snap = rcs[len(rcs)-1].snapshot.create_snapshot(path=path, name=SNAP_NAME)
snap_before = rc.snapshot.get_snapshot(746219)
snap_after = rc.snapshot.get_snapshot(748466)
if snap_before:
process_snap_diff(creds,
args.d,
snap_before,
snap_after
)
else:
w = QWalkWorker(creds,
Search(['--re', '.',
'--cols', 'dir_id,id,type,name,size,blocks,owner,change_time']),
args.d,
None, "qumulo-fs-index.txt", None)
def main():
parser = argparse.ArgumentParser(description='Test the qwalk.py script')
parser.add_argument('-s', help='Qumulo hostname', required=True)
parser.add_argument('-u',
help='Qumulo API user',
default=os.getenv('QUSER') or 'admin')
parser.add_argument('-p',
help='Qumulo API password',
default=os.getenv('QPASS') or 'admin')
parser.add_argument('-d', help='Test Directory', default='/')
try:
args, other_args = parser.parse_known_args()
except:
print("-" * 80)
parser.print_help()
print("-" * 80)
sys.exit(0)
# Everything will happen in a new subdirectory.
test_dir_name = 'test-qwalk'
creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
log_it("Log in to: %s" % (args.s))
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
parent_dir = '/'
if args.d != '/':
parent_dir = re.sub(r'/$', '', args.d)
log_it("Create directory: %s/%s" %
(parent_dir if parent_dir != '/' else '', test_dir_name))
test_dir = rc.fs.create_directory(dir_path=parent_dir, name=test_dir_name)
args.d = "%s/%s" % (parent_dir, 'test-qwalk')
flowers_dir = rc.fs.create_directory(dir_path=args.d, name='flowers')
foods_dir = rc.fs.create_directory(dir_path=args.d, name='foods')
log_it("Create files")
f = {}
f["cat"] = rc.fs.create_file(dir_id=test_dir['id'], name='cat.txt')
f["mouse"] = rc.fs.create_file(dir_id=test_dir['id'], name='mouse.jpg')
f["dog"] = rc.fs.create_file(dir_id=test_dir['id'], name='dog.jpeg')
f["bear"] = rc.fs.create_file(dir_id=test_dir['id'], name='bear.mov')
f["rose"] = rc.fs.create_file(dir_id=flowers_dir['id'], name='rose.jpg')
f["violet"] = rc.fs.create_file(dir_id=flowers_dir['id'],
name='violet.jpg')
f["cherry"] = rc.fs.create_file(dir_id=flowers_dir['id'],
name='cherry.mpeg')
f["pasta"] = rc.fs.create_file(dir_id=foods_dir['id'], name='pasta.txt')
f["greenbeans"] = rc.fs.create_file(dir_id=foods_dir['id'],
name='greenbeans.txt')
f["rice"] = rc.fs.create_file(dir_id=foods_dir['id'], name='rice.txt')
f["sushi"] = rc.fs.create_file(dir_id=foods_dir['id'], name='寿.漢')
f["sushi_test"] = rc.fs.create_file(dir_id=foods_dir['id'], name='寿.test')
rc.fs.set_file_attr(id_=f["greenbeans"]["id"], mode='0000')
log_it("Write data to files")
f_size = 1
for k, v in f.items():
fw = io.BytesIO(b'0123456789' * f_size)
fw.seek(0)
rc.fs.write_file(data_file=fw, id_=v["id"])
f_size *= 4
fw.close()
log_it("Start: DataReductionTest")
w = QWalkWorker(creds, DataReductionTest(['--perc', '1']), args.d, True,
LOG_FILE_NAME, None)
w.run()
print("-" * 80)
print(open(DataReductionTest.FILE_NAME).read().strip())
print("-" * 80)
w.run_class.work_done(w)
os.remove(DataReductionTest.FILE_NAME)
log_it("Done with DataReductionTest")
log_it("Start: ModeBitsChecker")
w = QWalkWorker(creds, ModeBitsChecker, args.d, True, LOG_FILE_NAME, None)
w.run()
print("-" * 80)
print(open(ModeBitsChecker.FILE_NAME).read().strip())
print("-" * 80)
w.run_class.work_done(w)
os.remove(ModeBitsChecker.FILE_NAME)
log_it("Done with ModeBitsChecker")
log_it("Start: SummarizeOwners")
w = QWalkWorker(creds, SummarizeOwners, args.d, True, LOG_FILE_NAME, None)
w.run()
w.run_class.work_done(w)
test_search(creds, args, ['--re', '.*jpeg'])
log_it("Start: ChangeExtension: 'jpeg' to 'jpg'")
w = QWalkWorker(creds, ChangeExtension(['--from', 'jpeg', '--to', 'jpg']),
args.d, True, LOG_FILE_NAME, None)
w.run()
log_it("Done : ChangeExtension: 'jpeg' to 'jpg'")
test_search(creds, args, ['--re', '.*jpeg'])
test_search(creds, args, ['--re', '.*'])
test_search(creds, args, ['--str', 'rose'])
test_search(creds, args, ['--str', 'pig'])
log_it("Delete directory: %s/%s" %
(parent_dir if parent_dir != '/' else '', test_dir_name))
rc.fs.delete_tree(id_=test_dir['id'])
def main() -> None:
parser = argparse.ArgumentParser(description="Test the qwalk.py script")
parser.add_argument("-s", help="Qumulo hostname", required=True)
parser.add_argument(
"-u", help="Qumulo API user", default=os.getenv("QUSER") or "admin"
)
parser.add_argument(
"-p", help="Qumulo API password", default=os.getenv("QPASS") or "admin"
)
parser.add_argument("-d", help="Test Directory", default="/test-qwalk-parent")
try:
args, _other_args = parser.parse_known_args()
except:
print("-" * 80)
parser.print_help()
print("-" * 80)
sys.exit(0)
# Everything will happen in a new subdirectory.
test_dir_name = "test-qwalk"
creds: Creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
log_it("Log in to: %s" % (args.s))
rc = RestClient(creds["QHOST"], 8000)
rc.login(creds["QUSER"], creds["QPASS"])
parent_dir = "/"
if args.d != "/":
parent_dir = re.sub(r"/$", "", args.d)
log_it(
"Create directory: %s/%s"
% (parent_dir if parent_dir != "/" else "", test_dir_name)
)
test_dir = rc.fs.create_directory(dir_path=parent_dir, name=test_dir_name)
args.d = "%s/%s" % (parent_dir, "test-qwalk")
flowers_dir = rc.fs.create_directory(dir_path=args.d, name="flowers")
foods_dir = rc.fs.create_directory(dir_path=args.d, name="foods")
log_it("Create files")
f = {}
f["cat"] = rc.fs.create_file(dir_id=test_dir["id"], name="cat.txt")
f["mouse"] = rc.fs.create_file(dir_id=test_dir["id"], name="mouse.jpg")
f["dog"] = rc.fs.create_file(dir_id=test_dir["id"], name="dog.jpeg")
f["bear"] = rc.fs.create_file(dir_id=test_dir["id"], name="bear.mov")
f["rose"] = rc.fs.create_file(dir_id=flowers_dir["id"], name="rose.jpg")
f["violet"] = rc.fs.create_file(dir_id=flowers_dir["id"], name="violet.jpg")
f["cherry"] = rc.fs.create_file(dir_id=flowers_dir["id"], name="cherry.mpeg")
f["pasta"] = rc.fs.create_file(dir_id=foods_dir["id"], name="pasta.txt")
f["greenbeans"] = rc.fs.create_file(dir_id=foods_dir["id"], name="greenbeans.txt")
f["rice"] = rc.fs.create_file(dir_id=foods_dir["id"], name="rice.txt")
f["sushi"] = rc.fs.create_file(dir_id=foods_dir["id"], name="寿.漢")
f["sushi_test"] = rc.fs.create_file(dir_id=foods_dir["id"], name="寿.test")
rc.fs.set_file_attr(id_=f["greenbeans"]["id"], mode="0000")
log_it("Write data to files")
f_size = 1
for _k, v in f.items():
fw = io.BytesIO(b"0123456789" * f_size)
fw.seek(0)
rc.fs.write_file(data_file=fw, id_=v["id"])
f_size *= 4
fw.close()
log_it("Test CopyDirectory")
w = QWalkWorker(
creds,
CopyDirectory(["--to_dir", parent_dir + "/test-qwalk-copy"]),
args.d,
None,
True,
LOG_FILE_NAME,
None,
)
w.run()
items = read_full_tree_flat(rc, parent_dir + "/test-qwalk-copy")
log_it("Copy item count: %s" % len(items))
print("-" * 80)
log_it("Test ApplyAcls")
log_it("acls: %s" % len(rc.fs.get_acl(id_=f["pasta"]["id"])["acl"]["aces"]))
w = QWalkWorker(
creds,
ApplyAcls(["--replace_acls", "examples/acls-everyone-all-access.json"]),
foods_dir["path"],
None,
True,
LOG_FILE_NAME,
None,
)
w.run()
log_it("acls: %s" % len(rc.fs.get_acl(id_=f["pasta"]["id"])["acl"]["aces"]))
w = QWalkWorker(
creds,
ApplyAcls(["--add_entry", "examples/ace-everyone-read-only.json"]),
foods_dir["path"],
None,
True,
LOG_FILE_NAME,
None,
)
w.run()
log_it("acls: %s" % len(rc.fs.get_acl(id_=f["pasta"]["id"])["acl"]["aces"]))
log_it("acls before : %s" % len(rc.fs.get_acl(id_=foods_dir["id"])["acl"]["aces"]))
w = QWalkWorker(
creds,
ApplyAcls(
[
"--add_entry",
"examples/ace-everyone-execute-traverse.json",
"--dirs_only",
]
),
test_dir["path"],
None,
True,
LOG_FILE_NAME,
None,
)
w.run()
log_it("acls after: %s" % len(rc.fs.get_acl(id_=foods_dir["id"])["acl"]["aces"]))
log_it("Done Test ApplyAcls")
print("-" * 80)
log_it("Test snapshot search after deleting file")
snap = rc.snapshot.create_snapshot(name="test-qwalk", id_=test_dir["id"])
rc.fs.delete(id_=f["pasta"]["id"])
test_search(creds, args.d, ["--str", "pasta"], snap["id"])
log_it("Test snapshot recover")
w = QWalkWorker(
creds,
CopyDirectory(["--to_dir", parent_dir + "/copy-from-snap"]),
args.d,
snap["id"],
True,
LOG_FILE_NAME,
None,
)
w.run()
items = read_full_tree_flat(rc, parent_dir + "/copy-from-snap")
log_it("Copy item count in snap: %s" % len(items))
rc.snapshot.delete_snapshot(snap["id"])
log_it("Deleted test snapshot")
print("-" * 80)
log_it("Start: DataReductionTest")
w = QWalkWorker(
creds,
DataReductionTest(["--perc", "1"]),
args.d,
None,
True,
LOG_FILE_NAME,
None,
)
w.run()
print("." * 80)
print(open(DataReductionTest.FILE_NAME).read().strip())
print("." * 80)
w.run_task.work_done(w)
os.remove(DataReductionTest.FILE_NAME)
log_it("Done with DataReductionTest")
print("-" * 80)
log_it("Start: ModeBitsChecker")
rc.fs.set_file_attr(id_=f["greenbeans"]["id"], mode="0000")
w = QWalkWorker(creds, ModeBitsChecker([]), args.d, None, True, LOG_FILE_NAME, None)
w.run()
print("." * 80)
print(open(ModeBitsChecker.FILE_NAME).read().strip())
print("." * 80)
w.run_task.work_done(w)
os.remove(ModeBitsChecker.FILE_NAME)
log_it("Done with ModeBitsChecker")
print("-" * 80)
log_it("Start: SummarizeOwners")
w = QWalkWorker(creds, SummarizeOwners([]), args.d, None, True, LOG_FILE_NAME, None)
w.run()
w.run_task.work_done(w)
test_search(creds, args.d, ["--re", ".*jpeg"])
log_it("Start: ChangeExtension: 'jpeg' to 'jpg'")
w = QWalkWorker(
creds,
ChangeExtension(["--from", "jpeg", "--to", "jpg"]),
args.d,
None,
True,
LOG_FILE_NAME,
None,
)
w.run()
log_it("Done : ChangeExtension: 'jpeg' to 'jpg'")
test_search(creds, args.d, ["--re", ".*jpeg"])
print("-" * 80)
test_search(creds, args.d, ["--re", ".*[.]txt"])
print("-" * 80)
test_search(creds, args.d, ["--str", "rose"])
print("-" * 80)
test_search(creds, args.d, ["--str", "pig"])
print("-" * 80)
log_it(
"Copy tree file count: %s"
% (
rc.fs.read_dir_aggregates(
path=parent_dir + "/test-qwalk-copy", max_entries=0
)["total_files"]
)
)
log_it(
"Delete directory: %s/%s"
% (parent_dir if parent_dir != "/" else "", test_dir_name)
)
rc.fs.delete_tree(id_=test_dir["id"])
rc.fs.delete_tree(path=parent_dir + "/test-qwalk-copy")
rc.fs.delete_tree(path=parent_dir + "/copy-from-snap")