def process_snap_diff(creds, path, snap_before, snap_after):
    q = multiprocessing.Queue()
    q_len = multiprocessing.Value("i", 0)
    q_lock = multiprocessing.Lock()

    pool = multiprocessing.Pool(MAX_WORKER_COUNT, 
                                 snap_worker,
                                 (creds, q, q_lock, q_len))

    rc = RestClient(creds["QHOST"], 8000)
    rc.login(creds["QUSER"], creds["QPASS"])
    results = rc.snapshot.get_all_snapshot_tree_diff(older_snap=snap_before['id'], newer_snap=snap_after['id'])
    ent_list = []
    for res in results:
        for ent in res['entries']:
            print(ent)
            add_to_q(q, q_lock, q_len, {"type":"diff_item", "value": ent})
    #         ent_list.append(ent)
    #         if len(ent_list) > 1:
    #             add_to_q(q, q_lock, q_len, ent_list)
    #             ent_list = []
    # add_to_q(q, q_lock, q_len, ent_list)
    while True:
        log_it("Queue length: %s" % q_len.value)
        time.sleep(WAIT_SECONDS)
        if q_len.value <= 0:
            break
Exemple #2
0
def main():
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('-s', help='Qumulo hostname', required=True)
    parser.add_argument('-u',
                        help='Qumulo API user',
                        default=os.getenv('QUSER') or 'admin')
    parser.add_argument('-p',
                        help='Qumulo API password',
                        default=os.getenv('QPASS') or 'admin')
    parser.add_argument('-d', help='Root Directory', default='/')
    try:
        args, other_args = parser.parse_known_args()
    except:
        print("-" * 80)
        parser.print_help()
        print("-" * 80)
        sys.exit(0)
    if args.d != '/':
        args.d = re.sub('/$', '', args.d) + '/'

    creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
    log_it("Log in to: %s" % (args.s))
    rc = RestClient(creds["QHOST"], 8000)
    rc.login(creds["QUSER"], creds["QPASS"])

    res = rc.snapshot.list_snapshot_statuses()
    snap_before = None
    snap_after = None
    for snap in sorted(res['entries'], key=lambda d: int(d['id'])):
        if snap['name'] == SNAP_NAME and snap['source_file_path'] == args.d:
            if snap_before is None:
                snap_before = snap
            elif snap_after is None:
                snap_after = snap

    if snap_before and not snap_after:
        snap_after = rc.snapshot.create_snapshot(path=args.d, name=SNAP_NAME)
        log_it("Created new snapshot %s on %s" % (snap_after['id'], args.d))

    if snap_before and snap_after:
        log_it("Diffing snaps on %s between %s and %s" %
               (args.d, snap_before['timestamp'][0:16],
                snap_after['timestamp'][0:16]))
        process_snap_diff(creds, args.d, snap_before['id'], snap_after['id'])
        rc.snapshot.delete_snapshot(snapshot_id=snap_before['id'])
    else:
        # initial tree walk
        snap_before = rc.snapshot.create_snapshot(path=args.d, name=SNAP_NAME)
        log_it("Initial tree walk for: %s+snap:%s" %
               (args.d, snap_before['id']))
        log_file = "output-qumulo-fs-index-%s-tree.txt" % (re.sub(
            "[^a-z0-9]+", "_", args.d))
        w = QWalkWorker(
            creds,
            Search([
                '--re', '.', '--cols',
                'dir_id,id,type,path,name,size,blocks,owner,change_time,link_target,NEW'
            ]), args.d, str(snap_before['id']), None, log_file, None)
        w.run()
def main():
    parser = argparse.ArgumentParser(description='')
    parser.add_argument('-s', help='Qumulo hostname', required=True)
    parser.add_argument('-u', help='Qumulo API user', 
                              default=os.getenv('QUSER') or 'admin')
    parser.add_argument('-p', help='Qumulo API password',
                              default=os.getenv('QPASS') or 'admin')
    parser.add_argument('-d', help='Root Directory', default='/')
    try:
        args, other_args = parser.parse_known_args()
    except:
        print("-"*80)
        parser.print_help()
        print("-"*80)
        sys.exit(0)
    if args.d != '/':
        args.d = re.sub('/$', '', args.d) + '/'

    creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
    log_it("Log in to: %s" % (args.s))
    rc = RestClient(creds["QHOST"], 8000)
    rc.login(creds["QUSER"], creds["QPASS"])

    res = rc.snapshot.list_snapshot_statuses()
    existing_snap = None
    for snap in res['entries']:
        if snap['name'] == SNAP_NAME and snap['source_file_path'] == args.d:
            existing_snap = snap
            break
    # snap = rcs[len(rcs)-1].snapshot.create_snapshot(path=path, name=SNAP_NAME)

    snap_before = rc.snapshot.get_snapshot(746219)
    snap_after = rc.snapshot.get_snapshot(748466)

    if snap_before:
        process_snap_diff(creds, 
                          args.d,
                          snap_before,
                          snap_after
                          )
    else:
        w = QWalkWorker(creds, 
                    Search(['--re', '.', 
                            '--cols', 'dir_id,id,type,name,size,blocks,owner,change_time']), 
                    args.d,
                    None, "qumulo-fs-index.txt", None)
def snap_worker(creds, q, q_lock, q_len):
    p_name = multiprocessing.current_process().name
    worker_id = int(re.match(r'.*?-([0-9])+', p_name).groups(1)[0])-1
    rc = RestClient(creds["QHOST"], 8000)
    rc.login(creds["QUSER"], creds["QPASS"])
    file_list = []
    while True:
        try:
            data = q.get(True, timeout=5)
            if data["value"]["op"] == "DELETE":
                print("DELETE  - %s" % data)
                pass
            elif data["type"] == "new_dir":
                print("NEW DIR  - %s" % data)
                for dd in rc.fs.read_entire_directory(id_ = data["id"]):
                    for d in dd["files"]:
                        print("NEW ITEM: %s" % d["name"])
            else:
                d = rc.fs.get_file_attr(path = data["value"]["path"])
                if data["value"]["op"] == "CREATE" and d["type"] == "FS_FILE_TYPE_DIRECTORY":
                    add_to_q(q, q_lock, q_len, {"type":"new_dir", "value": data["value"], "id": d["id"]})
                elif data["value"]["op"] == "CREATE":
                    print("NEW FILE - %s" % data)
                elif d["type"] != "FS_FILE_TYPE_DIRECTORY":
                    print("CHANGE   - %s" % data)
        except queue.Empty:
            # this is expected
            break
        except:
            # this is not expected
            log_it("!! Exception !!")
            log_it(sys.exc_info())
            traceback.print_exc(file=sys.stdout)
        time.sleep(random.random())
        with q_lock:
            q_len.value -= 1
def test_search(creds, args, search):
    log_it("Search: %s" % search)
    w = QWalkWorker(creds, Search(search), args.d, None, LOG_FILE_NAME, None)
    w.run()
    if os.path.exists(LOG_FILE_NAME):
        content = re.sub(r'[\r\n]+', ' ', open(LOG_FILE_NAME).read())
        log_it("FOUND!  : Search found - %s" % content)
        os.remove(LOG_FILE_NAME)
    else:
        log_it("NOTFOUND: Search failure: %s" % search)
Exemple #6
0
def test_search(
    creds: Creds, start_path: str, search: Sequence[str], snapshot: Optional[str] = None
) -> None:
    log_it("Search: %s" % search)
    w = QWalkWorker(
        creds, Search(search), start_path, snapshot, False, LOG_FILE_NAME, None
    )
    w.run()
    if os.path.exists(LOG_FILE_NAME):
        content = re.sub(r"[\r\n]+", " ", open(LOG_FILE_NAME).read())
        log_it("FOUND!  : Search found - %s" % content)
        os.remove(LOG_FILE_NAME)
    else:
        log_it("NOTFOUND: Search failure: %s" % search)
Exemple #7
0
def snap_worker(creds, q, q_lock, q_len, w_lock, w_file):
    p_name = multiprocessing.current_process().name
    worker_id = int(re.match(r'.*?-([0-9])+', p_name).groups(1)[0]) - 1
    rc = RestClient(creds["QHOST"], 8000)
    rc.login(creds["QUSER"], creds["QPASS"])
    file_list = []
    path_ids = {}

    while True:
        try:
            data = q.get(True, timeout=5)
            log_items = []
            for li in data["list"]:
                if li["op"] == "DELETE":
                    if "item" not in li:
                        old_item = rc.fs.get_file_attr(
                            path=li["path"], snapshot=data["snap_before_id"])
                    else:
                        old_item = li["item"]
                    if old_item["type"] == "FS_FILE_TYPE_DIRECTORY":
                        log_items.append(
                            log_item(rc, path_ids, "DELETE",
                                     data["snap_before_id"], li["dir_id"],
                                     old_item))
                        list_items = []
                        for dd in rc.fs.read_entire_directory(
                                id_=old_item["id"],
                                snapshot=data["snap_before_id"]):
                            for item in dd["files"]:
                                if item["type"] == "FS_FILE_TYPE_DIRECTORY":
                                    list_items.append({
                                        "op": "DELETE",
                                        "item": item,
                                        "path": item["path"],
                                        "dir_id": old_item["id"]
                                    })
                                else:
                                    log_items.append(
                                        log_item(rc, path_ids, "DELETE",
                                                 data["snap_before_id"],
                                                 old_item["id"], item))
                        add_to_q(
                            q, q_lock, q_len, {
                                "list": list_items,
                                "snap_before_id": data["snap_before_id"],
                                "snap_after_id": data["snap_after_id"]
                            })

                    else:
                        log_items.append(
                            log_item(rc, path_ids, "DELETE",
                                     data["snap_before_id"], li["dir_id"],
                                     old_item))
                    continue
                # item exists in new snapshot because it's not a delete
                if "item" not in li:
                    new_item = rc.fs.get_file_attr(
                        path=li["path"], snapshot=data["snap_after_id"])
                else:
                    new_item = li["item"]
                if li["op"] == "CREATE" and new_item[
                        "type"] == "FS_FILE_TYPE_DIRECTORY":
                    log_items.append(
                        log_item(rc, path_ids, "CREATE", data["snap_after_id"],
                                 li["dir_id"], new_item))
                    list_items = []
                    for dd in rc.fs.read_entire_directory(
                            id_=new_item["id"],
                            snapshot=data["snap_after_id"]):
                        for item in dd["files"]:
                            if item["type"] == "FS_FILE_TYPE_DIRECTORY":
                                list_items.append({
                                    "op": "CREATE",
                                    "item": item,
                                    "path": item["path"],
                                    "dir_id": new_item["id"]
                                })
                            else:
                                log_items.append(
                                    log_item(rc, path_ids, "CREATE",
                                             data["snap_after_id"],
                                             new_item["id"], item))
                    add_to_q(
                        q, q_lock, q_len, {
                            "list": list_items,
                            "snap_before_id": data["snap_before_id"],
                            "snap_after_id": data["snap_after_id"]
                        })
                elif li["op"] == "CREATE":
                    log_items.append(
                        log_item(rc, path_ids, "CREATE", data["snap_after_id"],
                                 None, new_item))
                elif new_item["type"] == "FS_FILE_TYPE_DIRECTORY":
                    # TODO: nothing, I think
                    pass
                elif new_item["type"] != "FS_FILE_TYPE_DIRECTORY":
                    log_items.append(
                        log_item(rc, path_ids, "MODIFY", data["snap_after_id"],
                                 None, new_item))
        except queue.Empty:
            # this is expected
            break
        except:
            # this is not expected
            log_it("!! Exception !!")
            log_it(sys.exc_info())
            traceback.print_exc(file=sys.stdout)
        with w_lock:
            fw = open(w_file, mode="a", encoding='utf-8')
            for li in log_items:
                fw.write(li + "\n")
            fw.close()
        log_items = []
        with q_lock:
            q_len.value -= 1
Exemple #8
0
def process_snap_diff(creds, path, snap_before_id, snap_after_id):
    q = multiprocessing.Queue()
    q_len = multiprocessing.Value("i", 0)
    q_lock = multiprocessing.Lock()
    w_lock = multiprocessing.Lock()
    w_file = "output-qumulo-fs-index-%s-change-log-%s-%s.txt" % (re.sub(
        "[^a-z0-9]+", "_", path), snap_before_id, snap_after_id)
    fw = open(w_file, mode="w", encoding='utf-8')
    fw.close()
    rc = RestClient(creds["QHOST"], 8000)
    rc.login(creds["QUSER"], creds["QPASS"])
    log_it("Run get_all_snapshot_tree_diff")
    results = rc.snapshot.get_all_snapshot_tree_diff(older_snap=snap_before_id,
                                                     newer_snap=snap_after_id)
    log_it("Done get_all_snapshot_tree_diff.")
    log_it("Creating worker pool.")
    pool = multiprocessing.Pool(MAX_WORKER_COUNT, snap_worker,
                                (creds, q, q_lock, q_len, w_lock, w_file))
    log_it("Add items to queue.")
    ent_list = []
    for res in results:
        for ent in res['entries']:
            ent["dir_id"] = None
            ent_list.append(ent)
            if len(ent_list) > 5:
                add_to_q(
                    q, q_lock, q_len, {
                        "list": ent_list,
                        "snap_before_id": snap_before_id,
                        "snap_after_id": snap_after_id
                    })
                ent_list = []
    add_to_q(
        q, q_lock, q_len, {
            "list": ent_list,
            "snap_before_id": snap_before_id,
            "snap_after_id": snap_after_id
        })
    log_it("Done adding items to queue.")
    while True:
        log_it("Queue length: %s" % q_len.value)
        time.sleep(WAIT_SECONDS)
        if q_len.value <= 0:
            break
def main():
    parser = argparse.ArgumentParser(description='Test the qwalk.py script')
    parser.add_argument('-s', help='Qumulo hostname', required=True)
    parser.add_argument('-u',
                        help='Qumulo API user',
                        default=os.getenv('QUSER') or 'admin')
    parser.add_argument('-p',
                        help='Qumulo API password',
                        default=os.getenv('QPASS') or 'admin')
    parser.add_argument('-d', help='Test Directory', default='/')

    try:
        args, other_args = parser.parse_known_args()
    except:
        print("-" * 80)
        parser.print_help()
        print("-" * 80)
        sys.exit(0)

    # Everything will happen in a new subdirectory.
    test_dir_name = 'test-qwalk'
    creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
    log_it("Log in to: %s" % (args.s))
    rc = RestClient(creds["QHOST"], 8000)
    rc.login(creds["QUSER"], creds["QPASS"])
    parent_dir = '/'
    if args.d != '/':
        parent_dir = re.sub(r'/$', '', args.d)
    log_it("Create directory: %s/%s" %
           (parent_dir if parent_dir != '/' else '', test_dir_name))
    test_dir = rc.fs.create_directory(dir_path=parent_dir, name=test_dir_name)
    args.d = "%s/%s" % (parent_dir, 'test-qwalk')
    flowers_dir = rc.fs.create_directory(dir_path=args.d, name='flowers')
    foods_dir = rc.fs.create_directory(dir_path=args.d, name='foods')

    log_it("Create files")
    f = {}
    f["cat"] = rc.fs.create_file(dir_id=test_dir['id'], name='cat.txt')
    f["mouse"] = rc.fs.create_file(dir_id=test_dir['id'], name='mouse.jpg')
    f["dog"] = rc.fs.create_file(dir_id=test_dir['id'], name='dog.jpeg')
    f["bear"] = rc.fs.create_file(dir_id=test_dir['id'], name='bear.mov')
    f["rose"] = rc.fs.create_file(dir_id=flowers_dir['id'], name='rose.jpg')
    f["violet"] = rc.fs.create_file(dir_id=flowers_dir['id'],
                                    name='violet.jpg')
    f["cherry"] = rc.fs.create_file(dir_id=flowers_dir['id'],
                                    name='cherry.mpeg')
    f["pasta"] = rc.fs.create_file(dir_id=foods_dir['id'], name='pasta.txt')
    f["greenbeans"] = rc.fs.create_file(dir_id=foods_dir['id'],
                                        name='greenbeans.txt')
    f["rice"] = rc.fs.create_file(dir_id=foods_dir['id'], name='rice.txt')
    f["sushi"] = rc.fs.create_file(dir_id=foods_dir['id'], name='寿.漢')
    f["sushi_test"] = rc.fs.create_file(dir_id=foods_dir['id'], name='寿.test')

    rc.fs.set_file_attr(id_=f["greenbeans"]["id"], mode='0000')

    log_it("Write data to files")
    f_size = 1
    for k, v in f.items():
        fw = io.BytesIO(b'0123456789' * f_size)
        fw.seek(0)
        rc.fs.write_file(data_file=fw, id_=v["id"])
        f_size *= 4
        fw.close()

    log_it("Start: DataReductionTest")
    w = QWalkWorker(creds, DataReductionTest(['--perc', '1']), args.d, True,
                    LOG_FILE_NAME, None)
    w.run()
    print("-" * 80)
    print(open(DataReductionTest.FILE_NAME).read().strip())
    print("-" * 80)
    w.run_class.work_done(w)
    os.remove(DataReductionTest.FILE_NAME)
    log_it("Done with DataReductionTest")

    log_it("Start: ModeBitsChecker")
    w = QWalkWorker(creds, ModeBitsChecker, args.d, True, LOG_FILE_NAME, None)
    w.run()
    print("-" * 80)
    print(open(ModeBitsChecker.FILE_NAME).read().strip())
    print("-" * 80)
    w.run_class.work_done(w)
    os.remove(ModeBitsChecker.FILE_NAME)
    log_it("Done with ModeBitsChecker")

    log_it("Start: SummarizeOwners")
    w = QWalkWorker(creds, SummarizeOwners, args.d, True, LOG_FILE_NAME, None)
    w.run()
    w.run_class.work_done(w)

    test_search(creds, args, ['--re', '.*jpeg'])
    log_it("Start: ChangeExtension: 'jpeg' to 'jpg'")
    w = QWalkWorker(creds, ChangeExtension(['--from', 'jpeg', '--to', 'jpg']),
                    args.d, True, LOG_FILE_NAME, None)
    w.run()
    log_it("Done : ChangeExtension: 'jpeg' to 'jpg'")
    test_search(creds, args, ['--re', '.*jpeg'])

    test_search(creds, args, ['--re', '.*'])
    test_search(creds, args, ['--str', 'rose'])
    test_search(creds, args, ['--str', 'pig'])

    log_it("Delete directory: %s/%s" %
           (parent_dir if parent_dir != '/' else '', test_dir_name))
    rc.fs.delete_tree(id_=test_dir['id'])
Exemple #10
0
def main() -> None:
    parser = argparse.ArgumentParser(description="Test the qwalk.py script")
    parser.add_argument("-s", help="Qumulo hostname", required=True)
    parser.add_argument(
        "-u", help="Qumulo API user", default=os.getenv("QUSER") or "admin"
    )
    parser.add_argument(
        "-p", help="Qumulo API password", default=os.getenv("QPASS") or "admin"
    )
    parser.add_argument("-d", help="Test Directory", default="/test-qwalk-parent")

    try:
        args, _other_args = parser.parse_known_args()
    except:
        print("-" * 80)
        parser.print_help()
        print("-" * 80)
        sys.exit(0)

    # Everything will happen in a new subdirectory.
    test_dir_name = "test-qwalk"
    creds: Creds = {"QHOST": args.s, "QUSER": args.u, "QPASS": args.p}
    log_it("Log in to: %s" % (args.s))
    rc = RestClient(creds["QHOST"], 8000)
    rc.login(creds["QUSER"], creds["QPASS"])
    parent_dir = "/"
    if args.d != "/":
        parent_dir = re.sub(r"/$", "", args.d)
    log_it(
        "Create directory: %s/%s"
        % (parent_dir if parent_dir != "/" else "", test_dir_name)
    )
    test_dir = rc.fs.create_directory(dir_path=parent_dir, name=test_dir_name)
    args.d = "%s/%s" % (parent_dir, "test-qwalk")
    flowers_dir = rc.fs.create_directory(dir_path=args.d, name="flowers")
    foods_dir = rc.fs.create_directory(dir_path=args.d, name="foods")

    log_it("Create files")
    f = {}
    f["cat"] = rc.fs.create_file(dir_id=test_dir["id"], name="cat.txt")
    f["mouse"] = rc.fs.create_file(dir_id=test_dir["id"], name="mouse.jpg")
    f["dog"] = rc.fs.create_file(dir_id=test_dir["id"], name="dog.jpeg")
    f["bear"] = rc.fs.create_file(dir_id=test_dir["id"], name="bear.mov")
    f["rose"] = rc.fs.create_file(dir_id=flowers_dir["id"], name="rose.jpg")
    f["violet"] = rc.fs.create_file(dir_id=flowers_dir["id"], name="violet.jpg")
    f["cherry"] = rc.fs.create_file(dir_id=flowers_dir["id"], name="cherry.mpeg")
    f["pasta"] = rc.fs.create_file(dir_id=foods_dir["id"], name="pasta.txt")
    f["greenbeans"] = rc.fs.create_file(dir_id=foods_dir["id"], name="greenbeans.txt")
    f["rice"] = rc.fs.create_file(dir_id=foods_dir["id"], name="rice.txt")
    f["sushi"] = rc.fs.create_file(dir_id=foods_dir["id"], name="寿.漢")
    f["sushi_test"] = rc.fs.create_file(dir_id=foods_dir["id"], name="寿.test")
    rc.fs.set_file_attr(id_=f["greenbeans"]["id"], mode="0000")
    log_it("Write data to files")
    f_size = 1
    for _k, v in f.items():
        fw = io.BytesIO(b"0123456789" * f_size)
        fw.seek(0)
        rc.fs.write_file(data_file=fw, id_=v["id"])
        f_size *= 4
        fw.close()

    log_it("Test CopyDirectory")
    w = QWalkWorker(
        creds,
        CopyDirectory(["--to_dir", parent_dir + "/test-qwalk-copy"]),
        args.d,
        None,
        True,
        LOG_FILE_NAME,
        None,
    )
    w.run()
    items = read_full_tree_flat(rc, parent_dir + "/test-qwalk-copy")
    log_it("Copy item count: %s" % len(items))

    print("-" * 80)
    log_it("Test ApplyAcls")
    log_it("acls: %s" % len(rc.fs.get_acl(id_=f["pasta"]["id"])["acl"]["aces"]))
    w = QWalkWorker(
        creds,
        ApplyAcls(["--replace_acls", "examples/acls-everyone-all-access.json"]),
        foods_dir["path"],
        None,
        True,
        LOG_FILE_NAME,
        None,
    )
    w.run()
    log_it("acls: %s" % len(rc.fs.get_acl(id_=f["pasta"]["id"])["acl"]["aces"]))
    w = QWalkWorker(
        creds,
        ApplyAcls(["--add_entry", "examples/ace-everyone-read-only.json"]),
        foods_dir["path"],
        None,
        True,
        LOG_FILE_NAME,
        None,
    )
    w.run()
    log_it("acls: %s" % len(rc.fs.get_acl(id_=f["pasta"]["id"])["acl"]["aces"]))
    log_it("acls before : %s" % len(rc.fs.get_acl(id_=foods_dir["id"])["acl"]["aces"]))
    w = QWalkWorker(
        creds,
        ApplyAcls(
            [
                "--add_entry",
                "examples/ace-everyone-execute-traverse.json",
                "--dirs_only",
            ]
        ),
        test_dir["path"],
        None,
        True,
        LOG_FILE_NAME,
        None,
    )
    w.run()
    log_it("acls after: %s" % len(rc.fs.get_acl(id_=foods_dir["id"])["acl"]["aces"]))
    log_it("Done Test ApplyAcls")
    print("-" * 80)

    log_it("Test snapshot search after deleting file")
    snap = rc.snapshot.create_snapshot(name="test-qwalk", id_=test_dir["id"])
    rc.fs.delete(id_=f["pasta"]["id"])
    test_search(creds, args.d, ["--str", "pasta"], snap["id"])

    log_it("Test snapshot recover")
    w = QWalkWorker(
        creds,
        CopyDirectory(["--to_dir", parent_dir + "/copy-from-snap"]),
        args.d,
        snap["id"],
        True,
        LOG_FILE_NAME,
        None,
    )
    w.run()
    items = read_full_tree_flat(rc, parent_dir + "/copy-from-snap")
    log_it("Copy item count in snap: %s" % len(items))

    rc.snapshot.delete_snapshot(snap["id"])
    log_it("Deleted test snapshot")
    print("-" * 80)

    log_it("Start: DataReductionTest")
    w = QWalkWorker(
        creds,
        DataReductionTest(["--perc", "1"]),
        args.d,
        None,
        True,
        LOG_FILE_NAME,
        None,
    )
    w.run()
    print("." * 80)
    print(open(DataReductionTest.FILE_NAME).read().strip())
    print("." * 80)
    w.run_task.work_done(w)
    os.remove(DataReductionTest.FILE_NAME)
    log_it("Done with DataReductionTest")
    print("-" * 80)

    log_it("Start: ModeBitsChecker")
    rc.fs.set_file_attr(id_=f["greenbeans"]["id"], mode="0000")
    w = QWalkWorker(creds, ModeBitsChecker([]), args.d, None, True, LOG_FILE_NAME, None)
    w.run()
    print("." * 80)
    print(open(ModeBitsChecker.FILE_NAME).read().strip())
    print("." * 80)
    w.run_task.work_done(w)
    os.remove(ModeBitsChecker.FILE_NAME)
    log_it("Done with ModeBitsChecker")
    print("-" * 80)

    log_it("Start: SummarizeOwners")
    w = QWalkWorker(creds, SummarizeOwners([]), args.d, None, True, LOG_FILE_NAME, None)
    w.run()
    w.run_task.work_done(w)

    test_search(creds, args.d, ["--re", ".*jpeg"])
    log_it("Start: ChangeExtension: 'jpeg' to 'jpg'")
    w = QWalkWorker(
        creds,
        ChangeExtension(["--from", "jpeg", "--to", "jpg"]),
        args.d,
        None,
        True,
        LOG_FILE_NAME,
        None,
    )
    w.run()
    log_it("Done : ChangeExtension: 'jpeg' to 'jpg'")
    test_search(creds, args.d, ["--re", ".*jpeg"])
    print("-" * 80)
    test_search(creds, args.d, ["--re", ".*[.]txt"])
    print("-" * 80)
    test_search(creds, args.d, ["--str", "rose"])
    print("-" * 80)
    test_search(creds, args.d, ["--str", "pig"])
    print("-" * 80)

    log_it(
        "Copy tree file count: %s"
        % (
            rc.fs.read_dir_aggregates(
                path=parent_dir + "/test-qwalk-copy", max_entries=0
            )["total_files"]
        )
    )

    log_it(
        "Delete directory: %s/%s"
        % (parent_dir if parent_dir != "/" else "", test_dir_name)
    )
    rc.fs.delete_tree(id_=test_dir["id"])
    rc.fs.delete_tree(path=parent_dir + "/test-qwalk-copy")
    rc.fs.delete_tree(path=parent_dir + "/copy-from-snap")