def test_rm_commit_offline(runner): r = runner # create files in root for idx in range(6): util.mkrandfile(f"root-{idx}.git", 5) util.mkrandfile(f"root-{idx}.annex", 2000) status = util.zerostatus() status["??"] += 12 util.assert_status(r, status=status) r.runcommand("gin", "commit", "root*") status["LC"] += 12 status["??"] -= 12 util.assert_status(r, status=status) r.runcommand("gin", "upload") status["LC"] -= 12 status["OK"] += 12 util.assert_status(r, status=status) for idx in range(2, 4): os.remove(f"root-{idx}.git") os.remove(f"root-{idx}.annex") status["OK"] -= 4 status["RM"] += 4 util.assert_status(r, status=status) r.runcommand("gin", "upload", ".") status["RM"] = 0 util.assert_status(r, status=status)
def _untracked_conflict(runner, size): loca, locb = runner fname = "dl_over_untracked" loca.cdrel() util.mkrandfile(fname, size) hasha = util.md5sum(fname) loca.runcommand("gin", "upload", fname) locb.cdrel() util.mkrandfile(fname, size + (size // 10)) hashb = util.md5sum(fname) out, err = locb.runcommand("gin", "download", exit=False) assert err, "Expected error, got nothing" assert owerrmsg in err assert err.endswith(fname) # resolution: rename untracked file and download locb.cdrel() os.rename(fname, fname + ".bak") locb.runcommand("gin", "download") # both files are in directory now # check if the hashes match locb.runcommand("gin", "getc", ".") allfiles = os.listdir() hashes = [util.md5sum(f) for f in allfiles if f.startswith(fname)] assert sorted(hashes) == sorted([hasha, hashb])
def test_add_directory_remote(runner): r = runner r.runcommand("gin", "init") r.repositories[r.cmdloc] = None ngit = 3 nannex = 2 # create files in root for idx in range(ngit): util.mkrandfile(f"root-{idx}.git", 3) for idx in range(nannex): util.mkrandfile(f"root-{idx}.annex", 200) status = util.zerostatus() status["??"] = ngit + nannex util.assert_status(r, status=status) r.runcommand("gin", "commit", ".") status["LC"] += ngit + nannex status["??"] -= ngit + nannex util.assert_status(r, status=status) fsremotedir = os.path.join(r.testroot.name, "annexdata") r.runcommand("gin", "add-remote", "--create", "lanbackup", f"dir:{fsremotedir}") r.runcommand("gin", "git", "remote", "-v") r.repositories[fsremotedir] = None r.runcommand("gin", "upload") status["OK"] += ngit + nannex status["LC"] -= ngit + nannex util.assert_status(r, status=status)
def test_workflow(runner): r = runner # create files in root gitfiles = list() for idx in range(8): fname = f"root-{idx}.git" util.mkrandfile(fname, 5) gitfiles.append(fname) annexfiles = list() for idx in range(5): fname = f"root-{idx}.annex" annexfiles.append(fname) util.mkrandfile(fname, 2000) # upload r.runcommand("gin", "upload", ".") # delete a git file and "upload" it os.unlink(gitfiles[-1]) r.runcommand("gin", "upload", gitfiles[-1]) gitfiles = gitfiles[:-1] # delete an annex file and "upload" it os.unlink(annexfiles[-1]) r.runcommand("gin", "upload", annexfiles[-1]) annexfiles = annexfiles[:-1] # delete everything and upload, naming all deleted files allfiles = gitfiles + annexfiles for fn in allfiles: os.unlink(fn) r.runcommand("gin", "upload", *allfiles)
def test_annex_filters(runner): r = runner N = 4 # Create some big data files for idx in range(N): util.mkrandfile(f"randfile{idx}", 200000) # 10x 200 MB files r.runcommand("gin", "upload", ".") # files should be links for idx in range(N): util.isannexed(r, f"randfile{idx}") status = util.zerostatus() status["OK"] = N util.assert_status(r, status=status) r.runcommand("gin", "remove-content", ".") status["OK"] = 0 status["NC"] = N util.assert_status(r, status=status) # TODO: Don't check on Windows # locking annexed files should turn them into symlinks r.runcommand("gin", "lock", ".") for idx in range(N): assert os.path.islink(f"randfile{idx}")
def test_create_remote_on_add(runner): r = runner r.runcommand("gin", "init") r.repositories[r.cmdloc] = None ngit = 3 nannex = 2 # create files in root for idx in range(ngit): util.mkrandfile(f"root-{idx}.git", 3) for idx in range(nannex): util.mkrandfile(f"root-{idx}.annex", 200) status = util.zerostatus() status["??"] = nannex + ngit util.assert_status(r, status=status) r.runcommand("gin", "commit", ".") status["LC"] = ngit + nannex status["??"] = 0 util.assert_status(r, status=status) r.login() repopath = f"{r.username}/{r.reponame}" r.runcommand("gin", "add-remote", "--create", "origin", f"test:{repopath}") r.runcommand("gin", "upload") status["OK"] += status["LC"] status["LC"] = 0 util.assert_status(r, status=status)
def test_create_remote_prompt(runner): r = runner r.runcommand("gin", "init") r.repositories[r.cmdloc] = None ngit = 3 nannex = 2 # create files in root for idx in range(ngit): util.mkrandfile(f"root-{idx}.git", 3) for idx in range(nannex): util.mkrandfile(f"root-{idx}.annex", 200) status = util.zerostatus() status["??"] = nannex + ngit util.assert_status(r, status=status) r.runcommand("gin", "commit", ".") status["LC"] = ngit + nannex status["??"] = 0 util.assert_status(r, status=status) r.login() repopath = f"{r.username}/{r.reponame}" out, err = r.runcommand("gin", "add-remote", "origin", f"test:{repopath}", inp="abort") assert not err, f"Expected empty error, got\n{err}" assert out.endswith("aborted") out, err = r.runcommand("git", "remote", "-v") assert not out, f"Expected empty output, got\n{out}" assert not err, f"Expected empty error, got\n{err}" out, err = r.runcommand("gin", "add-remote", "origin", f"test:{repopath}", inp="add anyway") out, err = r.runcommand("git", "remote", "-v") assert len(out.splitlines()) == 2, "Unexpected output" assert not err, f"Expected empty error, got\n{err}" out, err = r.runcommand("gin", "upload", exit=False) assert err, "Expected error, got nothing" r.runcommand("git", "remote", "rm", "origin") out, err = r.runcommand("gin", "add-remote", "origin", f"test:{repopath}", inp="create") out, err = r.runcommand("gin", "upload") status["OK"] += ngit + nannex status["LC"] -= ngit + nannex util.assert_status(r, status=status)
def test_push_conflict(runner): loca, locb = runner loca.cdrel() util.mkrandfile("newfile.git", 1) loca.runcommand("gin", "upload", "newfile.git") locb.cdrel() util.mkrandfile("newfile-b.git", 1) out, err = locb.runcommand("gin", "upload", "newfile-b.git", exit=False) assert err, "Expected error, got nothing" assert out.endswith(uperrmsg) locb.runcommand("gin", "download") locb.runcommand("gin", "upload", "newfile-b.git")
def create_files(r): fnames = list() # make a few annex and git files os.makedirs("smallfiles", exist_ok=True) os.makedirs("datafiles", exist_ok=True) for idx in range(5): name = os.path.join("smallfiles", f"smallfile-{idx:03}") util.mkrandfile(name, 20) fnames.append(name) for idx in range(10): name = os.path.join("datafiles", f"datafile-{idx:03}") util.mkrandfile(name, 2000) fnames.append(name) return fnames
def _tracked_conflict(runner, sizea, sizeb): loca, locb = runner # if a file involved in a pull conflict is in the annex, it will get # renamed and the error message will be different annexed = sizea > 50 or sizeb > 50 experr = acferrmsg if annexed else cferrmsg fname = "dl_over_tracked" loca.cdrel() util.mkrandfile(fname, sizea) loca.runcommand("gin", "upload", fname) hasha = util.md5sum(fname) locb.cdrel() util.mkrandfile(fname, sizeb) hashb = util.md5sum(fname) locb.runcommand("gin", "commit", fname) out, err = locb.runcommand("gin", "download", exit=False) assert err, "Expected error, got nothing" assert experr in err if not annexed: # resolution: rename file and sync locb.cdrel() os.rename(fname, fname + ".bak") locb.runcommand("gin", "sync") assert hasha == util.md5sum(fname) assert hashb == util.md5sum(fname + ".bak") else: # both files are in directory now # check if the hashes match locb.runcommand("gin", "getc", ".") allfiles = os.listdir() hashes = [util.md5sum(f) for f in allfiles if f.startswith(fname)] assert sorted(hashes) == sorted([hasha, hashb])
def test_add_gin_remote(runner): r = runner r.runcommand("gin", "init") r.repositories[r.cmdloc] = None ngit = 3 nannex = 2 # create files in root for idx in range(ngit): util.mkrandfile(f"root-{idx}.git", 3) for idx in range(nannex): util.mkrandfile(f"root-{idx}.annex", 200) status = util.zerostatus() status["??"] = nannex + ngit util.assert_status(r, status=status) r.runcommand("gin", "commit", ".") status["LC"] = ngit + nannex status["??"] = 0 util.assert_status(r, status=status) r.login() r.runcommand("gin", "create", "--no-clone", r.reponame, "Test repository for add remote") r.repositories[r.cmdloc] = r.reponame repopath = f"{r.username}/{r.reponame}" r.runcommand("gin", "add-remote", "origin", f"test:{repopath}") r.runcommand("gin", "upload") status["OK"] += ngit + nannex status["LC"] -= ngit + nannex util.assert_status(r, status=status) # remove remote, add it with a different name (not origin) and see if it # works util.mkrandfile(f"final-file.annex", 500) r.runcommand("gin", "git", "remote", "rm", "origin") r.runcommand("gin", "git", "config", "--unset", "gin.remote") r.runcommand("gin", "add-remote", "notorigin", f"test:{repopath}") r.runcommand("gin", "upload", "final-file.annex") status["OK"] += 1 util.assert_status(r, status=status)
def test_config_path(runner): r = runner conf = dict() reporoot = os.path.join(r.testroot.name, r.reponame) localconffile = os.path.join(reporoot, "config.yml") def writelocalconf(): with open(localconffile, "w") as conffile: conffile.write(yaml.dump(conf, default_flow_style=False)) # Create local config file which sets annex threshold to 0kb conf["annex"] = {"minsize": "0kB"} writelocalconf() # small files should now be added to annex util.mkrandfile("smallfile", 1) r.runcommand("gin", "upload", "smallfile") # smallfile should be annexed assert util.isannexed(r, "smallfile") # .md file should still be excluded because of the exclusion rule in the # global configuration util.mkrandfile("anotherfile.md", 10) r.runcommand("gin", "upload", "anotherfile.md") # anotherfile.md should not be a symlink assert not util.isannexed(r, "anotherfile.md") # config file should never be added to annex r.runcommand("gin", "upload", "config.yml") assert not util.isannexed(r, "config.yml") # changing gitannex binary in local configuration should have no effect conf["bin"] = {"gitannex": "ls"} writelocalconf() util.mkrandfile("somefile", 1000) r.runcommand("gin", "commit", ".") r.runcommand("gin", "upload", ".")
def test_errors_offline(orunner): r = orunner norepoerr = "[error] this command must be run from inside a gin repository" commands = [ "upload", "lock", "unlock", "get-content", "remove-content", ] # Unable to run any git commands outside a git repository for cmd in commands: # On directory out, err = r.runcommand("gin", cmd, ".", exit=False) assert err == norepoerr # On specific file out, err = r.runcommand("gin", cmd, "foobar", exit=False) assert err == norepoerr out, err = r.runcommand("gin", "download", exit=False) assert err == norepoerr r.cdrel(r.reponame) # Unable to run any command on file that does not exist out, err = r.runcommand("gin", "upload", "foobar", exit=False) assert out.endswith(" Nothing to do") for cmd in commands[1:]: out, err = r.runcommand("gin", cmd, "foobar", exit=False) assert out.endswith("No files matched foobar") assert err == "[error] 1 operation failed" # make a few annex and git files os.mkdir("smallfiles") r.cdrel("smallfiles") for idx in range(20): util.mkrandfile(f"smallfile-{idx:03}", 20) r.cdrel("..") os.mkdir("datafiles") r.cdrel("datafiles") for idx in range(5): util.mkrandfile(f"datafile-{idx:03}", 2000) r.cdrel("..") # Nothing to do on lock, unlock, getc, rmc on untracked file(s) nothingmsg = " Nothing to do" for cmd in commands[1:]: out, err = r.runcommand("gin", cmd, "smallfiles", exit=False) assert out.endswith(nothingmsg) assert not err out, err = r.runcommand("gin", cmd, "datafiles", exit=False) assert out.endswith(nothingmsg) assert not err out, err = r.runcommand("gin", cmd, "datafiles/*", exit=False) assert out.endswith(nothingmsg) assert not err out, err = r.runcommand("gin", "upload", "smallfiles") assert out, "Expected output, got nothing" assert not err # Nothing to do on lock, unlock, getc, rmc on non-annexed file(s) for cmd in commands[1:]: out, err = r.runcommand("gin", cmd, "smallfiles", exit=False) assert out.endswith(nothingmsg) assert not err out, err = r.runcommand("gin", "upload", "datafiles") # unlock, commit, modify, and lock before commit r.runcommand("gin", "unlock", "datafiles") r.runcommand("gin", "annex", "sync") # r.runcommand("gin", "commit", "datafiles") r.runcommand("ls", "-l", "datafiles") # modify the files for idx in range(3): util.mkrandfile(os.path.join("datafiles", f"datafile-{idx:03}"), 2000) out, err = r.runcommand("gin", "lock", "datafiles", exit=False) assert "Locking this file would discard" in out assert err == "[error] 3 operations failed" r.runcommand("gin", "upload", ".") r.runcommand("gin", "lock", ".") r.runcommand("gin", "rmc", "datafiles") # change git repo remote address/port and test get-content failure out, err = r.runcommand("git", "remote", "-v") name, address, *_ = out.split() r.runcommand("git", "remote", "set-url", name, "#_not_a_real_path") out, err = r.runcommand("gin", "get-content", "datafiles", exit=False) assert err, "Expected error, got nothing" # count output lines with failed message outlines = out.splitlines() failmsg = "failed: authorisation failed or remote storage unavailable" nfail = len([ol for ol in outlines if failmsg in ol]) assert nfail == 5 assert err.strip().endswith("[error] 5 operations failed") print("DONE!")
def test_create_from_local(runner): r = runner # create files in root for idx in range(51): util.mkrandfile(f"root-{idx}.git", 1) for idx in range(70, 91): util.mkrandfile(f"root-{idx}.annex", 100) # Create from local directory r.runcommand( "gin", "create", "--here", r.reponame, "Test repository for create --here. Created with test script") r.runcommand("gin", "upload", ".") util.assert_status(r, status={"OK": 72}) r.repositories[r.cmdloc] = r.reponame # gin upload command should not have created an extra commit out, err = r.runcommand("gin", "git", "rev-list", "--count", "HEAD") assert int(out) == 2, f"Expected 2 commits, got {out}" # Create more root files that will remain UNTRACKED for c in "abcdef": util.mkrandfile(f"root-file-{c}.untracked") # Create some subdirectories with files for idx in "abcdef": dirname = f"subdir-{idx}" os.mkdir(dirname) r.cdrel(dirname) for jdx in range(1, 11): util.mkrandfile(f"subfile-{jdx}.annex", 200) r.cdrel("..") # Upload all the files of the first subdirectory and 2 from the second r.runcommand("gin", "upload", "subdir-a", "subdir-b/subfile-5.annex", "subdir-b/subfile-10.annex") status = {"OK": 84, "??": 54} util.assert_status(r, status=status) # can also check each directory individually subb = {"??": 8} util.assert_status(r, path="subdir-b", status=subb) subcdef = {"??": 10} for p in "cdef": util.assert_status(r, path=f"subdir-{p}", status=subcdef) # Lock some files r.runcommand("gin", "lock", "root-70.annex", "root-75.annex", "root-84.annex") # Locked files should be marked TC util.assert_status(r, status={"TC": 3}) # Lock a whole directory r.runcommand("gin", "lock", "subdir-a") util.assert_status(r, status={"TC": 13}) # Check subdirectory only util.assert_status(r, path="subdir-a", status={"TC": 10}) # Check again but from within the subdir r.cdrel("subdir-a") util.assert_status(r, status={"TC": 10}) r.cdrel("..") # Re-unlock one of the files r.runcommand("gin", "unlock", "root-84.annex") util.assert_status(r, status={"TC": 12}) # check one of the remaining unlocked files explicitly util.assert_status(r, path="root-70.annex", status={"TC": 1}) # commit the type changes r.runcommand("gin", "commit") # no TCs left util.assert_status(r, status={"TC": 0}) # There should be no NC files so far util.assert_status(r, status={"NC": 0}) # drop some files and check the counts r.runcommand("gin", "rmc", "subdir-b/subfile-5.annex") util.assert_status(r, path="subdir-b", status={"NC": 1}) r.runcommand("gin", "rmc", "subdir-b") util.assert_status(r, path="subdir-b", status={"NC": 2}) r.runcommand("gin", "remove-content", "subdir-a") util.assert_status(r, path="subdir-b", status={"NC": 2}) util.assert_status(r, path="subdir-a", status={"NC": 10}) util.assert_status(r, status={"NC": 12}) print("Done!")
def test_local_only(runner): r = runner r.runcommand("gin", "init") r.repositories[r.cmdloc] = None ngit = 15 nannex = 10 nuntracked = 5 # create files in root for idx in range(ngit): util.mkrandfile(f"root-{idx}.git", 1) for idx in range(nannex): util.mkrandfile(f"root-{idx}.annex", 100) status = util.zerostatus() status["??"] = nannex + ngit util.assert_status(r, status=status) out, err = r.runcommand("gin", "commit", "*.annex") status["??"] -= nannex status["LC"] += nannex util.assert_status(r, status=status) out, err = r.runcommand("gin", "commit", ".") status["??"] -= ngit status["LC"] += ngit util.assert_status(r, status=status) # add some untracked files for idx in range(nuntracked): util.mkrandfile(f"untracked-{idx}", 70) status["??"] += nuntracked util.assert_status(r, status=status) # Upload does nothing out, err = r.runcommand("gin", "upload", exit=False) util.assert_status(r, status=status) assert err, "Expected error, got nothing" assert err == "[error] upload failed: no remote configured" # Upload should not add any new files out, err = r.runcommand("gin", "upload", ".", exit=False) util.assert_status(r, status=status) assert err, "Expected error, got nothing" assert err == "[error] upload failed: no remote configured" # gin upload command should not have created an extra commit assert util.getrevcount(r) == 3 # modify all tracked files for idx in range(ngit): util.mkrandfile(f"root-{idx}.git", 4) for idx in range(nannex): util.mkrandfile(f"root-{idx}.annex", 2100) status["LC"] -= ngit + nannex status["MD"] += ngit + nannex util.assert_status(r, status=status) # Commit all except untracked r.runcommand("gin", "commit", "*.annex", "*.git") status["LC"] += ngit + nannex status["MD"] -= ngit + nannex util.assert_status(r, status=status) # Should have 4 commits so far assert util.getrevcount(r) == 4 # Create some subdirectories with files for idx in "abcdef": dirname = f"subdir-{idx}" os.mkdir(dirname) r.cdrel(dirname) for jdx in range(10): util.mkrandfile(f"subfile-{jdx}.annex", 1500) r.cdrel("..") status["??"] += 60 util.assert_status(r, status=status) # Commit some files r.runcommand("gin", "commit", "subdir-c", "subdir-b/subfile-5.annex", "subdir-b/subfile-9.annex", "subdir-f") status["LC"] += 22 status["??"] -= 22 util.assert_status(r, status=status) subb = util.zerostatus() subb["LC"] = 2 subb["??"] = 8 util.assert_status(r, path="subdir-b", status=subb) tenuntracked = util.zerostatus() tenuntracked["??"] = 10 for idx in "ade": util.assert_status(r, path=f"subdir-{idx}", status=tenuntracked) status["NC"] = 0 util.assert_status(r, status=status) # Remove a few file and check their status os.remove("subdir-f/subfile-1.annex") os.remove("root-10.git") shutil.rmtree("subdir-c", onerror=util.force_rm) status["RM"] += 12 status["LC"] -= 12 # root-10.git + subdir-c + subdir-f/subfile-1.annex util.assert_status(r, status=status) # Do a gin ls on a deleted file onerm = util.zerostatus() onerm["RM"] = 1 util.assert_status(r, path="root-10.git", status=onerm) # Commit deletions r.runcommand("gin", "commit", "subdir-f/subfile-1.annex", "root-10.git", "subdir-c") status["RM"] = 0 util.assert_status(r, status=status) # Add new files, remove some existing ones, check status and upload util.mkrandfile("new-annex-file", 10021) util.mkrandfile("new-git-file", 10) shutil.rmtree("subdir-f", onerror=util.force_rm) status["RM"] += 9 status["??"] += 2 status["LC"] -= 9 util.assert_status(r, status=status) print("Done!")
def test_annex_filters(runner): r = runner # Create some big data files for idx in range(3): util.mkrandfile(f"randfile{idx}", 1000) r.runcommand("gin", "upload", ".") def isannexed(fname): return util.isannexed(r, fname) # annexed files should include path to annex objects in their git blob for idx in range(3): fname = f"randfile{idx}" assert isannexed(fname) # Create markdown, python, and 'foo' file # All these are extensions that are excluded from annex in the config # Make them "large" (bigger than annex threshold) excludedfiles = ["markdown.md", "python.py", "biscuits.foo"] for fname in excludedfiles: util.mkrandfile(fname, 500) r.runcommand("gin", "upload", *excludedfiles) for fname in excludedfiles: assert not isannexed(fname) # make a really big "script" util.mkrandfile("bigscript.py", 100000) # 100 MB r.runcommand("ls", "-lh") r.runcommand("gin", "upload", "bigscript.py") assert not isannexed("bigscript.py") # clear local directory and reclone r.runcommand("gin", "annex", "uninit") r.cdrel("..") shutil.rmtree(r.reponame, onerror=util.force_rm) repopath = f"{r.username}/{r.reponame}" r.runcommand("gin", "get", repopath) r.cdrel(r.reponame) # git files should be here status = util.zerostatus() status["OK"] = 4 status["NC"] = 3 util.assert_status(r, status=status) for fname in glob("randfile*"): assert isannexed(fname) # download first rand file r.runcommand("gin", "get-content", "randfile1") status["OK"] += 1 status["NC"] -= 1 util.assert_status(r, status=status) # download everything r.runcommand("gin", "getc", ".") status["OK"] += status["NC"] status["NC"] = 0 util.assert_status(r, status=status)
def run_checks(r, mode): # create files in root for idx in range(10, 15): util.mkrandfile(f"root-{idx}.git", 5) for idx in range(20, 24): util.mkrandfile(f"root-{idx}.annex", 2000) r.runcommand("gin", "commit", "root*") def check_files(): # check if the size of each file is < 10k # git files are 5k, annexed files are 2000k # pointer files should be a few bytes out, err = r.runcommand("git", "ls-files") allfiles = out.splitlines() for fname in allfiles: out, err = r.runcommand("git", "cat-file", "-s", f":{fname}") assert not err assert int(out) < 10240 if fname.endswith(".annex"): out, err = r.runcommand("git", "cat-file", "-p", f":{fname}") assert not err assert "annex" in out assert "MD5-" in out check_files() # create files in subdirectories os.mkdir("subdir-a") r.cdrel("subdir-a") for idx in range(10, 13): util.mkrandfile(f"a-{idx}.git", 5) for idx in range(20, 23): util.mkrandfile(f"a-{idx}.annex", 2000) # commit from inside r.runcommand("gin", "commit", ".") r.cdrel("..") check_files() os.mkdir("subdir-b") r.cdrel("subdir-b") for idx in range(10, 13): util.mkrandfile(f"b-{idx}.git", 5) for idx in range(20, 23): util.mkrandfile(f"b-{idx}.annex", 2000) r.cdrel("..") # commit from outside r.runcommand("gin", "commit", "subdir-b") check_files()
def test_errors(runner): r = runner norepoerr = "[error] this command must be run from inside a gin repository" commands = [ "upload", "lock", "unlock", "get-content", "remove-content", ] # Unable to run any git commands outside a git repository for cmd in commands: # On directory out, err = r.runcommand("gin", cmd, ".", exit=False) assert err == norepoerr # On specific file out, err = r.runcommand("gin", cmd, "foobar", exit=False) assert err == norepoerr out, err = r.runcommand("gin", "download", exit=False) assert err == norepoerr r.cdrel(r.reponame) # Unable to run any command on file that does not exist out, err = r.runcommand("gin", "upload", "foobar", exit=False) assert out.endswith(" Nothing to do") for cmd in commands[1:]: out, err = r.runcommand("gin", cmd, "foobar", exit=False) assert out.endswith("No files matched foobar") assert err == "[error] 1 operation failed" # make a few annex and git files os.mkdir("smallfiles") r.cdrel("smallfiles") for idx in range(20): util.mkrandfile(f"smallfile-{idx:03}", 20) r.cdrel("..") os.mkdir("datafiles") r.cdrel("datafiles") for idx in range(5): util.mkrandfile(f"datafile-{idx:03}", 2000) r.cdrel("..") # Nothing to do on lock, unlock, getc, rmc on untracked file(s) nothingmsg = " Nothing to do" for cmd in commands[1:]: out, err = r.runcommand("gin", cmd, "smallfiles", exit=False) assert out.endswith(nothingmsg) assert not err out, err = r.runcommand("gin", cmd, "datafiles", exit=False) assert out.endswith(nothingmsg) assert not err out, err = r.runcommand("gin", cmd, "datafiles/*", exit=False) assert out.endswith(nothingmsg) assert not err out, err = r.runcommand("gin", "upload", "smallfiles") assert out, "Expected output, got nothing" assert not err # Nothing to do on lock, unlock, getc, rmc on non-annexed file(s) for cmd in commands[1:]: out, err = r.runcommand("gin", cmd, "smallfiles", exit=False) assert out.endswith(nothingmsg) assert not err out, err = r.runcommand("gin", "upload", "datafiles") # unlock, commit, modify, and lock before commit r.runcommand("gin", "unlock", "datafiles") r.runcommand("gin", "annex", "sync") # r.runcommand("gin", "commit", "datafiles") r.runcommand("ls", "-l", "datafiles") # modify the files for idx in range(3): util.mkrandfile(os.path.join("datafiles", f"datafile-{idx:03}"), 2000) out, err = r.runcommand("gin", "lock", "datafiles", exit=False) assert "Locking this file would discard" in out assert err == "[error] 3 operations failed" r.runcommand("gin", "upload", ".") r.runcommand("gin", "lock", ".") r.runcommand("gin", "rmc", "datafiles") # change git repo remote address/port and test get-content failure out, err = r.runcommand("git", "remote", "-v") name, address, *_ = out.split() badaddress = "ssh://git@bad-hostname:22" r.runcommand("git", "remote", "set-url", name, badaddress) out, err = r.runcommand("gin", "get-content", "datafiles", exit=False) assert err, "Expected error, got nothing" # count output lines with failed message outlines = out.splitlines() failmsg = "failed: authorisation failed or remote storage unavailable" nfail = len([ol for ol in outlines if failmsg in ol]) assert nfail == 5 assert err.strip().endswith("[error] 5 operations failed") # revert remote change r.runcommand("git", "remote", "set-url", name, address) # Change gin and git server address:port in config and test failures goodconfdir = r.env["GIN_CONFIG_DIR"] badconftemp = tempfile.TemporaryDirectory(prefix="badconf") badconfdir = os.path.join(badconftemp.name, "conf") r.env["GIN_CONFIG_DIR"] = badconfdir shutil.copytree(goodconfdir, badconfdir) with open(os.path.join(goodconfdir, "config.yml")) as conffile: confdata = yaml.load(conffile.read(), Loader=yaml.SafeLoader) confdata["servers"]["test"]["web"]["port"] = 1 confdata["servers"]["test"]["git"]["port"] = 1 with open(os.path.join(badconfdir, "config.yml"), "w") as conffile: conffile.write(yaml.dump(confdata)) out, err = r.runcommand("gin", "create", "ThisShouldFail", exit=False) assert err, "Expected error, got nothing" errmsg = "[error] server refused connection" assert err == errmsg # TODO: simulate not-enough-free-space # Recover good config r.env["GIN_CONFIG_DIR"] = goodconfdir # delete all keys (there might be leftovers from aborted tests) err = "" while len(err) == 0: out, err = r.runcommand("gin", "keys", "--delete", "1", exit=False) # get content should fail now out, err = r.runcommand("gin", "get-content", "datafiles", exit=False) assert err, "Expected error, got nothing" # count output lines with failed message outlines = out.splitlines() failmsg = "failed: authorisation failed or remote storage unavailable" nfail = len([ol for ol in outlines if failmsg in ol]) assert nfail == 5 assert err.strip().endswith("[error] 5 operations failed") out, err = r.runcommand("gin", "download", exit=False) assert err, "Expected error, got nothing" assert err.strip() == "[error] download failed: permission denied" out, err = r.runcommand("gin", "upload", exit=False) assert err, "Expected error, got nothing" assert err.strip() == "[error] 1 operation failed" outlines = out.splitlines() assert outlines[-1].strip() == "upload failed: permission denied" # login to add key r.login() # set bad host key and check error with open(os.path.join(goodconfdir, "known_hosts"), "r+") as hostkeyfile: goodhostkey = hostkeyfile.read() badhostkey = goodhostkey.replace("A", "B") # assumes at least one A hostkeyfile.seek(0) hostkeyfile.write(badhostkey) # TODO: Check error messages out, err = r.runcommand("gin", "download", "--content", exit=False) assert err, "Expected error, got nothing" assert err.strip() ==\ "[error] download failed: server key does not match known host key" out, err = r.runcommand("gin", "get-content", "datafiles", exit=False) assert err, "Expected error, got nothing" # count output lines with failed message outlines = out.splitlines() failmsg = "failed: authorisation failed or remote storage unavailable" nfail = len([ol for ol in outlines if failmsg in ol]) assert nfail == 5 assert err.strip().endswith("[error] 5 operations failed") out, err = r.runcommand("gin", "upload", exit=False) assert err, "Expected error, got nothing" assert err.strip() == "[error] 1 operation failed" outlines = out.splitlines() assert outlines[-1].strip() ==\ "upload failed: server key does not match known host key" # login to fix key r.login() # Creating repository that already exists r.cdrel("..") out, err = r.runcommand("gin", "create", r.reponame, exit=False) assert err == ("[error] invalid repository name or " "repository with the same name already exists") # Creating repository but local directory already exists (non-empty) anotherrepo = f"gin-test-{randint(0, 9999):04}" print(f"Creating directory '{anotherrepo}'") os.mkdir(anotherrepo) util.mkrandfile(os.path.join(anotherrepo, f"hold"), 20) out, err = r.runcommand("gin", "create", anotherrepo, exit=False) assert ("Creating repository" in out and "OK" in out), f"Failed to create {anotherrepo} on server" assert out.endswith( f"Repository download failed.\n'{anotherrepo}' already exists in the " "current directory and is not empty." ) r.repositories[r.cmdloc] = anotherrepo assert err == "[error] 1 operation failed" out, err = r.runcommand("gin", "repos") assert anotherrepo in out print("DONE!")
def test_multiple_remotes_gitanddir(runner): r = runner r.login() r.runcommand("gin", "create", "--here", r.reponame, "Multi-remote test") r.repositories[r.cmdloc] = r.reponame here = f"{r.username}@{socket.gethostname()}" # add a directory remote fsremotedir = os.path.join(r.testroot.name, "annexdata") out, err = r.runcommand("gin", "add-remote", "--create", "datastore", f"dir:{fsremotedir}") r.repositories[fsremotedir] = None assert "Default remote: origin" in out out, err = r.runcommand("gin", "git", "config", "--get", "gin.remote") assert out.strip() == "origin" # upload to initialise datastore remote r.runcommand("gin", "upload", "--to=datastore") ngit = 5 nannex = 3 contentlocs = dict() for idx in range(ngit): fname = f"gitfile{idx}" util.mkrandfile(fname, 3) for idx in range(nannex): fname = f"annexfile{idx}" util.mkrandfile(fname, 900) contentlocs[fname] = list() r.runcommand("gin", "upload", "gitfile*") assert_locations(r, contentlocs) def revcountremote(remote): n, _ = r.runcommand("git", "rev-list", "--count", f"{remote}/master", "--") return int(n) # commits are no longer pushed to all remotes assert revcountremote("origin") == 2 assert revcountremote("datastore") == 1 r.runcommand("gin", "upload", "annexfile*") for fname in contentlocs: if "annexfile" in fname: contentlocs[fname].extend(["origin", here]) assert_locations(r, contentlocs) assert revcountremote("origin") == 3 assert revcountremote("datastore") == 1 # upload annexfile0 to datastore r.runcommand("gin", "upload", "--to", "datastore", "annexfile0") contentlocs["annexfile0"].append("GIN Storage [datastore]") assert_locations(r, contentlocs) # upload everything to datastore r.runcommand("gin", "upload", "--to", "datastore") for fname, locs in contentlocs.items(): if "GIN Storage [datastore]" not in locs: contentlocs[fname].append("GIN Storage [datastore]") assert_locations(r, contentlocs) assert revcountremote("origin") == revcountremote("datastore") == 3 util.mkrandfile("another annex file", 1000) r.runcommand("gin", "upload", "--to", "datastore", "another annex file") contentlocs["another annex file"] = ["GIN Storage [datastore]", here] assert_locations(r, contentlocs) assert revcountremote("origin") == 3 assert revcountremote("datastore") == 4 # add another directory remote fsremotedir = os.path.join(r.testroot.name, "annexdata-two") out, err = r.runcommand("gin", "add-remote", "--create", "lanstore", f"dir:{fsremotedir}") r.repositories[fsremotedir] = None assert "Default remote: origin" in out for idx in range(5): fname = f"moredata{idx}" contentlocs[fname] = list() util.mkrandfile(fname, 1000) # commit only r.runcommand("gin", "commit", "moredata*") for fname in contentlocs: if "moredata" in fname: contentlocs[fname].append(here) assert_locations(r, contentlocs) # upload only to non-gin remotes r.runcommand("gin", "upload", "--to", "datastore", "--to", "lanstore", "moredata*") assert revcountremote("origin") == 3 assert revcountremote("datastore") == revcountremote("lanstore") == 5 newlocs = ["GIN Storage [datastore]", "GIN Storage [lanstore]"] for fname in contentlocs: if "moredata" in fname: contentlocs[fname].extend(newlocs) assert_locations(r, contentlocs) # change default to datastore out, err = r.runcommand("gin", "use-remote") assert out.strip() == ":: Default remote: origin" out, err = r.runcommand("gin", "remotes") outlines = out.splitlines() assert len(outlines) == 4 for line in outlines: if "origin" in line: assert "[default]" in line r.runcommand("gin", "use-remote", "datastore") out, err = r.runcommand("gin", "use-remote") assert out.strip() == ":: Default remote: datastore" dsfname = "send-me-to-the-datastore" util.mkrandfile(dsfname, 1024) r.runcommand("gin", "upload", dsfname) contentlocs[dsfname] = [here, "GIN Storage [datastore]"] assert_locations(r, contentlocs) # upload everything everywhere r.runcommand("gin", "upload", "--to", "all", ".") allremotes = [ here, "origin", "GIN Storage [datastore]", "GIN Storage [lanstore]" ] for fname in contentlocs: contentlocs[fname] = allremotes assert_locations(r, contentlocs)
def test_annex_upgrade(runner): r = runner # checks if all files with suffix ".annex" are annexed files def assert_all_annex(): for root, dirs, files in os.walk("."): if ".git" in dirs: dirs.remove(".git") # don't visit .git directory for f in files: if f.endswith("annex"): fname = os.path.join(root, f) assert util.isannexed(r, fname) # add some file, commit, upload, then upgrade os.mkdir("first batch") fbfiles = 0 for idx in range(5): fname = os.path.join("first batch", f"small-{idx}.git") util.mkrandfile(fname, 5) fbfiles += 1 for idx in range(7, 9): fname = os.path.join("first batch", f"big-{idx}.annex") util.mkrandfile(fname, 2000) fbfiles += 1 status = util.zerostatus() status["??"] += fbfiles util.assert_status(r, status=status) r.runcommand("gin", "commit", "first batch") status["LC"] += status["??"] status["??"] = 0 util.assert_status(r, status=status) r.runcommand("gin", "upload") status["OK"] += status["LC"] status["LC"] = 0 util.assert_status(r, status=status) assert_all_annex() with open(os.path.join(".git", "config")) as gitconf: assert "version = 5" in gitconf.read() r.runcommand("gin", "annex", "upgrade") r.runcommand("gin", "init") util.assert_status(r, status=status) with open(os.path.join(".git", "config")) as gitconf: conf = gitconf.read() assert "version = 7" in conf assert "addunlocked" in conf assert_all_annex() os.mkdir("second batch") sbfiles = 0 for idx in range(20, 30): fname = os.path.join("second batch", f"small-{idx}.git") util.mkrandfile(fname, 5) sbfiles += 1 for idx in range(10, 15): fname = os.path.join("second batch", f"big-{idx}.annex") util.mkrandfile(fname, 2000) sbfiles += 1 status["??"] += sbfiles util.assert_status(r, status=status) r.runcommand("gin", "commit", ".") status["LC"] += status["??"] status["??"] = 0 util.assert_status(r, status=status) r.runcommand("gin", "upload") status["OK"] += status["LC"] status["LC"] = 0 util.assert_status(r, status=status) assert_all_annex()
def run_checks(r): # create files in root for idx in range(50): util.mkrandfile(f"root-{idx}.git", 5) for idx in range(70, 90): util.mkrandfile(f"root-{idx}.annex", 2000) status = util.zerostatus() status["??"] += 70 util.assert_status(r, status=status) r.runcommand("gin", "commit", "root*") status["LC"] += 70 status["??"] -= 70 util.assert_status(r, status=status) r.runcommand("gin", "upload") status["LC"] -= 70 status["OK"] += 70 util.assert_status(r, status=status) # gin upload command should not have created an extra commit assert util.getrevcount(r) == 2 # Create more root files that will remain UNTRACKED for idx in "abcdef": util.mkrandfile(f"root-file-{idx}.untracked", 1) status["??"] += 6 util.assert_status(r, status=status) # lock all annexed files r.runcommand("gin", "lock", ".") status["TC"] += 20 status["OK"] -= 20 util.assert_status(r, status=status) # commit typechange r.runcommand("gin", "commit") status["TC"] -= 20 status["OK"] += 20 util.assert_status(r, status=status) # modify all tracked files r.runcommand("gin", "unlock", ".") status["TC"] += 20 status["OK"] -= 20 util.assert_status(r, status=status) for idx in range(50): util.mkrandfile(f"root-{idx}.git", 4) for idx in range(70, 90): util.mkrandfile(f"root-{idx}.annex", 2100) status["OK"] = 0 status["TC"] = 0 status["MD"] = 70 util.assert_status(r, status=status) r.runcommand("gin", "commit", "*.git", "*.annex") status["LC"] += status["MD"] status["MD"] = 0 util.assert_status(r, status=status) # Upload all except untracked r.runcommand("gin", "upload", "*.annex", "*.git") status["LC"] = 0 status["MD"] = 0 status["OK"] = 70 util.assert_status(r, status=status) # Should have 4 commits so far assert util.getrevcount(r) == 4 # Create some subdirectories with files for idx in "abcdef": dirname = f"subdir-{idx}" os.mkdir(dirname) r.cdrel(dirname) for jdx in range(10): util.mkrandfile(f"subfile-{jdx}.annex", 1500) r.cdrel("..") status["??"] += 60 util.assert_status(r, status=status) # Upload the files in the first subdirectory only and a couple from the # second r.runcommand("gin", "upload", "subdir-a", os.path.join("subdir-b", "subfile-5.annex"), os.path.join("subdir-b", "subfile-9.annex")) status["OK"] += 12 status["??"] -= 12 util.assert_status(r, status=status) subb = util.zerostatus() subb["OK"] = 2 subb["??"] = 8 util.assert_status(r, path="subdir-b", status=subb) tenuntracked = util.zerostatus() tenuntracked["??"] = 10 for idx in "cdef": util.assert_status(r, path=f"subdir-{idx}", status=tenuntracked) # Lock some files r.runcommand("gin", "lock", "root-70.annex", "root-75.annex", "root-84.annex") status["TC"] += 3 status["OK"] -= 3 util.assert_status(r, status=status) # Lock a whole directory r.runcommand("gin", "lock", "subdir-a") status["TC"] += 10 status["OK"] -= 10 util.assert_status(r, status=status) # Check subdirectory only tenul = util.zerostatus() tenul["TC"] = 10 util.assert_status(r, path="subdir-a", status=tenul) # Check again from within the subdir r.cdrel("subdir-a") util.assert_status(r, status=tenul) r.cdrel("..") # Revert lock on one of the files r.runcommand("gin", "unlock", "root-84.annex") status["TC"] -= 1 status["OK"] += 1 util.assert_status(r, status=status) onetc = util.zerostatus() onetc["TC"] = 1 # Check one of the remaining locked files explicitly util.assert_status(r, status=onetc, path="root-70.annex") # There should be no NC files so far status["NC"] = 0 util.assert_status(r, status=status) # Drop some files r.runcommand("gin", "rmc", os.path.join("subdir-b", "subfile-5.annex")) status["NC"] += 1 status["OK"] -= 1 util.assert_status(r, status=status) # remove content in subdir-a r.runcommand("gin", "remove-content", "subdir-a") # removing content of TypeChanged files still shows them as unlocked until # the type change is committed util.assert_status(r, status=status) suba = util.zerostatus() suba["TC"] += 10 util.assert_status(r, status=suba, path="subdir-a") subb["OK"] -= 1 subb["NC"] += 1 util.assert_status(r, status=subb, path="subdir-b") # Upload everything and then rmc it r.runcommand("gin", "upload", ".") # subdir-a goes from TC to NC (unlocked, removed content, then commit) status["TC"] -= 10 status["NC"] += 10 # modified and untracked files become OK status["OK"] += status["TC"] + status["MD"] + status["LC"] + status["??"] # everything else becomes 0 status["TC"] = status["MD"] = status["LC"] = status["??"] = 0 util.assert_status(r, status=status) r.runcommand("gin", "rmc", ".") # annex files are now NC status["NC"] += 69 status["OK"] -= 69 util.assert_status(r, status=status) # remove a few files and check their status os.remove(os.path.join("subdir-a", "subfile-1.annex")) os.remove("root-10.git") shutil.rmtree("subdir-b", onerror=util.force_rm) status["RM"] += 12 status["NC"] -= 11 status["OK"] -= 1 util.assert_status(r, status=status) r.runcommand("gin", "commit", ".") status["RM"] = 0 util.assert_status(r, status=status) # Add new files, remove some existing ones, check status and upload util.mkrandfile("new-annex-file", 10021) util.mkrandfile("new-git-file", 10) shutil.rmtree("subdir-c", onerror=util.force_rm) status["RM"] += 10 status["??"] += 2 status["NC"] -= 10 util.assert_status(r, status=status) r.runcommand("gin", "upload", ".") status["RM"] = 0 status["OK"] += status["??"] status["??"] = 0 util.assert_status(r, status=status)
def test_remote_unavailable(runner): r = runner r.login() r.runcommand("gin", "create", "--here", r.reponame, "Multi-remote + unavailable test") r.repositories[r.cmdloc] = r.reponame here = f"{r.username}@{socket.gethostname()}" # add a directory remote datastoredir = os.path.join(r.testroot.name, "annexdata") out, err = r.runcommand("gin", "add-remote", "--create", "datastore", f"dir:{datastoredir}") datastore = "GIN Storage [datastore]" r.repositories[datastoredir] = None # add another directory remote lanstoredir = os.path.join(r.testroot.name, "annexdata-two") out, err = r.runcommand("gin", "add-remote", "--create", "lanstore", f"dir:{lanstoredir}") lanstore = "GIN Storage [lanstore]" r.repositories[lanstoredir] = None assert "Default remote: origin" in out out, err = r.runcommand("gin", "git", "config", "--get", "gin.remote") assert out.strip() == "origin" ngit = 5 nannex = 4 contentlocs = dict() for idx in range(ngit): fname = f"gitfile{idx}" util.mkrandfile(fname, 3) for idx in range(nannex): fname = f"annexfile{idx}" util.mkrandfile(fname, 900) contentlocs[fname] = list() r.runcommand("gin", "upload", "gitfile*") assert_locations(r, contentlocs) def revcountremote(remote): n, _ = r.runcommand("git", "rev-list", "--count", f"{remote}/master", "--") return int(n) # commits are no longer pushed to all remotes assert revcountremote("origin") == 2 r.runcommand("gin", "upload", "--to=all", "annexfile*") for fname in contentlocs: if "annexfile" in fname: contentlocs[fname].extend(["origin", here, datastore, lanstore]) assert_locations(r, contentlocs) assert revcountremote("origin") == revcountremote("datastore") == 3 # make lanstore inaccessible by renaming the path temporarily lanstoreunavaildir = f"{lanstoredir}-moved" print(f"Renaming {lanstoredir} to {lanstoreunavaildir}") os.rename(lanstoredir, lanstoreunavaildir) # add a few more annex files for idx in range(2): fname = f"newannexfile{idx}" util.mkrandfile(fname, 900) contentlocs[fname] = list() r.runcommand("gin", "commit", "newannexfile*") for fname in contentlocs: if fname.startswith("newannexfile"): contentlocs[fname].append(here) assert_locations(r, contentlocs) r.runcommand("gin", "upload", "--to=all", exit=False) for fname in contentlocs: if fname.startswith("newannexfile"): contentlocs[fname].extend(("origin", datastore)) assert_locations(r, contentlocs) print(f"Renaming {lanstoreunavaildir} to {lanstoredir}") os.rename(lanstoreunavaildir, lanstoredir) assert_locations(r, contentlocs) # upload again r.runcommand("gin", "upload", "--to=all", exit=False) for fname in contentlocs: if fname.startswith("newannexfile"): contentlocs[fname].append(lanstore) assert_locations(r, contentlocs)
def test_config_path(runner): # upload a few files just to make sure it's all good util.mkrandfile("file21", 10000) util.mkrandfile("file22", 10) runner.runcommand("gin", "upload", "file21", "file22")
def test_workflow(runner): r = runner # create files in root for idx in range(5): util.mkrandfile(f"root-{idx}.git", 5) for idx in range(7): util.mkrandfile(f"root-{idx}.annex", 2000) # compute hashes orighashes = hashfiles() # upload r.runcommand("gin", "upload", ".") # cleanup local repository r.runcommand("gin", "annex", "uninit") r.cdrel("..") shutil.rmtree(r.reponame, onerror=util.force_rm) # redownload and check the hashes repopath = f"{r.username}/{r.reponame}" r.runcommand("gin", "get", repopath) r.cdrel(r.reponame) # should have 5 OK files and 7 NC files status = util.zerostatus() status["OK"] = 5 status["NC"] = 7 util.assert_status(r, status=status) curhashes = hashfiles() for k in curhashes: orig = orighashes[k] cur = curhashes[k] if k.endswith("git"): assert orig == cur assert not util.isannexed(r, k) elif k.endswith("annex"): assert orig != cur assert util.isannexed(r, k) else: assert False, f"Unexpected file {k}" r.runcommand("gin", "get-content", "root-1.annex") status["OK"] += 1 status["NC"] -= 1 util.assert_status(r, status=status) curhashes = hashfiles() assert orighashes["root-1.annex"] == curhashes["root-1.annex"] assert util.isannexed(r, "root-1.annex") # download everything r.runcommand("gin", "getc", ".") # everything should be OK status["OK"] += status["NC"] status["NC"] = 0 util.assert_status(r, status=status) # all hashes should match original now curhashes = hashfiles() assert curhashes == orighashes