def test_add_directory_remote(runner):
r = runner
r.runcommand("gin", "init")
r.repositories[r.cmdloc] = None
ngit = 3
nannex = 2
# create files in root
for idx in range(ngit):
util.mkrandfile(f"root-{idx}.git", 3)
for idx in range(nannex):
util.mkrandfile(f"root-{idx}.annex", 200)
status = util.zerostatus()
status["??"] = ngit + nannex
util.assert_status(r, status=status)
r.runcommand("gin", "commit", ".")
status["LC"] += ngit + nannex
status["??"] -= ngit + nannex
util.assert_status(r, status=status)
fsremotedir = os.path.join(r.testroot.name, "annexdata")
r.runcommand("gin", "add-remote", "--create", "lanbackup",
f"dir:{fsremotedir}")
r.runcommand("gin", "git", "remote", "-v")
r.repositories[fsremotedir] = None
r.runcommand("gin", "upload")
status["OK"] += ngit + nannex
status["LC"] -= ngit + nannex
util.assert_status(r, status=status)
def test_annex_filters(runner):
r = runner
N = 4
# Create some big data files
for idx in range(N):
util.mkrandfile(f"randfile{idx}", 200000) # 10x 200 MB files
r.runcommand("gin", "upload", ".")
# files should be links
for idx in range(N):
util.isannexed(r, f"randfile{idx}")
status = util.zerostatus()
status["OK"] = N
util.assert_status(r, status=status)
r.runcommand("gin", "remove-content", ".")
status["OK"] = 0
status["NC"] = N
util.assert_status(r, status=status)
# TODO: Don't check on Windows
# locking annexed files should turn them into symlinks
r.runcommand("gin", "lock", ".")
for idx in range(N):
assert os.path.islink(f"randfile{idx}")
def test_rm_commit_offline(runner):
r = runner
# create files in root
for idx in range(6):
util.mkrandfile(f"root-{idx}.git", 5)
util.mkrandfile(f"root-{idx}.annex", 2000)
status = util.zerostatus()
status["??"] += 12
util.assert_status(r, status=status)
r.runcommand("gin", "commit", "root*")
status["LC"] += 12
status["??"] -= 12
util.assert_status(r, status=status)
r.runcommand("gin", "upload")
status["LC"] -= 12
status["OK"] += 12
util.assert_status(r, status=status)
for idx in range(2, 4):
os.remove(f"root-{idx}.git")
os.remove(f"root-{idx}.annex")
status["OK"] -= 4
status["RM"] += 4
util.assert_status(r, status=status)
r.runcommand("gin", "upload", ".")
status["RM"] = 0
util.assert_status(r, status=status)
def test_create_remote_on_add(runner):
r = runner
r.runcommand("gin", "init")
r.repositories[r.cmdloc] = None
ngit = 3
nannex = 2
# create files in root
for idx in range(ngit):
util.mkrandfile(f"root-{idx}.git", 3)
for idx in range(nannex):
util.mkrandfile(f"root-{idx}.annex", 200)
status = util.zerostatus()
status["??"] = nannex + ngit
util.assert_status(r, status=status)
r.runcommand("gin", "commit", ".")
status["LC"] = ngit + nannex
status["??"] = 0
util.assert_status(r, status=status)
r.login()
repopath = f"{r.username}/{r.reponame}"
r.runcommand("gin", "add-remote", "--create", "origin", f"test:{repopath}")
r.runcommand("gin", "upload")
status["OK"] += status["LC"]
status["LC"] = 0
util.assert_status(r, status=status)
def test_create_remote_prompt(runner):
r = runner
r.runcommand("gin", "init")
r.repositories[r.cmdloc] = None
ngit = 3
nannex = 2
# create files in root
for idx in range(ngit):
util.mkrandfile(f"root-{idx}.git", 3)
for idx in range(nannex):
util.mkrandfile(f"root-{idx}.annex", 200)
status = util.zerostatus()
status["??"] = nannex + ngit
util.assert_status(r, status=status)
r.runcommand("gin", "commit", ".")
status["LC"] = ngit + nannex
status["??"] = 0
util.assert_status(r, status=status)
r.login()
repopath = f"{r.username}/{r.reponame}"
out, err = r.runcommand("gin",
"add-remote",
"origin",
f"test:{repopath}",
inp="abort")
assert not err, f"Expected empty error, got\n{err}"
assert out.endswith("aborted")
out, err = r.runcommand("git", "remote", "-v")
assert not out, f"Expected empty output, got\n{out}"
assert not err, f"Expected empty error, got\n{err}"
out, err = r.runcommand("gin",
"add-remote",
"origin",
f"test:{repopath}",
inp="add anyway")
out, err = r.runcommand("git", "remote", "-v")
assert len(out.splitlines()) == 2, "Unexpected output"
assert not err, f"Expected empty error, got\n{err}"
out, err = r.runcommand("gin", "upload", exit=False)
assert err, "Expected error, got nothing"
r.runcommand("git", "remote", "rm", "origin")
out, err = r.runcommand("gin",
"add-remote",
"origin",
f"test:{repopath}",
inp="create")
out, err = r.runcommand("gin", "upload")
status["OK"] += ngit + nannex
status["LC"] -= ngit + nannex
util.assert_status(r, status=status)
def test_add_gin_remote(runner):
r = runner
r.runcommand("gin", "init")
r.repositories[r.cmdloc] = None
ngit = 3
nannex = 2
# create files in root
for idx in range(ngit):
util.mkrandfile(f"root-{idx}.git", 3)
for idx in range(nannex):
util.mkrandfile(f"root-{idx}.annex", 200)
status = util.zerostatus()
status["??"] = nannex + ngit
util.assert_status(r, status=status)
r.runcommand("gin", "commit", ".")
status["LC"] = ngit + nannex
status["??"] = 0
util.assert_status(r, status=status)
r.login()
r.runcommand("gin", "create", "--no-clone", r.reponame,
"Test repository for add remote")
r.repositories[r.cmdloc] = r.reponame
repopath = f"{r.username}/{r.reponame}"
r.runcommand("gin", "add-remote", "origin", f"test:{repopath}")
r.runcommand("gin", "upload")
status["OK"] += ngit + nannex
status["LC"] -= ngit + nannex
util.assert_status(r, status=status)
# remove remote, add it with a different name (not origin) and see if it
# works
util.mkrandfile(f"final-file.annex", 500)
r.runcommand("gin", "git", "remote", "rm", "origin")
r.runcommand("gin", "git", "config", "--unset", "gin.remote")
r.runcommand("gin", "add-remote", "notorigin", f"test:{repopath}")
r.runcommand("gin", "upload", "final-file.annex")
status["OK"] += 1
util.assert_status(r, status=status)
def test_annex_filters(runner):
r = runner
# Create some big data files
for idx in range(3):
util.mkrandfile(f"randfile{idx}", 1000)
r.runcommand("gin", "upload", ".")
def isannexed(fname):
return util.isannexed(r, fname)
# annexed files should include path to annex objects in their git blob
for idx in range(3):
fname = f"randfile{idx}"
assert isannexed(fname)
# Create markdown, python, and 'foo' file
# All these are extensions that are excluded from annex in the config
# Make them "large" (bigger than annex threshold)
excludedfiles = ["markdown.md", "python.py", "biscuits.foo"]
for fname in excludedfiles:
util.mkrandfile(fname, 500)
r.runcommand("gin", "upload", *excludedfiles)
for fname in excludedfiles:
assert not isannexed(fname)
# make a really big "script"
util.mkrandfile("bigscript.py", 100000) # 100 MB
r.runcommand("ls", "-lh")
r.runcommand("gin", "upload", "bigscript.py")
assert not isannexed("bigscript.py")
# clear local directory and reclone
r.runcommand("gin", "annex", "uninit")
r.cdrel("..")
shutil.rmtree(r.reponame, onerror=util.force_rm)
repopath = f"{r.username}/{r.reponame}"
r.runcommand("gin", "get", repopath)
r.cdrel(r.reponame)
# git files should be here
status = util.zerostatus()
status["OK"] = 4
status["NC"] = 3
util.assert_status(r, status=status)
for fname in glob("randfile*"):
assert isannexed(fname)
# download first rand file
r.runcommand("gin", "get-content", "randfile1")
status["OK"] += 1
status["NC"] -= 1
util.assert_status(r, status=status)
# download everything
r.runcommand("gin", "getc", ".")
status["OK"] += status["NC"]
status["NC"] = 0
util.assert_status(r, status=status)
def test_annex_upgrade(runner):
r = runner
# checks if all files with suffix ".annex" are annexed files
def assert_all_annex():
for root, dirs, files in os.walk("."):
if ".git" in dirs:
dirs.remove(".git") # don't visit .git directory
for f in files:
if f.endswith("annex"):
fname = os.path.join(root, f)
assert util.isannexed(r, fname)
# add some file, commit, upload, then upgrade
os.mkdir("first batch")
fbfiles = 0
for idx in range(5):
fname = os.path.join("first batch", f"small-{idx}.git")
util.mkrandfile(fname, 5)
fbfiles += 1
for idx in range(7, 9):
fname = os.path.join("first batch", f"big-{idx}.annex")
util.mkrandfile(fname, 2000)
fbfiles += 1
status = util.zerostatus()
status["??"] += fbfiles
util.assert_status(r, status=status)
r.runcommand("gin", "commit", "first batch")
status["LC"] += status["??"]
status["??"] = 0
util.assert_status(r, status=status)
r.runcommand("gin", "upload")
status["OK"] += status["LC"]
status["LC"] = 0
util.assert_status(r, status=status)
assert_all_annex()
with open(os.path.join(".git", "config")) as gitconf:
assert "version = 5" in gitconf.read()
r.runcommand("gin", "annex", "upgrade")
r.runcommand("gin", "init")
util.assert_status(r, status=status)
with open(os.path.join(".git", "config")) as gitconf:
conf = gitconf.read()
assert "version = 7" in conf
assert "addunlocked" in conf
assert_all_annex()
os.mkdir("second batch")
sbfiles = 0
for idx in range(20, 30):
fname = os.path.join("second batch", f"small-{idx}.git")
util.mkrandfile(fname, 5)
sbfiles += 1
for idx in range(10, 15):
fname = os.path.join("second batch", f"big-{idx}.annex")
util.mkrandfile(fname, 2000)
sbfiles += 1
status["??"] += sbfiles
util.assert_status(r, status=status)
r.runcommand("gin", "commit", ".")
status["LC"] += status["??"]
status["??"] = 0
util.assert_status(r, status=status)
r.runcommand("gin", "upload")
status["OK"] += status["LC"]
status["LC"] = 0
util.assert_status(r, status=status)
assert_all_annex()
def run_checks(r):
# create files in root
for idx in range(50):
util.mkrandfile(f"root-{idx}.git", 5)
for idx in range(70, 90):
util.mkrandfile(f"root-{idx}.annex", 2000)
status = util.zerostatus()
status["??"] += 70
util.assert_status(r, status=status)
r.runcommand("gin", "commit", "root*")
status["LC"] += 70
status["??"] -= 70
util.assert_status(r, status=status)
r.runcommand("gin", "upload")
status["LC"] -= 70
status["OK"] += 70
util.assert_status(r, status=status)
# gin upload command should not have created an extra commit
assert util.getrevcount(r) == 2
# Create more root files that will remain UNTRACKED
for idx in "abcdef":
util.mkrandfile(f"root-file-{idx}.untracked", 1)
status["??"] += 6
util.assert_status(r, status=status)
# lock all annexed files
r.runcommand("gin", "lock", ".")
status["TC"] += 20
status["OK"] -= 20
util.assert_status(r, status=status)
# commit typechange
r.runcommand("gin", "commit")
status["TC"] -= 20
status["OK"] += 20
util.assert_status(r, status=status)
# modify all tracked files
r.runcommand("gin", "unlock", ".")
status["TC"] += 20
status["OK"] -= 20
util.assert_status(r, status=status)
for idx in range(50):
util.mkrandfile(f"root-{idx}.git", 4)
for idx in range(70, 90):
util.mkrandfile(f"root-{idx}.annex", 2100)
status["OK"] = 0
status["TC"] = 0
status["MD"] = 70
util.assert_status(r, status=status)
r.runcommand("gin", "commit", "*.git", "*.annex")
status["LC"] += status["MD"]
status["MD"] = 0
util.assert_status(r, status=status)
# Upload all except untracked
r.runcommand("gin", "upload", "*.annex", "*.git")
status["LC"] = 0
status["MD"] = 0
status["OK"] = 70
util.assert_status(r, status=status)
# Should have 4 commits so far
assert util.getrevcount(r) == 4
# Create some subdirectories with files
for idx in "abcdef":
dirname = f"subdir-{idx}"
os.mkdir(dirname)
r.cdrel(dirname)
for jdx in range(10):
util.mkrandfile(f"subfile-{jdx}.annex", 1500)
r.cdrel("..")
status["??"] += 60
util.assert_status(r, status=status)
# Upload the files in the first subdirectory only and a couple from the
# second
r.runcommand("gin", "upload", "subdir-a",
os.path.join("subdir-b", "subfile-5.annex"),
os.path.join("subdir-b", "subfile-9.annex"))
status["OK"] += 12
status["??"] -= 12
util.assert_status(r, status=status)
subb = util.zerostatus()
subb["OK"] = 2
subb["??"] = 8
util.assert_status(r, path="subdir-b", status=subb)
tenuntracked = util.zerostatus()
tenuntracked["??"] = 10
for idx in "cdef":
util.assert_status(r, path=f"subdir-{idx}", status=tenuntracked)
# Lock some files
r.runcommand("gin", "lock", "root-70.annex", "root-75.annex",
"root-84.annex")
status["TC"] += 3
status["OK"] -= 3
util.assert_status(r, status=status)
# Lock a whole directory
r.runcommand("gin", "lock", "subdir-a")
status["TC"] += 10
status["OK"] -= 10
util.assert_status(r, status=status)
# Check subdirectory only
tenul = util.zerostatus()
tenul["TC"] = 10
util.assert_status(r, path="subdir-a", status=tenul)
# Check again from within the subdir
r.cdrel("subdir-a")
util.assert_status(r, status=tenul)
r.cdrel("..")
# Revert lock on one of the files
r.runcommand("gin", "unlock", "root-84.annex")
status["TC"] -= 1
status["OK"] += 1
util.assert_status(r, status=status)
onetc = util.zerostatus()
onetc["TC"] = 1
# Check one of the remaining locked files explicitly
util.assert_status(r, status=onetc, path="root-70.annex")
# There should be no NC files so far
status["NC"] = 0
util.assert_status(r, status=status)
# Drop some files
r.runcommand("gin", "rmc", os.path.join("subdir-b", "subfile-5.annex"))
status["NC"] += 1
status["OK"] -= 1
util.assert_status(r, status=status)
# remove content in subdir-a
r.runcommand("gin", "remove-content", "subdir-a")
# removing content of TypeChanged files still shows them as unlocked until
# the type change is committed
util.assert_status(r, status=status)
suba = util.zerostatus()
suba["TC"] += 10
util.assert_status(r, status=suba, path="subdir-a")
subb["OK"] -= 1
subb["NC"] += 1
util.assert_status(r, status=subb, path="subdir-b")
# Upload everything and then rmc it
r.runcommand("gin", "upload", ".")
# subdir-a goes from TC to NC (unlocked, removed content, then commit)
status["TC"] -= 10
status["NC"] += 10
# modified and untracked files become OK
status["OK"] += status["TC"] + status["MD"] + status["LC"] + status["??"]
# everything else becomes 0
status["TC"] = status["MD"] = status["LC"] = status["??"] = 0
util.assert_status(r, status=status)
r.runcommand("gin", "rmc", ".")
# annex files are now NC
status["NC"] += 69
status["OK"] -= 69
util.assert_status(r, status=status)
# remove a few files and check their status
os.remove(os.path.join("subdir-a", "subfile-1.annex"))
os.remove("root-10.git")
shutil.rmtree("subdir-b", onerror=util.force_rm)
status["RM"] += 12
status["NC"] -= 11
status["OK"] -= 1
util.assert_status(r, status=status)
r.runcommand("gin", "commit", ".")
status["RM"] = 0
util.assert_status(r, status=status)
# Add new files, remove some existing ones, check status and upload
util.mkrandfile("new-annex-file", 10021)
util.mkrandfile("new-git-file", 10)
shutil.rmtree("subdir-c", onerror=util.force_rm)
status["RM"] += 10
status["??"] += 2
status["NC"] -= 10
util.assert_status(r, status=status)
r.runcommand("gin", "upload", ".")
status["RM"] = 0
status["OK"] += status["??"]
status["??"] = 0
util.assert_status(r, status=status)
def test_local_only(runner):
r = runner
r.runcommand("gin", "init")
r.repositories[r.cmdloc] = None
ngit = 15
nannex = 10
nuntracked = 5
# create files in root
for idx in range(ngit):
util.mkrandfile(f"root-{idx}.git", 1)
for idx in range(nannex):
util.mkrandfile(f"root-{idx}.annex", 100)
status = util.zerostatus()
status["??"] = nannex + ngit
util.assert_status(r, status=status)
out, err = r.runcommand("gin", "commit", "*.annex")
status["??"] -= nannex
status["LC"] += nannex
util.assert_status(r, status=status)
out, err = r.runcommand("gin", "commit", ".")
status["??"] -= ngit
status["LC"] += ngit
util.assert_status(r, status=status)
# add some untracked files
for idx in range(nuntracked):
util.mkrandfile(f"untracked-{idx}", 70)
status["??"] += nuntracked
util.assert_status(r, status=status)
# Upload does nothing
out, err = r.runcommand("gin", "upload", exit=False)
util.assert_status(r, status=status)
assert err, "Expected error, got nothing"
assert err == "[error] upload failed: no remote configured"
# Upload should not add any new files
out, err = r.runcommand("gin", "upload", ".", exit=False)
util.assert_status(r, status=status)
assert err, "Expected error, got nothing"
assert err == "[error] upload failed: no remote configured"
# gin upload command should not have created an extra commit
assert util.getrevcount(r) == 3
# modify all tracked files
for idx in range(ngit):
util.mkrandfile(f"root-{idx}.git", 4)
for idx in range(nannex):
util.mkrandfile(f"root-{idx}.annex", 2100)
status["LC"] -= ngit + nannex
status["MD"] += ngit + nannex
util.assert_status(r, status=status)
# Commit all except untracked
r.runcommand("gin", "commit", "*.annex", "*.git")
status["LC"] += ngit + nannex
status["MD"] -= ngit + nannex
util.assert_status(r, status=status)
# Should have 4 commits so far
assert util.getrevcount(r) == 4
# Create some subdirectories with files
for idx in "abcdef":
dirname = f"subdir-{idx}"
os.mkdir(dirname)
r.cdrel(dirname)
for jdx in range(10):
util.mkrandfile(f"subfile-{jdx}.annex", 1500)
r.cdrel("..")
status["??"] += 60
util.assert_status(r, status=status)
# Commit some files
r.runcommand("gin", "commit", "subdir-c", "subdir-b/subfile-5.annex",
"subdir-b/subfile-9.annex", "subdir-f")
status["LC"] += 22
status["??"] -= 22
util.assert_status(r, status=status)
subb = util.zerostatus()
subb["LC"] = 2
subb["??"] = 8
util.assert_status(r, path="subdir-b", status=subb)
tenuntracked = util.zerostatus()
tenuntracked["??"] = 10
for idx in "ade":
util.assert_status(r, path=f"subdir-{idx}", status=tenuntracked)
status["NC"] = 0
util.assert_status(r, status=status)
# Remove a few file and check their status
os.remove("subdir-f/subfile-1.annex")
os.remove("root-10.git")
shutil.rmtree("subdir-c", onerror=util.force_rm)
status["RM"] += 12
status["LC"] -= 12 # root-10.git + subdir-c + subdir-f/subfile-1.annex
util.assert_status(r, status=status)
# Do a gin ls on a deleted file
onerm = util.zerostatus()
onerm["RM"] = 1
util.assert_status(r, path="root-10.git", status=onerm)
# Commit deletions
r.runcommand("gin", "commit", "subdir-f/subfile-1.annex", "root-10.git",
"subdir-c")
status["RM"] = 0
util.assert_status(r, status=status)
# Add new files, remove some existing ones, check status and upload
util.mkrandfile("new-annex-file", 10021)
util.mkrandfile("new-git-file", 10)
shutil.rmtree("subdir-f", onerror=util.force_rm)
status["RM"] += 9
status["??"] += 2
status["LC"] -= 9
util.assert_status(r, status=status)
print("Done!")
def test_workflow(runner):
r = runner
# create files in root
for idx in range(5):
util.mkrandfile(f"root-{idx}.git", 5)
for idx in range(7):
util.mkrandfile(f"root-{idx}.annex", 2000)
# compute hashes
orighashes = hashfiles()
# upload
r.runcommand("gin", "upload", ".")
# cleanup local repository
r.runcommand("gin", "annex", "uninit")
r.cdrel("..")
shutil.rmtree(r.reponame, onerror=util.force_rm)
# redownload and check the hashes
repopath = f"{r.username}/{r.reponame}"
r.runcommand("gin", "get", repopath)
r.cdrel(r.reponame)
# should have 5 OK files and 7 NC files
status = util.zerostatus()
status["OK"] = 5
status["NC"] = 7
util.assert_status(r, status=status)
curhashes = hashfiles()
for k in curhashes:
orig = orighashes[k]
cur = curhashes[k]
if k.endswith("git"):
assert orig == cur
assert not util.isannexed(r, k)
elif k.endswith("annex"):
assert orig != cur
assert util.isannexed(r, k)
else:
assert False, f"Unexpected file {k}"
r.runcommand("gin", "get-content", "root-1.annex")
status["OK"] += 1
status["NC"] -= 1
util.assert_status(r, status=status)
curhashes = hashfiles()
assert orighashes["root-1.annex"] == curhashes["root-1.annex"]
assert util.isannexed(r, "root-1.annex")
# download everything
r.runcommand("gin", "getc", ".")
# everything should be OK
status["OK"] += status["NC"]
status["NC"] = 0
util.assert_status(r, status=status)
# all hashes should match original now
curhashes = hashfiles()
assert curhashes == orighashes