def test_rsync_mode():
"""Check that rsync mode is faster than default mode."""
mkdir("work")
mkdir("work2")
GitRepository.create("git")
for _ in range(1000):
name = str(uuid.uuid1(clock_seq=int(1000 * time.time())))
touch(os.path.join("git", name + ".py"))
touch(os.path.join("git", name + ".pyc"))
touch(os.path.join("git", name + ".o"))
touch(os.path.join("git", name + ".ali"))
with open("git/.gitignore", "w") as fd:
fd.write("*.pyc\n")
fd.write("*.o\n")
fd.write("*.ali\n")
m = CheckoutManager(name="myrepo", working_dir="work")
start = time.time()
m.update(vcs="external", url=os.path.abspath("git"))
total_sync_tree = time.time() - start
os.environ["E3_ENABLE_FEATURE"] = "use-rsync"
m = CheckoutManager(name="myrepo", working_dir="work2")
start = time.time()
m.update(vcs="external", url=os.path.abspath("git"))
total_rsync = time.time() - start
if which("rsync"):
assert total_rsync * 4 < total_sync_tree
def update_external(self, url, revision):
"""Update working dir using a local directory.
:param url: path to the repository
:type url: str
:param revision: ignored
:type revision: None
"""
# Expand env variables and ~
url = os.path.expandvars(os.path.expanduser(url))
if os.path.isdir(self.working_dir):
old_commit = get_filetree_state(self.working_dir)
else:
old_commit = ""
ignore_list = []
if os.path.isdir(os.path.join(url, ".git")):
# It seems that this is a git repository. Get the list of files to
# ignore
try:
g = GitRepository(working_tree=url)
ignore_list = g.git_cmd(
[
"ls-files",
"-o",
"--ignored",
"--exclude-standard",
"--directory",
],
output=PIPE,
).out
ignore_list = [
"/%s" % l.strip().rstrip("/")
for l in ignore_list.splitlines()
]
logger.debug("Ignore in external: %s", ignore_list)
except Exception:
# don't crash on exception
pass
sync_tree(
url,
self.working_dir,
preserve_timestamps=False,
delete_ignore=True,
ignore=list(VCS_IGNORE_LIST) + ignore_list,
)
new_commit = get_filetree_state(self.working_dir)
if new_commit == old_commit:
return ReturnValue.unchanged, old_commit, new_commit
else:
return ReturnValue.success, old_commit, new_commit
def run(self, args):
sandbox = SandBox(root_dir=args.sandbox)
sandbox.create_dirs()
if args.spec_git_url:
g = GitRepository(sandbox.specs_dir)
if e3.log.default_output_stream is not None:
g.log_stream = e3.log.default_output_stream
g.init()
g.update(args.spec_git_url, args.spec_git_branch, force=True)
sandbox.dump_configuration()
sandbox.write_scripts()
def do_checkout(self):
"""Get sources from vcs to sandbox vcs_dir."""
repo_name = self.data.repo_name
repo_url = self.data.repo_data['url']
repo_revision = self.data.repo_data['revision']
repo_vcs = self.data.repo_data['vcs']
if repo_vcs != 'git':
logger.error('%s vcs type not supported', repo_vcs)
self.status = STATUS.failure
return
repo_dir = os.path.join(self.sandbox.vcs_dir, repo_name)
g = GitRepository(repo_dir)
if e3.log.default_output_stream is not None:
g.log_stream = e3.log.default_output_stream
g.init()
g.update(repo_url, repo_revision, force=True)
self.status = STATUS.success
def do_checkout(self) -> None:
"""Get sources from vcs to sandbox vcs_dir."""
repo_name = self.data.repo_name
repo_url = self.data.repo_data["url"]
repo_revision = self.data.repo_data["revision"]
repo_vcs = self.data.repo_data["vcs"]
if repo_vcs != "git":
logger.error("%s vcs type not supported", repo_vcs)
self.__status = STATUS.failure
return
assert self.sandbox.vcs_dir is not None
repo_dir = os.path.join(self.sandbox.vcs_dir, repo_name)
g = GitRepository(repo_dir)
if e3.log.default_output_stream is not None:
g.log_stream = e3.log.default_output_stream
g.init()
g.update(repo_url, repo_revision, force=True)
self.__status = STATUS.success
def run(self, args):
sandbox = SandBox()
sandbox.root_dir = args.sandbox
sandbox.create_dirs()
if args.spec_git_url:
mkdir(sandbox.spec_dir)
g = GitRepository(sandbox.spec_dir)
if e3.log.default_output_stream is not None:
g.log_stream = e3.log.default_output_stream
g.init()
g.update(args.spec_git_url, args.spec_git_branch, force=True)
sandbox.dump_configuration()
sandbox.write_scripts()
def run(self, args):
sandbox = SandBox()
sandbox.root_dir = args.sandbox
if args.specs_dir:
sandbox.specs_dir = args.specs_dir
if args.create_sandbox:
sandbox.create_dirs()
if args.create_sandbox and args.spec_git_url:
mkdir(sandbox.specs_dir)
g = GitRepository(sandbox.specs_dir)
if e3.log.default_output_stream is not None:
g.log_stream = e3.log.default_output_stream
g.init()
g.update(args.spec_git_url, args.spec_git_branch, force=True)
sandbox.dump_configuration()
sandbox.write_scripts()
asr = AnodSpecRepository(sandbox.specs_dir)
check_api_version(asr.api_version)
# Load plan content if needed
if args.plan:
if not os.path.isfile(args.plan):
raise SandBoxError("plan file %s does not exist" % args.plan,
origin="SandBoxExec.run")
with open(args.plan, "r") as plan_fd:
plan_content = ["def main_entry_point():"]
plan_content += [
" %s" % line for line in plan_fd.read().splitlines()
]
plan_content = "\n".join(plan_content)
env = BaseEnv()
cm = PlanContext(server=env)
store = None
resolver = getattr(
AnodContext,
str(args.resolver),
AnodContext.always_create_source_resolver,
)
logger.debug("Using resolver %s", resolver.__name__)
# Declare available actions and their signature
def anod_action(module,
build=None,
host=None,
target=None,
qualifier=None):
pass # all: no cover
for a in ("anod_install", "anod_build", "anod_test"):
cm.register_action(a, anod_action)
# Load the plan and execute
plan = Plan(data={})
plan.load_chunk(plan_content)
actions = cm.execute(plan, "main_entry_point")
ac = AnodContext(asr, default_env=env)
for action in actions:
ac.add_anod_action(
action.module,
action,
action.action.replace("anod_", "", 1),
action.qualifier,
)
# Check if machine plan is locally schedulable
action_list = ac.schedule(resolver)
e = ElectrolytJobFactory(sandbox, asr, store, dry_run=args.dry_run)
e.run(action_list)
def git_specs_dir(git, tmpdir):
"""Create a git repo to check out specs from it."""
del git # skip or fail when git is not installed
# Create a e3-specs git repository for all tests in this module
specs_dir = str(tmpdir.mkdir("e3-specs-git-repo"))
try:
specs_source_dir = os.path.join(os.path.dirname(__file__), "specs")
for spec_file in os.listdir(specs_source_dir):
if os.path.isfile(os.path.join(specs_source_dir, spec_file)):
e3.fs.cp(
os.path.join(specs_source_dir, spec_file),
os.path.join(specs_dir, spec_file),
)
# Put prolog file
create_prolog(specs_dir)
if os.path.isdir(os.path.join(specs_dir, ".git")):
return
g = GitRepository(specs_dir)
g.init()
g.git_cmd(["config", "user.email", '"*****@*****.**"'])
g.git_cmd(["config", "user.name", '"test"'])
g.git_cmd(["add", "-A"])
g.git_cmd(["commit", "-m", "'add all'"])
yield "file://%s" % specs_dir.replace("\\", "/")
finally:
e3.fs.rm(specs_dir, True)
def test_git_repo():
working_tree = unixpath(os.path.join(os.getcwd(), 'working_tree'))
working_tree2 = os.path.join(os.getcwd(), 'working_tree2')
repo = GitRepository(working_tree)
repo.init()
os.chdir(working_tree)
new_file = unixpath(os.path.join(working_tree, 'new.txt'))
echo_to_file(new_file, 'new\n')
repo.git_cmd(['add', 'new.txt'])
repo.git_cmd(['config', 'user.email', '*****@*****.**'])
repo.git_cmd(['config', 'user.name', 'e3 core'])
repo.git_cmd(['commit', '-m', 'new file'])
repo.git_cmd(['tag', '-a', '-m', 'new tag', '20.0855369232'])
with open('log.txt', 'w') as f:
repo.write_log(f)
with open('log.txt', 'r') as f:
commits = list(repo.parse_log(f))
assert 'new file' in commits[0]['message']
assert commits[0]['email'] == '*****@*****.**'
new_sha = commits[0]['sha']
assert '20.0855369232' in repo.describe()
assert new_sha == repo.rev_parse()
with pytest.raises(GitError) as err:
repo.describe('g')
assert 'describe --always g' in str(err)
echo_to_file(new_file, 'new line\n', append=True)
with open('commit1.diff', 'wb') as f:
repo.write_local_diff(f)
with open('commit1.diff', 'rb') as f:
assert b'+new line' in f.read()
echo_to_file(new_file, 10000 * '*')
repo.git_cmd(['commit', '-a', '-m', 'file update'])
with open('log2.txt', 'w') as f:
repo.write_log(f)
with open('log2.txt', 'r') as f:
commits = list(repo.parse_log(f, max_diff_size=1000))
# assert b'diff too long' not in commits[1]['diff']
assert 'file update' in commits[0]['message']
assert 'diff too long' in commits[0]['diff']
assert 'new file' in commits[1]['message']
assert commits[1]['sha'] != commits[0]['sha']
assert commits[1]['diff'] != commits[0]['diff']
repo2 = GitRepository(working_tree2)
repo2.init(url=working_tree, remote='tree1')
try:
repo2.update(url=working_tree, refspec='master')
except GitError:
if sys.platform == 'win32':
# some git versions on windows do not support that well
# ignore this test for now???
pass
else:
repo2.rev_parse() == repo.rev_parse()
def test_svn_checkout(self, compute_changelog, e3_feature):
os.environ["GIT_AUTHOR_EMAIL"] = "*****@*****.**"
os.environ["GIT_AUTHOR_NAME"] = "e3 core"
os.environ["GIT_COMMITTER_NAME"] = "*****@*****.**"
os.environ["GIT_COMMITTER_EMAIL"] = "e3 core"
os.environ["E3_ENABLE_FEATURE"] = e3_feature
url = SVNRepository.create("svn", initial_content_path=self.repo_data)
url2 = SVNRepository.create("svn2",
initial_content_path=self.repo_data2)
url3 = GitRepository.create("git", initial_content_path=self.repo_data)
url4 = GitRepository.create("git2",
initial_content_path=self.repo_data2)
r = SVNRepository(working_copy=os.path.abspath("svn_checkout"))
r.update(url=url)
m = CheckoutManager(name="myrepo",
working_dir=".",
compute_changelog=compute_changelog)
logging.info("update of non existing url")
result = m.update(vcs="svn", url=url + "wrong_url")
assert result == ReturnValue.failure
logging.info("update of existing url")
result = m.update(vcs="svn", url=url)
assert result == ReturnValue.success
logging.info("update of existing url with no changes")
result = m.update(vcs="svn", url=url)
assert result == ReturnValue.unchanged
# Update the repository
logging.info("Do a checkin in svn repository")
with open(os.path.join("svn_checkout", "file3.txt"), "w") as fd:
fd.write("new file!")
r.svn_cmd(["add", "file3.txt"])
r.svn_cmd(["commit", "file3.txt", "-m", "checkin"])
logging.info("Check that we see the update")
result = m.update(vcs="svn", url=url)
assert result == ReturnValue.success
assert os.path.isfile(os.path.join("svn_checkout", "file3.txt"))
logging.info("Do a local modification in the working dir")
with open(os.path.join("myrepo", "file3.txt"), "w") as fd:
fd.write("new file modified!")
logging.info("And then do an update and check that cleanup was done")
result = m.update(vcs="svn", url=url)
assert result == ReturnValue.unchanged
with open(os.path.join("myrepo", "file3.txt")) as fd:
assert fd.read().strip() == "new file!"
logging.info("Check that we can switch from one svn url to another")
result = m.update(vcs="svn", url=url2)
assert result == ReturnValue.success
assert os.path.isfile(os.path.join("myrepo", "file1.txt", "data2.txt"))
logging.info("Check that we can switch from one svn url to a git repo")
result = m.update(vcs="git", url=url3, revision="master")
assert result == ReturnValue.success
assert os.path.isfile(os.path.join("myrepo", "file1.txt"))
logging.info(
"Check that we can switch from one git url to another one")
result = m.update(vcs="git", url=url4, revision="master")
assert result == ReturnValue.success
assert os.path.isfile(os.path.join("myrepo", "file1.txt", "data2.txt"))
logging.info("Check that in case of no changes unchanged is returned")
result = m.update(vcs="git", url=url4, revision="master")
assert result == ReturnValue.unchanged
logging.info("Check that changes are detected in git reposotories")
with open(os.path.join("git2", "file3.txt"), "w") as fd:
fd.write("new file!")
r = GitRepository(os.path.abspath("git2"))
r.git_cmd(["add", "file3.txt"])
r.git_cmd(["commit", "-m", "new file"])
result = m.update(vcs="git", url=url4, revision="master")
assert result == ReturnValue.success
assert os.path.isfile(os.path.join("myrepo", "file3.txt"))
logging.info("Check that local modifications are discarded")
with open(os.path.join("myrepo", "file3.txt"), "w") as fd:
fd.write("new file modified!")
result = m.update(vcs="git", url=url4, revision="master")
assert result == ReturnValue.unchanged
with open(os.path.join("myrepo", "file3.txt")) as fd:
assert fd.read().strip() == "new file!"
result = m.update(vcs="git",
url=url4 + "non-existing",
revision="master")
assert result == ReturnValue.failure
result = m.update(vcs="git", url=url4)
assert result == ReturnValue.failure
# Add a .gitignore and use an external
touch("git2/file4.txt")
touch("git2/ignore_file.txt")
with open("git2/.gitignore", "w") as fd:
fd.write("/ignore_file.txt")
result = m.update(vcs="external", url=os.path.abspath("git2"))
assert os.path.isfile(os.path.join(m.working_dir, "file4.txt"))
assert not os.path.isfile(
os.path.join(m.working_dir, "ignore_file.txt"))
assert result == ReturnValue.success
result = m.update(vcs="external", url=os.path.abspath("git2"))
assert result == ReturnValue.unchanged
def update_git(self, url, revision):
"""Update working dir using a Git repository.
:param url: git repository url
:type url: str
:param revision: git revision
:type revision: str
"""
# For git repositories revision cannot be None
if revision is None:
return ReturnValue.failure, None, None
g = GitRepository(working_tree=self.working_dir)
g.log_stream = e3.log.default_output_stream
old_commit, new_commit = None, None
# Create a remote for which name is bind to a an url
# This ensure that when the url does not change, git will not
# redownload all the objects on each objects (and thus avoid
# disk space leaks).
if isinstance(url, str):
remote_name = hashlib.sha256(url.encode("utf-8")).hexdigest()
else:
remote_name = hashlib.sha256(url).hexdigest()
g.init()
# Do the remote addition manually as in that context we can ignore
# safely any error returned by this command.
try:
remote_list = g.git_cmd(["remote"], output=PIPE).out.splitlines()
if remote_name not in remote_list:
g.git_cmd(["remote", "add", remote_name, url])
except Exception:
# Ignore exception as it probably means that remote already exist
# In case of real error the failure will be detected later.
pass
try:
old_commit = g.rev_parse()
# Using fetch + checkout ensure caching is effective
g.git_cmd(
["fetch", "-f", remote_name,
"%s:refs/e3-checkout" % revision])
g.checkout("refs/e3-checkout", force=True)
new_commit = g.rev_parse()
# Verify that there is no local change
p = g.git_cmd(["status", "--porcelain"], output=PIPE)
if p.out:
logger.error(
"Repository %s is locally modified, saving "
"diff in stash\n%s",
self.name,
p.out,
)
g.git_cmd(["stash", "save", "-u"])
if old_commit == new_commit:
result = ReturnValue.unchanged
elif self.compute_changelog:
# Fetch the change log and dump it into the changelog file
with closing(
tempfile.NamedTemporaryFile(mode="w",
delete=False)) as fd:
g.write_log(fd,
rev_range="%s..%s" % (old_commit, new_commit))
tmp_filename = fd.name
try:
with open(tmp_filename) as fd:
commits = [
commit
for commit in g.parse_log(fd, max_diff_size=1024)
]
finally:
rm(tmp_filename)
with open(self.changelog_file, "w") as fd:
json.dump(commits, fd)
# We have removed local changes or updated the git repository
result = ReturnValue.success
else:
result = ReturnValue.success
except GitError:
logger.exception("Error during git update %s" % self.name)
result = ReturnValue.failure
return result, old_commit, new_commit
def test_git_repo():
working_tree = os.path.join(os.getcwd(), 'working_tree')
working_tree2 = os.path.join(os.getcwd(), 'working_tree2')
repo = GitRepository(working_tree)
repo.init()
os.chdir(working_tree)
new_file = os.path.join(working_tree, 'new.txt')
echo_to_file(new_file, 'new\n')
commit_note = unixpath(os.path.join(working_tree, 'commit_note.txt'))
echo_to_file(
commit_note, '\n'.join(
('Code-Review+2: Nobody <*****@*****.**>',
'Submitted-by: Nobody <*****@*****.**>',
'Submitted-at: Thu, 08 Jun 2017 18:40:11 +0200')))
repo.git_cmd(['add', 'new.txt'])
repo.git_cmd(['config', 'user.email', '*****@*****.**'])
repo.git_cmd(['config', 'user.name', 'e3 core'])
repo.git_cmd(['commit', '-m', 'new file'])
repo.git_cmd(['tag', '-a', '-m', 'new tag', '20.0855369232'])
repo.git_cmd(
['notes', '--ref', 'review', 'add', 'HEAD', '-F', commit_note])
# try with gerrit notes
with open('log.txt', 'w') as f:
repo.write_log(f, with_gerrit_notes=True)
with open('log.txt', 'r') as f:
commits = list(repo.parse_log(f))
assert '*****@*****.**' in commits[0]['notes']['Code-Review+2']
# try with an invalid note
repo.git_cmd([
'notes', '--ref', 'review', 'add', 'HEAD', '-f', '-m', 'invalid-note'
])
with open('log.txt', 'w') as f:
repo.write_log(f, with_gerrit_notes=True)
with open('log.txt', 'r') as f:
commits = list(repo.parse_log(f))
assert commits[0]['notes'] is None
# try again without gerrit notes
with open('log.txt', 'w') as f:
repo.write_log(f)
with open('log.txt', 'r') as f:
commits = list(repo.parse_log(f))
assert 'new file' in commits[0]['message']
assert commits[0]['email'] == '*****@*****.**'
new_sha = commits[0]['sha']
assert '20.0855369232' in repo.describe()
assert new_sha == repo.rev_parse()
with pytest.raises(GitError) as err:
repo.describe('g')
assert 'describe --always g' in str(err)
echo_to_file(new_file, 'new line\n', append=True)
with open('commit1.diff', 'wb') as f:
repo.write_local_diff(f)
with open('commit1.diff', 'rb') as f:
assert b'+new line' in f.read()
echo_to_file(new_file, 10000 * '*')
repo.git_cmd(['commit', '-a', '-m', 'file update'])
with open('log2.txt', 'w') as f:
repo.write_log(f)
with open('log2.txt', 'r') as f:
commits = list(repo.parse_log(f, max_diff_size=1000))
# assert b'diff too long' not in commits[1]['diff']
assert 'file update' in commits[0]['message']
assert 'diff too long' in commits[0]['diff']
assert 'new file' in commits[1]['message']
assert commits[1]['sha'] != commits[0]['sha']
assert commits[1]['diff'] != commits[0]['diff']
repo2 = GitRepository(working_tree2)
giturl = 'file://%s' % working_tree.replace('\\', '/')
repo2.init(url=giturl, remote='tree1')
repo2.update(url=giturl, refspec='master')
repo2.rev_parse() == repo.rev_parse()
repo2.fetch_gerrit_notes(url=giturl)
p = repo2.git_cmd(['notes', '--ref=review', 'show', new_sha],
output=subprocess.PIPE)
assert 'invalid-note' in p.out
def test_git_repo():
working_tree = os.path.join(os.getcwd(), "working_tree")
working_tree2 = os.path.join(os.getcwd(), "working_tree2")
repo = GitRepository(working_tree)
repo.init()
os.chdir(working_tree)
new_file = os.path.join(working_tree, "new.txt")
echo_to_file(new_file, "new\n")
commit_note = unixpath(os.path.join(working_tree, "commit_note.txt"))
echo_to_file(
commit_note,
"\n".join((
"Code-Review+2: Nobody <*****@*****.**>",
"Submitted-by: Nobody <*****@*****.**>",
"Submitted-at: Thu, 08 Jun 2017 18:40:11 +0200",
)),
)
repo.git_cmd(["add", "new.txt"])
repo.git_cmd(["config", "user.email", "*****@*****.**"])
repo.git_cmd(["config", "user.name", "e3 core"])
repo.git_cmd(["commit", "-m", "new file"])
repo.git_cmd(["tag", "-a", "-m", "new tag", "20.0855369232"])
repo.git_cmd(
["notes", "--ref", "review", "add", "HEAD", "-F", commit_note])
# try with gerrit notes
with open("log.txt", "w") as f:
repo.write_log(f, with_gerrit_notes=True)
with open("log.txt") as f:
commits = list(repo.parse_log(f))
assert "*****@*****.**" in commits[0]["notes"]["Code-Review+2"]
# try with an invalid note
repo.git_cmd([
"notes", "--ref", "review", "add", "HEAD", "-f", "-m", "invalid-note"
])
with open("log.txt", "w") as f:
repo.write_log(f, with_gerrit_notes=True)
with open("log.txt") as f:
commits = list(repo.parse_log(f))
assert commits[0]["notes"] is None
# try again without gerrit notes
with open("log.txt", "w") as f:
repo.write_log(f)
with open("log.txt") as f:
commits = list(repo.parse_log(f))
assert "new file" in commits[0]["message"]
assert commits[0]["email"] == "*****@*****.**"
new_sha = commits[0]["sha"]
assert "20.0855369232" in repo.describe()
assert new_sha == repo.rev_parse()
with pytest.raises(GitError) as err:
repo.describe("g")
assert "describe --always g" in str(err)
echo_to_file(new_file, "new line\n", append=True)
with open("commit1.diff", "wb") as f:
repo.write_local_diff(f)
with open("commit1.diff", "rb") as f:
assert b"+new line" in f.read()
echo_to_file(new_file, 10000 * "*")
repo.git_cmd(["commit", "-a", "-m", "file update"])
with open("log2.txt", "w") as f:
repo.write_log(f)
with open("log2.txt") as f:
commits = list(repo.parse_log(f, max_diff_size=1000))
# assert b'diff too long' not in commits[1]['diff']
assert "file update" in commits[0]["message"]
assert "diff too long" in commits[0]["diff"]
assert "new file" in commits[1]["message"]
assert commits[1]["sha"] != commits[0]["sha"]
assert commits[1]["diff"] != commits[0]["diff"]
repo2 = GitRepository(working_tree2)
giturl = "file://%s" % working_tree.replace("\\", "/")
repo2.init(url=giturl, remote="tree1")
repo2.update(url=giturl, refspec="master")
repo2.rev_parse() == repo.rev_parse()
repo2.fetch_gerrit_notes(url=giturl)
p = repo2.git_cmd(["notes", "--ref=review", "show", new_sha],
output=subprocess.PIPE)
assert "invalid-note" in p.out
def test_git_non_utf8():
"""Test with non utf-8 encoding in changelog."""
working_tree = os.path.join(os.getcwd(), "working_tree")
repo = GitRepository(working_tree)
repo.init()
os.chdir(working_tree)
new_file = os.path.join(working_tree, "new.txt")
commit_msg = os.path.join(working_tree, "commit.txt")
with open(commit_msg, "wb") as fd:
fd.write(b"\x03\xff")
with open(new_file, "wb") as fd:
fd.write(b"\x03\xff")
repo.git_cmd(["add", "new.txt"])
repo.git_cmd(["config", "user.email", "*****@*****.**"])
repo.git_cmd(["config", "user.name", "e3 core"])
repo.git_cmd(["commit", "-F", commit_msg])
with closing(tempfile.NamedTemporaryFile(mode="w", delete=False)) as fd:
repo.write_log(fd)
tmp_filename = fd.name
try:
with open(tmp_filename) as fd:
commits = list(repo.parse_log(fd, max_diff_size=1024))
finally:
rm(tmp_filename)
assert "\\x03\\xff" in commits[0]["diff"]
def e3fake_git_dir(git, tmpdir):
"""Create a fake e3 git repo for the test suite."""
del git # skip or fail when git is not installed
# Create a e3-fake git repository
e3_fake_git_dir = str(tmpdir.mkdir('e3-fake-git-repo'))
try:
e3_fake_dir = os.path.join(os.path.dirname(__file__), 'e3-fake')
for source_file in os.listdir(e3_fake_dir):
if os.path.isfile(os.path.join(e3_fake_dir, source_file)):
e3.fs.cp(os.path.join(e3_fake_dir, source_file),
os.path.join(e3_fake_git_dir, source_file))
if os.path.isdir(os.path.join(e3_fake_git_dir, '.git')):
return
g = GitRepository(e3_fake_git_dir)
g.init()
g.git_cmd(['config', 'user.email', '"*****@*****.**"'])
g.git_cmd(['config', 'user.name', '"test"'])
g.git_cmd(['add', '-A'])
g.git_cmd(['commit', '-m', "'add all'"])
yield e3_fake_git_dir
finally:
e3.fs.rm(e3_fake_git_dir, True)
def git_specs_dir(git, tmpdir):
"""Create a git repo to check out specs from it."""
del git # skip or fail when git is not installed
# Create a e3-specs git repository for all tests in this module
specs_dir = str(tmpdir.mkdir('e3-specs-git-repo'))
try:
specs_source_dir = os.path.join(os.path.dirname(__file__), 'specs')
for spec_file in os.listdir(specs_source_dir):
if os.path.isfile(os.path.join(specs_source_dir, spec_file)):
e3.fs.cp(os.path.join(specs_source_dir, spec_file),
os.path.join(specs_dir, spec_file))
# Put prolog file
create_prolog(specs_dir)
if os.path.isdir(os.path.join(specs_dir, '.git')):
return
g = GitRepository(specs_dir)
g.init()
g.git_cmd(['config', 'user.email', '"*****@*****.**"'])
g.git_cmd(['config', 'user.name', '"test"'])
g.git_cmd(['add', '-A'])
g.git_cmd(['commit', '-m', "'add all'"])
yield 'file://%s' % specs_dir.replace('\\', '/')
finally:
e3.fs.rm(specs_dir, True)
def update_external(
self, url: str, revision: Optional[str]
) -> tuple[ReturnValue, str, str]:
"""Update working dir using a local directory.
:param url: path to the repository
:param revision: ignored
If <url>/.git is a directory then git ls-files will be called to get
the list of files to ignore.
"""
if os.path.isdir(self.working_dir):
old_commit = get_filetree_state(self.working_dir)
else:
old_commit = ""
ignore_list: list[str] = []
if which("rsync") and "use-rsync" in os.environ.get(
"E3_ENABLE_FEATURE", ""
).split(","):
# Run rsync using -a but without preserving timestamps. --update switch
# is also used to skip files that are older in the user directory than
# in the checkout itself. This ensure rsync remain efficient event when
# timestamps are not preserved (otherwise rsync has to compute checksum
# of all file as quick check cannot be used when timestamp is not
# preserved).
rsync_cmd = [
"rsync",
"--update",
"-rlpgoD",
f"{unixpath(url)}/",
f"{unixpath(self.working_dir)}",
"--delete-excluded",
] + [f"--exclude={el}" for el in VCS_IGNORE_LIST]
if os.path.isdir(os.path.join(url, ".git")) and os.path.isfile(
os.path.join(url, ".gitignore")
):
rsync_cmd.append("--filter=:- .gitignore")
p = Run(rsync_cmd, cwd=url, output=None)
if p.status != 0:
raise e3.error.E3Error("rsync failed")
else:
if os.path.isdir(os.path.join(url, ".git")):
# It seems that this is a git repository. Get the list of files to
# ignore
try:
g = GitRepository(working_tree=url)
ignore_list_lines = g.git_cmd(
[
"ls-files",
"-o",
"--ignored",
"--exclude-standard",
"--directory",
],
output=PIPE,
).out
ignore_list = [
f"/{f.strip().rstrip('/')}"
for f in ignore_list_lines.splitlines()
]
logger.debug("Ignore in external: %s", ignore_list)
except Exception: # defensive code
# don't crash on exception
pass
sync_tree(
url,
self.working_dir,
preserve_timestamps=False,
delete_ignore=True,
ignore=list(VCS_IGNORE_LIST) + ignore_list,
)
new_commit = get_filetree_state(self.working_dir)
if new_commit == old_commit:
return ReturnValue.unchanged, old_commit, new_commit
else:
return ReturnValue.success, old_commit, new_commit