def test_fetch_concurrent_timeout(self, monkeypatch, tmpdir):
# Much shorter timeout
monkeypatch.setattr('fmf.utils.FETCH_LOCK_TIMEOUT', 2)
def long_run(*args, **kwargs):
# Longer than timeout
time.sleep(7)
# Patch run to use sleep instead
monkeypatch.setattr('fmf.utils.run', long_run)
# Background thread to fetch() the same destination acquiring lock
def target():
utils.fetch(GIT_REPO, '0.10', destination=str(tmpdir))
thread = threading.Thread(target=target)
thread.start()
# Small sleep to mitigate race
time.sleep(2)
# "Real" fetch shouldn't get the lock
with pytest.raises(utils.GeneralError):
utils.fetch(GIT_REPO, '0.10', destination=str(tmpdir))
# Wait on parallel thread to finish
thread.join()
def test_env(self):
# Nonexistent repo on github makes git to ask for password
# Set handler for user input as echo to return immediately
with pytest.raises(utils.GeneralError) as error:
utils.fetch('https://github.com/psss/fmf-nope-nope.git',
env={"GIT_ASKPASS": "******"})
# Assert 'git clone' string in exception's message
assert "git clone" in error.value.args[0]
def test_fetch_default_branch(self):
# On GitHub 'master' is the default
repo = utils.fetch(GIT_REPO)
output, _ = run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], repo)
assert 'master' in output
# Fedora uses 'rawide'
repo = utils.fetch(GIT_REPO_FEDORA)
output, _ = run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], repo)
assert 'rawhide' in output
def test_destination(self, tmpdir, trailing):
# Does not exist
dest = str(tmpdir.join('branch_new' + trailing))
repo = utils.fetch(GIT_REPO, destination=dest)
assert repo == dest
assert os.path.isfile(os.path.join(repo, 'fmf.spec'))
# Is an empty directory
dest = str(tmpdir.mkdir('another' + trailing))
repo = utils.fetch(GIT_REPO, destination=dest)
assert repo == dest
assert os.path.isfile(os.path.join(repo, 'fmf.spec'))
def test_out_of_sync_ref(self, ref):
""" Solve Your branch is behind ... """
repo = utils.fetch(GIT_REPO, ref)
out, err = run(["git", "rev-parse", "HEAD"], repo)
old_ref = out
# Move head one commit back, doesn't invalidate FETCH!
out, err = run(["git", "reset", "--hard", "HEAD^1"], repo)
out, err = run(["git", "rev-parse", "HEAD"], repo)
assert out != old_ref
# Fetch again, it should move the head back to origin/master
repo = utils.fetch(GIT_REPO, ref)
out, err = run(["git", "rev-parse", "HEAD"], repo)
assert out == old_ref
def test_switch_branches(self):
# Default branch
repo = utils.fetch(GIT_REPO)
output, _ = run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], repo)
assert 'master' in output
# Custom commit
repo = utils.fetch(GIT_REPO, '0.12')
output, _ = run(['git', 'rev-parse', 'HEAD'], repo)
assert '6570aa5' in output
# Back to the default branch
repo = utils.fetch(GIT_REPO)
output, _ = run(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], repo)
assert 'master' in output
def test_fetch_concurrent(self):
def do_fetch():
try:
repo = utils.fetch(GIT_REPO, '0.10')
q.put(True)
except Exception as error:
q.put(error)
# make sure cache is empty (is there a better way how to target repo?)
repo = utils.fetch(GIT_REPO, '0.10')
shutil.rmtree(repo)
q = queue.Queue()
threads = []
for i in range(10):
threads.append(threading.Thread(target=do_fetch))
for t in threads:
t.start()
for t in threads:
t.join()
# for number of threads check their results
all_good = True
for t in threads:
value = q.get()
if isinstance(value, Exception):
print(value) # so it is visible in the output
all_good = False
assert all_good
def test_invalid_destination(self, tmpdir):
# Is a file
dest = tmpdir.join('file')
dest.write('content')
with pytest.raises(utils.GeneralError):
repo = utils.fetch(GIT_REPO, destination=str(dest))
# Is a directory, but not empty
dest = tmpdir.mkdir('yet_another')
dest.join('some_file').write('content')
with pytest.raises(utils.GeneralError) as error:
repo = utils.fetch(GIT_REPO, destination=str(dest))
# Git's error message
assert ("already exists and is not an empty"
in error.value.args[1].stderr)
# We report same error message as before
assert str(error.value) == str(error.value.args[1])
def node(reference):
"""
Return Tree node referenced by the fmf identifier
Keys supported in the reference:
url .... git repository url (optional)
ref .... branch, tag or commit (default branch if not provided)
path ... metadata tree root ('.' by default)
name ... tree node name ('/' by default)
See the documentation for the full fmf id specification:
https://fmf.readthedocs.io/en/latest/concept.html#identifiers
Raises ReferenceError if referenced node does not exist.
"""
# Fetch remote git repository
if 'url' in reference:
path = reference.get('path', '.').lstrip('/')
# Create lock path to fetch/read git from URL to the cache
cache_dir = utils.get_cache_directory()
# Use .read.lock suffix (different from the inner fetch lock)
lock_path = os.path.join(
cache_dir, reference["url"].replace('/', '_')) + '.read.lock'
try:
with FileLock(lock_path, timeout=NODE_LOCK_TIMEOUT) as lock:
# Write PID to lockfile so we know which process got it
with open(lock.lock_file, 'w') as lock_file:
lock_file.write(str(os.getpid()))
repository = utils.fetch(
reference.get('url'), reference.get('ref'))
root = os.path.join(repository, path)
tree = Tree(root)
except Timeout:
raise utils.GeneralError(
"Failed to acquire lock for {0} within {1} seconds".format(
lock_path, NODE_LOCK_TIMEOUT))
# Use local files
else:
root = reference.get('path', '.')
if not root.startswith('/') and root != '.':
raise utils.ReferenceError(
'Relative path "%s" specified.' % root)
tree = Tree(root)
found_node = tree.find(reference.get('name', '/'))
if found_node is None:
raise utils.ReferenceError(
"No tree node found for '{0}' reference".format(reference))
# FIXME Should be able to remove .cache if required
return found_node
def test_fetch_deprecation(self):
with pytest.warns(FutureWarning):
repo = utils.fetch(GIT_REPO, '0.10')
assert utils.os.path.isfile(utils.os.path.join(repo, 'fmf.spec'))
def test_fetch_valid_id(self):
repo = utils.fetch(GIT_REPO, '0.10')
assert utils.os.path.isfile(utils.os.path.join(repo, 'fmf.spec'))
def target():
utils.fetch(GIT_REPO, '0.10', destination=str(tmpdir))
def do_fetch():
try:
repo = utils.fetch(GIT_REPO, '0.10')
q.put(True)
except Exception as error:
q.put(error)
def test_fetch_invalid_url(self):
with pytest.raises(utils.GeneralError):
utils.fetch('invalid')
def test_fetch_invalid_ref(self):
with pytest.raises(utils.GeneralError):
utils.fetch(GIT_REPO, 'invalid')
def test_cache_expiration(self):
repo = utils.fetch(GIT_REPO)
fetch_head = (os.path.join(repo, '.git', 'FETCH_HEAD'))
os.remove(fetch_head)
repo = utils.fetch(GIT_REPO)
assert os.path.isfile(fetch_head)
def test_invalid_cache_directory(self, monkeypatch):
with pytest.raises(utils.GeneralError):
monkeypatch.setenv("XDG_CACHE_HOME", "/etc")
utils.fetch(GIT_REPO)