def update_central_repos(self):
logger = logging.getLogger(__name__)
# TODO: the first time we clone, ssh might want to verify the server an we might need to manually accept it.
# TODO: How can we automate this?
self.shas = {}
first = True # only return branches of the first repository
for repo in self.config['repos']:
logger.debug("Repo url {}".format(repo['url']))
if repo['type'] != 'git':
raise Exception("Unsupported repository {}".format(
repo['type']))
repo_local_name = self.get_repo_local_name(repo)
logger.debug("Local repo dir {}".format(repo_local_name))
# TODO have a root directory for each project that is under the server root
# TODO allow the user to supply a local directory
local_repo_path = os.path.join(self.server['repositories'],
repo_local_name)
logger.debug("Local repo path {}".format(local_repo_path))
if not os.path.exists(local_repo_path):
logger.debug("clone repo for the first time")
if 'credentials' in repo:
os.environ[
'GIT_SSH_COMMAND'] = "ssh -i " + repo['credentials']
cmd_list = [git, 'clone', repo['url'], repo_local_name]
logger.debug(' '.join(cmd_list))
with cwd(self.server['repositories']):
code, out = _system(cmd_list)
# get current sha ?? In which branch?
if first:
old_branches = {}
else:
logger.debug("update repository")
if first:
old_branches = self.get_branches(local_repo_path)
cmd_list = [git, 'pull']
with cwd(local_repo_path):
code, out = _system(cmd_list)
if first:
new_branches = self.get_branches(local_repo_path)
first = False
else:
branches = self.get_branches(local_repo_path)
self.shas[repo['name']] = branches[repo['branch']]
#logger.debug(yaml.dump(new_branches))
self.old_branches = old_branches
self.new_branches = new_branches
return
def test_repos(self, tmpdir):
temp_dir = str(tmpdir)
print(temp_dir)
self.setup_repos(temp_dir, {}, 2)
_system("python check.py --server {} --config {} {}".format(
self.server_file, self.config_file, debug))
assert os.path.exists(os.path.join(self.repositories, 'repo0'))
assert os.listdir(os.path.join(self.repositories,
'repo0/')) == ['.git']
assert not os.path.exists(os.path.join(self.workdir, '1', 'repo0/'))
assert os.path.exists(os.path.join(self.repositories, 'repo1'))
assert os.listdir(os.path.join(self.repositories,
'repo1/')) == ['.git']
assert not os.path.exists(os.path.join(self.workdir, '1', 'repo1/'))
# update the repository
with cwd(self.client[0]):
with open('README.txt', 'w') as fh:
fh.write("first line\n")
_system("git add .")
_system("git commit -m 'first' --author 'Foo Bar <*****@*****.**>'")
_system("git push")
with cwd(self.client[1]):
with open('selftest.py', 'w') as fh:
fh.write("assert True\n")
_system("git add .")
_system(
"git commit -m 'start with test' --author 'Zee No <*****@*****.**>'"
)
_system("git push")
_system("python check.py --server {} --config {} {}".format(
self.server_file, self.config_file, debug))
assert os.listdir(os.path.join(self.repositories,
'repo0/')) == ['README.txt', '.git']
assert os.path.exists(os.path.join(self.workdir, '1', 'repo0/'))
assert os.listdir(os.path.join(self.workdir, '1',
'repo0/')) == ['README.txt', '.git']
assert os.listdir(os.path.join(self.repositories,
'repo1/')) == ['selftest.py', '.git']
assert os.path.exists(os.path.join(self.workdir, '1', 'repo1/'))
assert os.listdir(os.path.join(self.workdir, '1',
'repo1/')) == ['selftest.py', '.git']
def test_run_tests(self, tmpdir):
temp_dir = str(tmpdir)
print(temp_dir)
self.setup_repos(temp_dir,
{'steps': ["cli: python repo0/selftest.py"]}, 1)
# update the repository
with cwd(self.client[0]):
with open('selftest.py', 'w') as fh:
fh.write("exit(13)\n")
_system("git add .")
_system("git commit -m 'first' --author 'Foo Bar <*****@*****.**>'")
_system("git push")
code, out = capture2(
"python check.py --server {} --config {} {}".format(
self.server_file, self.config_file, debug),
shell=True)
print(out)
assert code == 1, "One test failure repored"
#assert out == "" # TODO: this will fail if --debug is on, but also becaues there is some output from the git commands.
assert os.listdir(os.path.join(self.repositories,
'repo0/')) == ['selftest.py', '.git']
assert os.path.exists(os.path.join(self.workdir, '1', 'repo0/'))
assert os.listdir(os.path.join(self.workdir, '1',
'repo0/')) == ['selftest.py', '.git']
with cwd(self.client[0]):
with open('selftest.py', 'w') as fh:
fh.write("exit(0)\n")
_system("git add .")
_system("git commit -m 'second' --author 'Foo Bar <*****@*****.**>'")
_system("git push")
code, out = capture2(
"python check.py --server {} --config {} {}".format(
self.server_file, self.config_file, debug),
shell=True)
print(out)
assert code == 0, "test sucess repored: "
#assert out == "" # TODO: this will fail if --debug is on, but also becaues there is some output from the git commands.
assert os.listdir(os.path.join(self.repositories,
'repo0/')) == ['selftest.py', '.git']
assert os.path.exists(os.path.join(self.workdir, '2', 'repo0/'))
assert os.listdir(os.path.join(self.workdir, '2',
'repo0/')) == ['selftest.py', '.git']
def get_branches(self, path):
branches = {}
with cwd(path):
_system([git, 'pack-refs', '--all'])
if os.path.exists('.git/packed-refs'):
# It seems the file does not exist if the repository is empty
with open('.git/packed-refs') as fh:
for line in fh:
if re.search(r'\A#', line):
continue
m = re.search(r'\A(\S+)\s+refs/remotes/origin/(.*)',
line)
if m:
branches[m.group(2)] = m.group(1)
return branches
def clone_repositories(self):
logger = logging.getLogger(__name__)
logger.debug("Clone the repositories")
for repo in self.config['repos']:
repo_name = repo['name']
repo_local_name = self.get_repo_local_name(repo)
code, out = _system([
git, 'clone',
os.path.join(self.server['repositories'], repo_local_name),
repo_local_name
])
if code != 0:
raise Exception("Could not clone repo")
with cwd(repo_local_name):
logger.debug("Check out the given sha")
code, out = _system([git, 'checkout', self.shas[repo_name]])
if code != 0:
raise Exception(
"Could not checkout sha1 {} in repository {}".format(
sha1, repo_local_name))
def build(self):
logger = logging.getLogger(__name__)
build_number = self.get_next_build_number()
logger.debug("Build number: {}".format(build_number))
# TODO store the build in some queue and also allow the parallel execution of jobs on agents
# update_local_repositories()
build_parent_directory = os.path.join(self.server['workdir'],
str(build_number))
logger.debug("Build parent dir: {}".format(build_parent_directory))
os.mkdir(build_parent_directory)
bg = self.add_build_logger(build_parent_directory)
logger.debug("Starting Build {} in directory: {}".format(
build_number, build_parent_directory))
results = {'status': 'success'}
if 'matrix' in self.config:
results['matrix'] = {}
subbuild = 0
for case in self.config['matrix']:
subbuild += 1
logger.debug("On agent '{}' schedule exe: '{}'".format(
case['agent'], case['exe']))
if case['agent'] not in self.server['agents']:
results['status'] = 'failure'
results['matrix'][subbuild] = {
'agent': case['agent'],
'error': "Agent is not available.",
}
continue
if case['agent'] == 'master':
# TODO use the limit to run in parallel
build_directory = os.path.join(build_parent_directory,
str(subbuild))
os.mkdir(build_directory)
with cwd(build_directory):
self.clone_repositories()
code, out = _system(case['exe'])
results['matrix'][subbuild] = {
'exit': code,
'agent': case['agent'],
'exe': case['exe'],
'out': out,
}
if code != 0:
results['status'] = 'failure'
else:
pass
results['status'] = 'failure'
# TODO
# ssh host
# scp our code, the configuration files (that are appropriate to that machine)
# clone the directories
# run the build
else:
build_directory = build_parent_directory
with cwd(build_directory):
self.clone_repositories()
if 'steps' in self.config:
results['steps'] = []
logger.debug("Run the steps defined in the configuration")
for step in self.config['steps']:
logger.debug(step)
m = re.search(r'\Acli:\s*(.*)', step)
cmd = m.group(1)
logger.debug(cmd)
code, out = _system(cmd)
results['steps'].append({
'exit': code,
'agent': 'master',
'out': out,
'step': step,
})
if code != 0:
results['status'] = 'failure'
break
with open(os.path.join(self.server['db'],
str(build_number) + '.json'), "w") as fh:
#json.dump(results, fh)
json.dump(results,
fh,
sort_keys=True,
indent=4,
separators=(',', ': '),
ensure_ascii=False)
logger.removeHandler(bg)
return build_number
def setup_repos(self, temp_dir, user_config, count):
remote_repos = os.path.join(temp_dir, 'remote_repos') # bare repos
client_dir = os.path.join(temp_dir, 'client') # workspace of users
root = os.path.join(temp_dir, 'server')
self.server_file = os.path.join(root, 'server.yml')
self.config_file = os.path.join(
root, 'config.yml') # this might be in a repository
self.repositories = os.path.join(
root, 'repositories') # here is where we'll clone repos
self.db = os.path.join(root, 'db') # here is where we'll clone repos
self.workdir = os.path.join(root, 'workdir')
self.repo = []
self.client = []
for i in range(count):
self.repo.append(os.path.join(remote_repos, 'repo' + str(i)))
self.client.append(os.path.join(client_dir, 'repo' + str(i)))
os.mkdir(remote_repos)
os.mkdir(client_dir)
os.mkdir(root)
os.mkdir(self.repositories)
os.mkdir(self.workdir)
os.mkdir(self.db)
# create config files
server_config = {
'repositories': self.repositories,
'db': self.db,
'workdir': self.workdir,
'agents': {
'master': {
'limit': 1
}
}
}
with open(self.server_file, 'w') as fh:
fh.write(
yaml.dump(server_config,
explicit_start=True,
default_flow_style=False))
# in a subdirectory crete a git repository
for i in range(count):
os.mkdir(self.repo[i])
with cwd(self.repo[i]):
_system("git init --bare")
with cwd(client_dir):
_system("git clone " + self.repo[i])
user_config['repos'] = []
for i in range(count):
user_config['repos'].append({
'name': 'repo' + str(i),
'branch': 'master',
'type': 'git',
'url': self.repo[i],
})
with open(self.config_file, 'w') as fh:
fh.write(
yaml.dump(user_config,
explicit_start=True,
default_flow_style=False))
def test_repo(self, tmpdir):
temp_dir = str(tmpdir)
print(temp_dir)
self.setup_repos(temp_dir, {}, 1)
_system("python check.py --server {} --config {} {}".format(
self.server_file, self.config_file, debug))
assert os.path.exists(os.path.join(self.repositories, 'repo0'))
assert os.listdir(os.path.join(self.repositories,
'repo0/')) == ['.git']
assert not os.path.exists(os.path.join(self.workdir, '1', 'repo0/'))
# update the repository
with cwd(self.client[0]):
with open('README.txt', 'w') as fh:
fh.write("first line\n")
_system("git add .")
_system("git commit -m 'first' --author 'Foo Bar <*****@*****.**>'")
_system("git push")
_system("python check.py --server {} --config {} {}".format(
self.server_file, self.config_file, debug))
assert os.listdir(os.path.join(self.repositories,
'repo0/')) == ['README.txt', '.git']
assert os.path.exists(os.path.join(self.workdir, '1', 'repo0/'))
assert os.listdir(os.path.join(self.workdir, '1',
'repo0/')) == ['README.txt', '.git']
# check if the sha change was noticed git rev-parse HEAD
# create a branch, see if the new branch is noticed
with cwd(self.client[0]):
with open('TODO', 'w') as fh:
fh.write("Some TODO text\n")
_system("git checkout -b todo")
_system("git add .")
_system(
"git commit -m 'add test' --author 'Foo Bar <*****@*****.**>'")
_system("git push --set-upstream origin todo")
_system("git checkout master")
with open('MASTER', 'w') as fh:
fh.write("Some MASTER text\n")
_system("git add .")
_system(
"git commit -m 'add master' --author 'Foo Bar <*****@*****.**>'")
_system("git push")
code, out = capture2(
"python check.py --server {} --config {} {}".format(
self.server_file, self.config_file, debug),
shell=True)
print(out)
assert code == 0
# first workdir did not change
assert os.path.exists(os.path.join(self.workdir, '1', 'repo0/'))
assert os.listdir(os.path.join(self.workdir, '1',
'repo0/')) == ['README.txt', '.git']
# second workdir has the new file as well
assert os.path.exists(os.path.join(self.workdir, '2', 'repo0/'))
assert os.listdir(os.path.join(
self.workdir, '2', 'repo0/')) == ['MASTER', 'README.txt', '.git']
# second workdir has the new file as well
assert os.path.exists(os.path.join(self.workdir, '3', 'repo0/'))
assert os.listdir(os.path.join(
self.workdir, '3', 'repo0/')) == ['TODO', 'README.txt', '.git']
def test_run_matrix(self, tmpdir):
temp_dir = str(tmpdir)
print(temp_dir)
self.setup_repos(
temp_dir, {
'matrix': [
{
'agent': 'master',
'exe': 'python repo0/code.py Foo',
},
{
'agent': 'master',
'exe': 'python repo0/code.py',
},
{
'agent': 'master',
'exe': 'python repo0/code.py Bar',
},
{
'agent': 'master',
'exe': 'python repo0/code.py crash',
},
]
}, 1)
# update the repository
with cwd(self.client[0]):
with open('code.py', 'w') as fh:
fh.write("""
import sys
if len(sys.argv) < 2:
exit("Missing parameter")
print("hello " + sys.argv[1])
if sys.argv[1] == "crash":
v = 0
print(42/v)
""")
_system("git add .")
_system("git commit -m 'first' --author 'Foo Bar <*****@*****.**>'")
_system("git push")
code, out = capture2(
"python check.py --server {} --config {} {}".format(
self.server_file, self.config_file, debug),
shell=True)
print(out)
assert code == 1, "some tests failed in the matrix"
#assert out == "" # TODO: this will fail if --debug is on, but also becaues there is some output from the git commands.
assert os.listdir(os.path.join(self.repositories,
'repo0/')) == ['code.py', '.git']
assert os.path.exists(os.path.join(self.workdir, '1/1', 'repo0/'))
assert os.listdir(os.path.join(self.workdir, '1/1',
'repo0/')) == ['code.py', '.git']
assert os.path.exists(os.path.join(self.workdir, '1/2', 'repo0/'))
assert os.listdir(os.path.join(self.workdir, '1/2',
'repo0/')) == ['code.py', '.git']
results_file = os.path.join(self.db, '1.json')
assert os.path.exists(results_file)
with open(results_file) as fh:
results = json.load(fh)
last = results['matrix'].pop('4')
assert results == {
'status': 'failure',
'matrix': {
'1': {
'exit': 0,
'agent': 'master',
'exe': 'python repo0/code.py Foo',
'out': 'hello Foo\n'
},
'2': {
'exit': 1,
'agent': 'master',
'exe': 'python repo0/code.py',
'out': 'Missing parameter\n'
},
'3': {
'exit': 0,
'agent': 'master',
'exe': 'python repo0/code.py Bar',
'out': 'hello Bar\n'
},
},
}
out_of_last = last.pop('out')
assert last == {
'exit': 1,
'agent': 'master',
'exe': 'python repo0/code.py crash',
}
# Then end of the error message has changed so we don't test the specifics
# In Python 2: integer division or modulo by zero
# In Python 3: division by zero
assert 'hello crash\nTraceback (most recent call last):\n File "repo0/code.py", line 8, in <module>\n print(42/v)\nZeroDivisionError:' in out_of_last