def test_exec_with_retry_force_return_code(mocker, fake_fabric, remote, force_return_code):
fabric_object = fake_fabric(force_return_code)
mocker.patch("loktar.cmd.local", return_value=fabric_object)
mocker.patch("loktar.cmd.run", return_value=fabric_object)
assert exec_command_with_retry("ls -la", remote, 2, force_return_code=force_return_code) is force_return_code
def test_exec_with_retry(mocker, remote):
mocker.patch("loktar.cmd.local", return_value=FakeFabricSuccess())
mocker.patch("loktar.cmd.run", return_value=FakeFabricSuccess())
assert exec_command_with_retry("ls -la", remote, 2) is True
def worker(logger=os.environ.get("LOGGER", None), rebuild=os.environ.get("REBUILD", False)):
db = Job()
job_id = os.environ.get("JOB_ID", None)
detect_pr_collision = os.environ.get("JOB_ID", False)
if job_id is None:
raise JobIdUnknown
job = db.get_job(job_id)
commit_id = job["commit_id"]
committer = job["committer"]
job["lastAccess"] = time.time()
job["status"] = 10
db.set_job(job_id, job)
if logger is not None:
logger_info = logger.info
logger_error = logger.error
else:
log = Log()
logger_info = log.info
logger_error = log.error
#define_job_status_on_github_commit()
with hide('output', 'running', 'warnings'):
# TODO(Re-implement the line workspace = prepare_test_env(job["git_branch"]) && os.remove("{0}.tar".format(workspace)))
# workspace = prepare_test_env(job["git_branch"])
# TODO(Implement the file transfer to the CI server)
# transfer_file("PUT", remote_path="/tmp", local_path="{0}.tar".format(workspace))
# os.remove("{0}.tar".format(workspace))
logger_info('Preparing the test environment')
unique_name_dir = str(uuid4())
workspace = '/mnt/ci/{0}'.format(unique_name_dir)
os.makedirs(workspace)
try:
if not exec_command_with_retry('git clone [email protected]:{0}/{1}.git {2}'
.format(GITHUB_INFO["organization"], GITHUB_INFO["repository"], workspace),
0, MAX_RETRY_GITHUB):
raise PrepareEnvFail
except PrepareEnvFail:
local('rm -rf {0}*'.format(workspace))
raise
id_pr = None
scm = Github(GITHUB_INFO['login']['user'], GITHUB_INFO['login']['password'])
# For backwards compatibility
if job["git_branch"] == 'master':
with lcd(os.path.join(workspace, 'ci')):
config = json.loads(fetch_github_file('https://raw.githubusercontent.com/{0}/{1}/{2}/config.json'
.format(GITHUB_INFO["organization"],
GITHUB_INFO["repository"],
job["git_branch"]),
GITHUB_TOKEN))
map_pkg = config['packages']
git_diff = local('git diff HEAD~ HEAD --name-only', capture=True)
git_diff = filter(None, list(set(git_diff.split('\n'))))
dict_message_files = {}
packages = {pkg['pkg_name']: pkg for pkg in map_pkg}
modified_packages = {package_from_path(path, packages) for path in git_diff} - {None}
exclude_dep = get_excluded_deps(packages, dict_message_files, modified_packages)
logger_info('The following packages\' dependencies will be ignored if unmodified: {0}'
.format(exclude_dep))
dep_graph = dependency_graph_from_modified_packages(workspace,
packages,
modified_packages,
exclude_dep)
else:
define_job_status_on_github_commit(commit_id,
'pending',
"", # Put an url
context='Master Task - Task Launcher',
description='Checking other Pull Requests for conflicts')
id_pr = scm.search_pull_request_id(job["git_branch"])
# Get relevant information from the pull request
config, dep_graph, modified_packages = pull_request_information(id_pr, workspace)
pull_requests = scm.get_pull_requests()
pull_requests = filter(lambda prequest: prequest.mergeable, pull_requests)
id_prs = {prequest.number: prequest for prequest in pull_requests}
if detect_pr_collision:
logger_info('Testing for inconsistencies in PR #{0}'.format(set(id_prs.keys()) - {id_pr}))
original_pr = scm.get_pull_request(id_pr)
for id_pr_other in set(id_prs.keys()) - {id_pr}:
# If the other PR is more recent, do not check for inconsistency
if id_prs[id_pr_other].created_at > original_pr.created_at:
logger_info('PR #{0} is more recent than #{1}'
.format(id_pr_other, id_pr))
continue
try: # We do not care for errors on other PRs
_, dep_graph_other, _ = pull_request_information(id_pr_other, workspace)
except Exception as exc:
logger_error(traceback.format_exc())
logger_info('Got an exception on PR #{0}: {1}'.format(id_pr_other, exc))
continue
nodes_intersect = set(dep_graph_other.nodes()) & set(dep_graph.nodes())
if nodes_intersect:
logger_info('Intersecting packages: {0}'.format(nodes_intersect))
logger_info('Dependencies levels for tested PR #{0}: {1}'
.format(id_pr, gen_dependencies_level(dep_graph)))
define_job_status_on_github_commit(commit_id,
'error',
'',
context='Master Task - Task Launcher',
description=('Intersecting packages with PR #{0}: {1}'
.format(id_pr_other, ', '.join(nodes_intersect))))
raise PullRequestCollision('PR #{0} has intersecting packages with PR #{1}: {2}'
.format(id_pr_other, id_pr, nodes_intersect))
map_pkg = config["packages"]
packages = {pkg["pkg_name"]: pkg for pkg in map_pkg}
do_not_touch_packages = get_do_not_touch_packages(id_pr, packages, scm, dep_graph, rebuild)
# We can safely delete the nodes from the graph
cleaned_dep_graph = copy.deepcopy(dep_graph)
cleaned_dep_graph.remove_nodes_from(do_not_touch_packages)
# Replace dep_lvl if we use another dep_graph
_, dep_lvl_to_use, _ = gen_dependencies_level(cleaned_dep_graph)
# Setting a green build for packages that should not be touched and are not in the build to keep traceability
# Those packages are packages that were not affected by their loktar.dependency.
for package in do_not_touch_packages:
context = build_params_to_context(package, 'Not rebuilt')
define_job_status_on_github_commit(commit_id, 'success', 'http://github.com', context,
'Bro, this package seems damn fine.')
no_problem, dep_lvl, name_graph_img = gen_dependencies_level(dep_graph, draw=True)
if id_pr is not None:
scm.create_pull_request_comment(id_pr,
comment=u'Hey noobs, here go your dependency levels:\n\n```\n' +
yaml.safe_dump(dep_lvl) +
u'\n```\n' +
u''
.format(GITHUB_INFO["organization"], name_graph_img),
check_unique=True)
# Replace dep_lvl if we calculated another dep_lvl to use
dep_lvl = dep_lvl if dep_lvl_to_use is None else dep_lvl_to_use
logger_info('=' * 20)
logger_info('Dependency levels used for this build')
logger_info('=' * 20)
logger_info(dep_lvl)
logger_info('=' * 20)
params = {
'commit_id': commit_id,
'committer': committer,
'dep_lvl': serialize(dep_lvl),
'job["git_branch"]': job["git_branch"],
'modified_packages': serialize(modified_packages)
}
if no_problem:
define_job_status_on_github_commit(commit_id,
'success',
'', # TODO put an url
context='Master Task - Task Launcher',
description='Task Manager succeeded!')
# TODO (write information in the share memory)
return {
"commit_id": commit_id,
"committer": committer,
"test_env_path": workspace,
"git_branch": job["git_branch"],
"dep_lvl": dep_lvl
}
else:
logger_error('dependency level problem encountered')
define_job_status_on_github_commit(commit_id,
'error',
"", # put an url
context='Master Task - Task Launcher',
description='Dependency level problem encountered')
def prepare_test_env(branch, **kwargs):
"""Prepare the test environment
Args:
branch (str): Name of the branch the repository should be checkout to.
Keyword Args:
github_organization (str): this is the github organization for get back the repository, default value None.
Also can be set by environment variable LOKTAR_GITHUB_INFO_ORGANIZATION
github_repository (str): this is the target repository to download, default value None
Also can be set by environment variable LOKTAR_GITHUB_INFO_REPOSITORY
skip_git_clone (bool): Skip the git clone if is another process who cloned the repository, default to false
unique_name_dir (str): If the unique name dir for the location where the repository is cloned is generated by another process
Raises:
PrepareEnvFail: Failed to prepare the environment.
"""
logger = Log()
temporary_root = "/tmp/ci"
unique_name_dir = kwargs.get("unique_name_dir", str(uuid4()))
unique_path_dir = "{0}/{1}".format(temporary_root, unique_name_dir)
archive = "{0}.tar.gz".format(unique_name_dir)
logger.info("Preparing the test environment")
github_organization = kwargs.get("github_organization", GITHUB_INFO["organization"])
github_repository = kwargs.get("github_repository", GITHUB_INFO["repository"])
if not os.path.exists(unique_path_dir):
os.mkdir(unique_path_dir)
try:
if not kwargs.get("skip_git_clone", False):
if not exec_command_with_retry("git clone -b {0} --single-branch [email protected]:{1}/{2}.git {3}"
.format(branch, github_organization, github_repository, unique_path_dir),
0,
MAX_RETRY_GITHUB):
raise PrepareEnvFail(
"The git clone can't the repository: {}/{}, check if you have the correct crendentials"
.format(github_organization, github_repository))
with lcd(unique_path_dir):
if not exec_command_with_retry("git fetch origin master", 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail("Can't fetch the master branch from origin")
if branch != "master":
if not exec_command_with_retry("git config --global user.email '*****@*****.**'", 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail("Git config error on user.email")
if not exec_command_with_retry("git config --global user.name 'Your Name'", 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail("Git config error on user.name")
if not exec_command_with_retry("git merge --no-ff --no-edit FETCH_HEAD", 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail("Can't merge the FETCH_HEAD (master branch)")
local("rm -rf {0}/.git".format(unique_path_dir))
with lcd(temporary_root):
if not exec_command_with_retry("tar -czf {0} {1}".format(archive, unique_name_dir), 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail("Can't create the archive")
logger.info("The test env is ready!")
except NetworkError as exc:
logger.error(exc)
raise
except PrepareEnvFail:
local("rm -rf {0}*".format(unique_path_dir))
raise
return "{0}/{1}".format(temporary_root, archive)
def prepare_test_env(branch, **kwargs):
"""Prepare the test environment
Args:
branch (str): Name of the branch the repository should be checkout to.
Keyword Args:
github_organization (str): this is the github organization for get back the repository, default value None.
Also can be set by environment variable LOKTAR_GITHUB_INFO_ORGANIZATION
github_repository (str): this is the target repository to download, default value None
Also can be set by environment variable LOKTAR_GITHUB_INFO_REPOSITORY
Raises:
PrepareEnvFail: Failed to prepare the environment.
"""
logger = Log()
unique_name_dir = str(uuid4())
unique_path_dir = "/tmp/{0}".format(unique_name_dir)
archive = "{0}.tar.gz".format(unique_name_dir)
logger.info("Preparing the test environment")
github_organization = kwargs.get("github_organization", GITHUB_INFO["organization"])
github_repository = kwargs.get("github_repository", GITHUB_INFO["repository"])
os.mkdir(unique_path_dir)
try:
if not exec_command_with_retry(
"git clone -b {0} --single-branch [email protected]:{1}/{2}.git {3}".format(
branch, github_organization, github_repository, unique_path_dir
),
0,
MAX_RETRY_GITHUB,
):
raise PrepareEnvFail(
"The git clone can't the repository: {}/{}, check if you have the correct crendentials".format(
github_organization, github_repository
)
)
with lcd(unique_path_dir):
if not exec_command_with_retry("git fetch origin master", 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail
if branch != "master":
if not exec_command_with_retry("git config --global user.email '*****@*****.**'", 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail
if not exec_command_with_retry("git config --global user.name 'Your Name'", 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail
if not exec_command_with_retry("git merge --no-ff --no-edit FETCH_HEAD", 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail
local("rm -rf {0}/.git".format(unique_path_dir))
with lcd("/tmp"):
if not exec_command_with_retry("tar -czf {0} {1}".format(archive, unique_name_dir), 0, MAX_RETRY_GITHUB):
raise PrepareEnvFail
logger.info("The test env is ready!")
except NetworkError as exc:
logger.error(exc)
raise
except PrepareEnvFail:
local("rm -rf {0}*".format(unique_path_dir))
raise
return "/tmp/{0}".format(archive)
