def docker_pull(image_tag):
assert image_tag
assert isinstance(image_tag, str)
# Exit early if the image already exists locally.
exists, image_location = _image_exists_locally(image_tag)
if exists:
return True, image_location
image_location = _image_location(image_tag)
command = 'sudo docker pull {}'.format(image_location)
_, _, returncode = ShellWrapper.run_commands(command, shell=True)
if returncode != 0:
# Image is not cached. Attempt to pull from bugswarm/images.
image_location = '{}:{}'.format(DOCKER_HUB_REPO, image_tag)
command = 'sudo docker pull {}'.format(image_location)
_, _, returncode = ShellWrapper.run_commands(command, shell=True)
if returncode != 0:
# Image is not in bugswarm/images
log.error('Could not download the image', image_location)
else:
log.info('Downloaded the image', image_location + '.')
else:
log.info('Downloaded the image', image_location + '.')
return returncode == 0, image_location
def remove_all_images():
log.info(
'Removing all containers and Docker images (except Travis images).'
)
command = 'docker rm $(docker ps -a -q); docker rmi -f $(docker images -a | grep -v "travis")'
ShellWrapper.run_commands(command,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
shell=True)
def fetch_pr_data(self, job):
owner, project_name = job.repo.split('/')
# owner_dir = self.get_repo_owner_dir(owner)
command = 'cd {} ;'.format(self.get_stored_repo_path(job))
command += 'git fetch origin refs/pull/*/head:refs/remotes/origin/pr/* ;'
ShellWrapper.run_commands(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
def is_travis_installed():
command = 'travis'
result, _, _ = ShellWrapper.run_commands(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
return 'Usage: travis COMMAND' in result
def docker_run(image_tag, use_sandbox, use_pipe_stdin, use_rm):
assert isinstance(image_tag, str) and not image_tag.isspace()
assert isinstance(use_sandbox, bool)
assert isinstance(use_pipe_stdin, bool)
assert isinstance(use_rm, bool)
# First, try to pull the image.
ok = docker_pull(image_tag)
if not ok:
return False
# Communicate progress to the user.
host_sandbox = _default_host_sandbox()
container_sandbox = CONTAINER_SANDBOX_DEFAULT
if use_sandbox:
if not os.path.exists(host_sandbox):
log.info('Creating', host_sandbox, 'as the host sandbox.')
os.makedirs(host_sandbox, exist_ok=True)
log.info('Binding host sandbox', host_sandbox, 'to container directory', container_sandbox)
# Communicate progress to the user.
if use_pipe_stdin:
log.info('Entering the container and executing the contents of stdin inside the container.')
else:
log.info('Entering the container.')
if use_rm:
log.info('The container will be cleaned up after use.')
image_location = _image_location(image_tag)
# Prepare the arguments for the docker run command.
volume_args = ['-v', '{}:{}'.format(host_sandbox, container_sandbox)] if use_sandbox else []
# The -t option must not be used in order to use a heredoc.
input_args = ['-i'] if use_pipe_stdin else ['-i', '-t']
subprocess_input = sys.stdin.read() if use_pipe_stdin else None
subprocess_universal_newlines = use_pipe_stdin
rm_args = ['--rm'] if use_rm else []
# If we're using a shared directory, we need to modify the start script to change the permissions of the shared
# directory on the container side. However, this will also change the permissions on the host side.
script_args = [SCRIPT_DEFAULT]
if use_sandbox:
start_command = '"sudo chmod -R 777 {} && cd {} && umask 000 && cd .. && {}"'.format(
container_sandbox, container_sandbox, SCRIPT_DEFAULT)
# These arguments represent a command of the following form:
# /bin/bash -c "sudo chmod 777 <container_sandbox> && cd <container_sandbox> && umask 000 && /bin/bash"
# So bash will execute chmod and umask and then start a new bash shell. From the user's perspective, the chmod
# and umask commands happen transparently. That is, the user only sees the final new bash shell.
script_args = [SCRIPT_DEFAULT, '-c', start_command]
# Try to run the image.
# The tail arguments must be at the end of the command.
tail_args = [image_location] + script_args
args = ['sudo', 'docker', 'run', '--privileged'] + rm_args + volume_args + input_args + tail_args
command = ' '.join(args)
_, _, returncode = ShellWrapper.run_commands(command,
input=subprocess_input,
universal_newlines=subprocess_universal_newlines,
shell=True)
return returncode == 0
def get_reproducer_version() -> str:
stdout, stderr, returncode = ShellWrapper.run_commands('git rev-parse HEAD', stdout=subprocess.PIPE, shell=True)
if returncode:
msg = 'Error getting reproducer version: {}'.format(stderr)
log.error(msg)
raise IOError(msg)
return stdout
def travis_open(repo: str, build_num: str) -> str:
open_command = ' '.join(
['travis open --print --repo', repo, build_num])
url, _, _ = ShellWrapper.run_commands(open_command,
stdout=subprocess.PIPE,
shell=True)
return url
def get_branch_of_sha(repo: str, commit: str) -> Optional[str]:
# Change into the repo directory and then find the latest commit before the commit date.
repo_path = Utils._canonical_repo_path(repo)
cd_command = 'cd {}'.format(repo_path)
git_command = 'git branch --contains {}'.format(commit)
result, _, _ = ShellWrapper.run_commands(cd_command, git_command, stdout=subprocess.PIPE, shell=True)
return result
def clone_project_repo(self, repo):
owner, project_name = repo.split('/')
owner_dir = self.get_repo_owner_dir(owner)
os.makedirs(owner_dir, exist_ok=True)
destination = os.path.join(self.config.stored_repos_dir, repo)
clone_command = 'git clone {} {}'.format(Utils.construct_github_repo_url(repo), destination)
_, _, returncode = ShellWrapper.run_commands(clone_command, shell=True)
return returncode
def _docker_image_inspect(image_tag):
image_location = _image_location(image_tag)
command = 'sudo docker image inspect {}'.format(image_location)
_, _, returncode = ShellWrapper.run_commands(command,
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
# For a non-existent image, docker image inspect has a non-zero exit status.
if returncode == 0:
log.info('The image', image_location, 'already exists locally and is up to date.')
return returncode == 0
def get_latest_commit_for_repo(repo: str) -> str:
repo_path = Utils._canonical_repo_path(repo)
cd_command = 'cd {}'.format(repo_path)
git_command = 'git rev-parse HEAD'
result, _, _ = ShellWrapper.run_commands(cd_command,
git_command,
stdout=subprocess.PIPE,
shell=True)
return result
def get_parents_of_commit(repo: str, commit: str,
base_branch: str) -> Optional[str]:
repo_path = Utils._canonical_repo_path(repo)
cd_command = 'cd {}'.format(repo_path)
git_command = 'git rev-list {}..master --first-parent --reverse'.format(
commit)
result, _, _ = ShellWrapper.run_commands(cd_command,
git_command,
stdout=subprocess.PIPE,
shell=True)
return result
def get_pr_commit_base(repo: str, commit: str, base_branch: str, trigger_commit_timestamp: str) -> Optional[str]:
# Change into the repo directory and then find the latest commit before the commit date.
repo_path = Utils._canonical_repo_path(repo)
cd_command = 'cd {}'.format(repo_path)
# # Super hack.
# commit_date = self.github.get_commit_date(repo, commit)
# if commit_date is None:
# return None
# git_command = 'git rev-list -n 1 --skip 1 --before="{}" --branches="{}"'.format(trigger_commit_timestamp,
# base_branch)
git_command = 'git rev-list -n 1 --skip 1 --before="{}" {}'.format(trigger_commit_timestamp, base_branch)
result, _, _ = ShellWrapper.run_commands(cd_command, git_command, stdout=subprocess.PIPE, shell=True)
return result
def store_git_log(repo: str) -> bool:
repo_path = Utils._canonical_repo_path(repo)
if os.path.isfile(Utils.get_git_log_path(repo)):
log.info('git log of', repo, 'already exists.')
return True
command_1 = 'cd {}'.format(repo_path)
command_2 = 'git fetch origin refs/pull/*/head:refs/remotes/origin/pr/* > /dev/null 2>&1'
command_3 = 'TZ=UTC git log --branches --remotes --pretty=format:"%H %cd" --date="format-local:' \
'%Y-%m-%dT%H:%M:%SZ" >> git_log.txt'
log.info('Processing git log', repo, 'into', repo_path)
_, _, returncode = ShellWrapper.run_commands(command_1, command_2, command_3,
stdout=subprocess.PIPE, shell=True)
return returncode == 0
def docker_pull(image_tag):
assert image_tag
assert isinstance(image_tag, str)
# Exit early if the image already exists locally.
if _image_exists_locally(image_tag):
return True
image_location = _image_location(image_tag)
command = 'sudo docker pull {}'.format(image_location)
_, _, returncode = ShellWrapper.run_commands(command, shell=True)
if returncode != 0:
log.error('Could not download the image', image_location, 'from Docker Hub.')
else:
log.info('Downloaded the image', image_location + '.')
return returncode == 0
def clone_repo(repo: str) -> bool:
# We use the git clone exit code to determine if the clone succeeded. But we need to distinguish between fatal
# errors (repository not found) and circumstances that prevent cloning (clone destination directory already
# exists and is not empty). But git returns the same exit code for both errors and unexpected events. So, in
# order to make the distinction, we manually check for cases like the preexistence of the clone destination.
repo_path = Utils._canonical_repo_path(repo)
if os.path.isdir(repo_path) and os.listdir(repo_path):
# The clone destination directory already exists, so we can return early and indicate to the caller that the
# clone succeeded.
log.info('Clone of', repo, 'seems to already exist.')
return True
clone_command = 'git clone https://github.com/{}.git {} > /dev/null 2>&1'.format(repo, repo_path)
log.info('Cloning', repo, 'into', repo_path)
_, _, returncode = ShellWrapper.run_commands(clone_command, stdout=subprocess.PIPE, shell=True)
return returncode == 0
def gen_script(utils, job, dependence_solver):
"""
Invoke travis-build to generate the build script.
"""
build_sh = os.path.join('reproduce_tmp', job.job_id + '.sh')
reproducing_dir = utils.get_reproducing_repo_dir(job)
if dependence_solver:
from bugswarm.dependency_solver.dependency_solver import fix_dict
pip_patch_result = os.path.join(utils.get_jobpair_dir(job),
'{}-pip-patch.json'.format(job.job_id))
commit_time = job.build.commit_time
if not commit_time:
github_wrapper = GitHubWrapper(GITHUB_TOKENS)
_, commit_json = github_wrapper.get(
'https://api.github.com/repos/{}/git/commits/{}'.format(
job.repo, job.travis_merge_sha))
commit_time = commit_json['committer']['date']
yaml_path = os.path.join(reproducing_dir, '.travis.yml')
yaml_dict = job.config
fixed_yaml_dict, pip_patch, apt_patch = fix_dict(
reproducing_dir, yaml_dict, commit_time)
with open(yaml_path, "w+") as f:
yaml.dump(fixed_yaml_dict, f)
if pip_patch:
write_json(pip_patch_result, pip_patch)
# update travis compile path based on https://github.com/travis-ci/travis-build/pull/1137
travis_command = '~/.travis/travis-build/bin/travis compile > {}'.format(
build_sh)
else:
# default travis compile should include build number and job number to resolve the matrix
travis_command = '~/.travis/travis-build/bin/travis compile {} > {}'.format(
job.build_job, build_sh)
cd_command = 'cd {}'.format(reproducing_dir)
_, stderr, returncode = ShellWrapper.run_commands(cd_command,
travis_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if returncode != 0:
raise ReproduceError(
'Encountered an error while generating the build script with travis-build: {}.'
.format(stderr))
def gen_script(build_sh_path, job_number, reproducing_dir):
"""
Invoke travis-build to generate the build script.
"""
# travis_command = '/home/anandsaw/.travis/travis-build/bin/travis compile {} > {}'.format(job_number, build_sh_path)
travis_command = '/Users/anandsaw/.travis/travis-build/bin/travis compile {} > {}'.format(
job_number, build_sh_path)
print("Creating travis build script with command: " + travis_command)
cd_command = 'cd {}'.format(reproducing_dir)
_, stderr, returncode = ShellWrapper.run_commands(cd_command,
travis_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if returncode != 0:
raise ValueError(
'Encountered an error while generating the build script with travis-build: {}.'
.format(stderr))
def travis_show(repo: str, build_num: str) -> str:
show_command = ' '.join(['travis show --repo', repo, build_num])
result, _, _ = ShellWrapper.run_commands(show_command,
stdout=subprocess.PIPE,
shell=True)
return result
def remove_current_task_dir(self):
command = 'rm -rf {}'.format(self.config.current_task_dir)
log.debug(command)
ShellWrapper.run_commands(command, shell=True)
def remove_workspace_dir(self):
log.info('Removing workspace directory.')
command = 'rm -rf {} 2> /dev/null'.format(self.config.workspace_dir)
ShellWrapper.run_commands(command, shell=True)
def clean_workspace_job_dir(self, job):
log.info('cleaning workspace job directory.')
command = 'rm -rf {}'.format(self.get_workspace_sha_dir(job))
ShellWrapper.run_commands(command, shell=True)
def remove_project_repos_dir(self):
log.info('Removing project_repos directory.')
command = 'rm -rf {} 2> /dev/null'.format(self.config.stored_repos_dir)
ShellWrapper.run_commands(command, shell=True)
def remove_image_in_shell(full_image_name):
log.info('Removing a Docker image.')
command = 'docker rmi -f {}'.format(full_image_name)
_, _, returncode = ShellWrapper.run_commands(command, shell=True)
return returncode
def travis_history(repo: str, history_file_path: str):
history_command = ' '.join(
['travis history --all --date -r', repo, '>', history_file_path])
ShellWrapper.run_commands(history_command,
stdout=subprocess.PIPE,
shell=True)
