def docker_pull(image_tag):
assert image_tag
assert isinstance(image_tag, str)
# Exit early if the image already exists locally.
if _image_exists_locally(image_tag):
return True
image_location = _image_location(image_tag)
command = 'sudo docker pull {}'.format(image_location)
_, _, returncode = ShellWrapper.run_commands(command, shell=True)
if returncode != 0:
log.error('Could not download the image', image_location, 'from Docker Hub.')
else:
log.info('Downloaded the image', image_location + '.')
return returncode == 0
def gen_script(utils, job, dependence_solver):
"""
Invoke travis-build to generate the build script.
"""
build_sh = os.path.join('reproduce_tmp', job.job_id + '.sh')
reproducing_dir = utils.get_reproducing_repo_dir(job)
if dependence_solver:
from bugswarm.dependency_solver.dependency_solver import fix_dict
pip_patch_result = os.path.join(utils.get_jobpair_dir(job),
'{}-pip-patch.json'.format(job.job_id))
commit_time = job.build.commit_time
if not commit_time:
github_wrapper = GitHubWrapper(GITHUB_TOKENS)
_, commit_json = github_wrapper.get(
'https://api.github.com/repos/{}/git/commits/{}'.format(
job.repo, job.travis_merge_sha))
commit_time = commit_json['committer']['date']
yaml_path = os.path.join(reproducing_dir, '.travis.yml')
yaml_dict = job.config
fixed_yaml_dict, pip_patch, apt_patch = fix_dict(
reproducing_dir, yaml_dict, commit_time)
with open(yaml_path, "w+") as f:
yaml.dump(fixed_yaml_dict, f)
if pip_patch:
write_json(pip_patch_result, pip_patch)
# update travis compile path based on https://github.com/travis-ci/travis-build/pull/1137
travis_command = '~/.travis/travis-build/bin/travis compile > {}'.format(
build_sh)
else:
# default travis compile should include build number and job number to resolve the matrix
travis_command = '~/.travis/travis-build/bin/travis compile {} > {}'.format(
job.build_job, build_sh)
cd_command = 'cd {}'.format(reproducing_dir)
_, stderr, returncode = ShellWrapper.run_commands(cd_command,
travis_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if returncode != 0:
raise ReproduceError(
'Encountered an error while generating the build script with travis-build: {}.'
.format(stderr))
def get_pr_commit_base(repo: str, commit: str, base_branch: str,
trigger_commit_timestamp: str) -> Optional[str]:
# Change into the repo directory and then find the latest commit before the commit date.
repo_path = Utils._canonical_repo_path(repo)
cd_command = 'cd {}'.format(repo_path)
# # Super hack.
# commit_date = self.github.get_commit_date(repo, commit)
# if commit_date is None:
# return None
# git_command = 'git rev-list -n 1 --skip 1 --before="{}" --branches="{}"'.format(trigger_commit_timestamp,
# base_branch)
git_command = 'git rev-list -n 1 --skip 1 --before="{}" {}'.format(
trigger_commit_timestamp, base_branch)
result, _, _ = ShellWrapper.run_commands(cd_command,
git_command,
stdout=subprocess.PIPE,
shell=True)
return result
def gen_script(build_sh_path, job_number, reproducing_dir):
"""
Invoke travis-build to generate the build script.
"""
# travis_command = '/home/anandsaw/.travis/travis-build/bin/travis compile {} > {}'.format(job_number, build_sh_path)
travis_command = '/Users/anandsaw/.travis/travis-build/bin/travis compile {} > {}'.format(
job_number, build_sh_path)
print("Creating travis build script with command: " + travis_command)
cd_command = 'cd {}'.format(reproducing_dir)
_, stderr, returncode = ShellWrapper.run_commands(cd_command,
travis_command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
if returncode != 0:
raise ValueError(
'Encountered an error while generating the build script with travis-build: {}.'
.format(stderr))
def clone_repo(repo: str) -> bool:
# We use the git clone exit code to determine if the clone succeeded. But we need to distinguish between fatal
# errors (repository not found) and circumstances that prevent cloning (clone destination directory already
# exists and is not empty). But git returns the same exit code for both errors and unexpected events. So, in
# order to make the distinction, we manually check for cases like the preexistence of the clone destination.
repo_path = Utils._canonical_repo_path(repo)
if os.path.isdir(repo_path) and os.listdir(repo_path):
# The clone destination directory already exists, so we can return early and indicate to the caller that the
# clone succeeded.
log.info('Clone of', repo, 'seems to already exist.')
return True
clone_command = 'git clone https://github.com/{}.git {} > /dev/null 2>&1'.format(
repo, repo_path)
log.info('Cloning', repo, 'into', repo_path)
_, _, returncode = ShellWrapper.run_commands(clone_command,
stdout=subprocess.PIPE,
shell=True)
return returncode == 0
def get_latest_commit_for_repo(repo: str) -> str:
repo_path = Utils._canonical_repo_path(repo)
cd_command = 'cd {}'.format(repo_path)
git_command = 'git rev-parse HEAD'
result, _, _ = ShellWrapper.run_commands(cd_command, git_command, stdout=subprocess.PIPE, shell=True)
return result
def get_parents_of_commit(repo: str, commit: str, base_branch: str) -> Optional[str]:
repo_path = Utils._canonical_repo_path(repo)
cd_command = 'cd {}'.format(repo_path)
git_command = 'git rev-list {}..master --first-parent --reverse'.format(commit)
result, _, _ = ShellWrapper.run_commands(cd_command, git_command, stdout=subprocess.PIPE, shell=True)
return result
def remove_current_task_dir(self):
command = 'rm -rf {}'.format(self.config.current_task_dir)
log.debug(command)
ShellWrapper.run_commands(command, shell=True)
def remove_workspace_dir(self):
log.info('Removing workspace directory.')
command = 'rm -rf {} 2> /dev/null'.format(self.config.workspace_dir)
ShellWrapper.run_commands(command, shell=True)
def clean_workspace_job_dir(self, job):
log.info('cleaning workspace job directory.')
command = 'rm -rf {}'.format(self.get_workspace_sha_dir(job))
ShellWrapper.run_commands(command, shell=True)
def remove_project_repos_dir(self):
log.info('Removing project_repos directory.')
command = 'rm -rf {} 2> /dev/null'.format(self.config.stored_repos_dir)
ShellWrapper.run_commands(command, shell=True)
def travis_show(repo: str, build_num: str) -> str:
show_command = ' '.join(['travis show --repo', repo, build_num])
result, _, _ = ShellWrapper.run_commands(show_command,
stdout=subprocess.PIPE,
shell=True)
return result
def docker_run(image_tag, use_sandbox, use_pipe_stdin, use_rm):
assert isinstance(image_tag, str) and not image_tag.isspace()
assert isinstance(use_sandbox, bool)
assert isinstance(use_pipe_stdin, bool)
assert isinstance(use_rm, bool)
# First, try to pull the image.
ok, image_location = docker_pull(image_tag)
if not ok:
return False
# Communicate progress to the user.
host_sandbox = _default_host_sandbox()
container_sandbox = CONTAINER_SANDBOX_DEFAULT
if use_sandbox:
if not os.path.exists(host_sandbox):
log.info('Creating', host_sandbox, 'as the host sandbox.')
os.makedirs(host_sandbox, exist_ok=True)
log.info('Binding host sandbox', host_sandbox,
'to container directory', container_sandbox)
# Communicate progress to the user.
if use_pipe_stdin:
log.info(
'Entering the container and executing the contents of stdin inside the container.'
)
else:
log.info('Entering the container.')
if use_rm:
log.info('The container will be cleaned up after use.')
# Prepare the arguments for the docker run command.
volume_args = ['-v', '{}:{}'.format(host_sandbox, container_sandbox)
] if use_sandbox else []
# The -t option must not be used in order to use a heredoc.
input_args = ['-i'] if use_pipe_stdin else ['-i', '-t']
subprocess_input = sys.stdin.read() if use_pipe_stdin else None
subprocess_universal_newlines = use_pipe_stdin
rm_args = ['--rm'] if use_rm else []
# If we're using a shared directory, we need to modify the start script to change the permissions of the shared
# directory on the container side. However, this will also change the permissions on the host side.
script_args = [SCRIPT_DEFAULT]
if use_sandbox:
start_command = '"sudo chmod -R 777 {} && cd {} && umask 000 && cd .. && {}"'.format(
container_sandbox, container_sandbox, SCRIPT_DEFAULT)
# These arguments represent a command of the following form:
# /bin/bash -c "sudo chmod 777 <container_sandbox> && cd <container_sandbox> && umask 000 && /bin/bash"
# So bash will execute chmod and umask and then start a new bash shell. From the user's perspective, the chmod
# and umask commands happen transparently. That is, the user only sees the final new bash shell.
script_args = [SCRIPT_DEFAULT, '-c', start_command]
# Try to run the image.
# The tail arguments must be at the end of the command.
tail_args = [image_location] + script_args
args = ['sudo', 'docker', 'run', '--privileged'
] + rm_args + volume_args + input_args + tail_args
command = ' '.join(args)
print(command)
_, _, returncode = ShellWrapper.run_commands(
command,
input=subprocess_input,
universal_newlines=subprocess_universal_newlines,
shell=True)
return returncode == 0
def fetch_pr_data(self, job):
owner, project_name = job.repo.split('/')
# owner_dir = self.get_repo_owner_dir(owner)
command = 'cd {} ;'.format(self.get_stored_repo_path(job))
command += 'git fetch origin refs/pull/*/head:refs/remotes/origin/pr/* ;'
ShellWrapper.run_commands(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
def is_travis_installed():
command = 'travis'
result, _, _ = ShellWrapper.run_commands(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
return 'Usage: travis COMMAND' in result
def remove_image_in_shell(full_image_name):
log.info('Removing a Docker image.')
command = 'docker rmi -f {}'.format(full_image_name)
_, _, returncode = ShellWrapper.run_commands(command, shell=True)
return returncode
def remove_all_images():
log.info('Removing all containers and Docker images (except Travis images).')
command = 'docker rm $(docker ps -a -q); docker rmi -f $(docker images -a | grep -v "travis")'
ShellWrapper.run_commands(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True)
def travis_history(repo: str, history_file_path: str):
history_command = ' '.join(
['travis history --all --date -r', repo, '>', history_file_path])
ShellWrapper.run_commands(history_command,
stdout=subprocess.PIPE,
shell=True)