def build_image(self, path, dockerfile, full_image_name): image = None try: image = self.client.images.build(path=path, dockerfile=dockerfile, tag=full_image_name) except docker.errors.BuildError as e: log.debug(e) raise ReproduceError('Encountered a build error while building a Docker image: {}'.format(e)) except docker.errors.APIError as e: raise ReproduceError('Encountered a Docker API error while building a Docker image: {}'.format(e)) except KeyboardInterrupt: log.error('Caught a KeyboardInterrupt while building a Docker image.') return image
def _modify_script(utils: Utils, jobpair: JobPair): for j in jobpair.jobs: script_path = join(utils.get_jobpair_dir(jobpair.jobs[0]), j.job_id + '.sh') if not isfile(script_path): log.error('Script file not found at', script_path) return 1 lines = [] with open(script_path) as f: found_cd_line = False for l in f: if r'travis_cmd cd\ ' + j.repo in l: found_cd_line = True lines.append(_replace_repo_path(j, l)) elif 'export TRAVIS_BUILD_DIR=$HOME/build/' in l: lines.append(_replace_repo_path(j, l)) else: lines.append(l) if not found_cd_line: raise ReproduceError('found_cd_line is False for {}'.format( j.job_id)) with open( join(utils.get_jobpair_dir(jobpair.jobs[0]), j.job_id + '-p.sh'), 'w') as f: for l in lines: f.write(l)
def _copy_repo_tar(utils: Utils, jobpair: JobPair): for j in jobpair.jobs: if not utils.get_repo_tar_path_in_task(j): raise ReproduceError( 'Cannot find the repository tar file to copy for {}.'.format( j.job_id)) utils.copy_repo_tar_from_storage_into_jobpair_dir(j)
def _copy_original_log(utils: Utils, jobpair: JobPair): for j in jobpair.jobs: original_log_path = utils.get_orig_log_path(j.job_id) if not download_log(j.job_id, original_log_path): raise ReproduceError( 'Error while copying the original log for {}.'.format( j.job_id)) utils.copy_orig_log_into_jobpair_dir(j)
def _write_package_dockerfile(utils: Utils, jobpair: JobPair): failed_job_id = jobpair.jobs[0].job_id passed_job_id = jobpair.jobs[1].job_id failed_dockerfile_path = join(utils.get_jobpair_dir(jobpair.jobs[0]), failed_job_id + '-Dockerfile') passed_dockerfile_path = join(utils.get_jobpair_dir(jobpair.jobs[1]), passed_job_id + '-Dockerfile') with open(failed_dockerfile_path) as f: failed_lines = list(map(str.strip, f.readlines())) with open(passed_dockerfile_path) as f: passed_lines = list(map(str.strip, f.readlines())) # Check that both the failed job and the passed job used the same Travis Docker image. if failed_lines[0] != passed_lines[0]: raise ReproduceError( 'The failed job and the passed job used different Travis Docker images.' ) lines = [ failed_lines[0], # Remove PPA and clean APT 'RUN sudo rm -rf /var/lib/apt/lists/*', 'RUN sudo rm -rf /etc/apt/sources.list.d/*', 'RUN sudo apt-get clean', # Update OpenSSL and libssl to avoid using deprecated versions of TLS (TLSv1.0 and TLSv1.1). # TODO: Do we actually only want to do this when deriving from an image that has an out-of-date version of TLS? 'RUN sudo apt-get update && sudo apt-get install --only-upgrade openssl libssl-dev', # Add the repositories. 'ADD failed.tar /home/travis/build/failed/', 'ADD passed.tar /home/travis/build/passed/', # Add the original logs. 'ADD {}-orig.log /home/travis/build/'.format(failed_job_id), 'ADD {}-orig.log /home/travis/build/'.format(passed_job_id), 'RUN chmod 777 -R /home/travis/build', # Add the build scripts. 'ADD {}-p.sh /usr/local/bin/run_failed.sh'.format(failed_job_id), 'ADD {}-p.sh /usr/local/bin/run_passed.sh'.format(passed_job_id), 'RUN chmod +x /usr/local/bin/run_failed.sh', 'RUN chmod +x /usr/local/bin/run_passed.sh', # Set the user to use when running the image. 'USER travis', ] # Append a newline to each line and then concatenate all the lines. content = ''.join(map(lambda l: l + '\n', lines)) package_dockerfile = utils.get_abs_jobpair_dockerfile_path(jobpair) with open(package_dockerfile, 'w') as f: f.write(content)
def _get_reproduced_result(analyzer, reproduced_log_path, job_id, trigger_sha, repo): reproduced_result = analyzer.analyze_single_log(reproduced_log_path, job_id, trigger_sha=trigger_sha, repo=repo) if reproduced_result.get('not_in_supported_language') is True: raise ReproduceError( 'Reproduced log was not generated from a job in a supported programming language. ' 'The primary language was "{}."'.format( reproduced_result['primary_language'])) return reproduced_result
def push_image(self, image_tag): # Push to Docker Hub try: result = self.client.images.push(self.utils.config.docker_hub_repo, tag=image_tag, stream=False, auth_config=self.docker_hub_auth_config) result = result.splitlines() result = result[-1] dictionary = ast.literal_eval(result) if "error" in dictionary.keys(): log.error('Error: ', dictionary.get('error')) elif "status" in dictionary.keys(): log.info('Status: ', dictionary.get('status')) except docker.errors.APIError: raise ReproduceError('Encountered a Docker API error while pushing a Docker image to Docker Hub.') except KeyboardInterrupt: log.error('Caught a KeyboardInterrupt while pushing a Docker image to Docker Hub.') # Push to Registry if not self.utils.config.docker_registry_repo: log.warning('Docker Private Registry info not being set. Skipping.') return try: result = self.client.images.push(self.utils.config.docker_registry_repo, tag=image_tag, stream=False, auth_config=self.docker_registry_auth_config) result = result.splitlines() result = result[-1] dictionary = ast.literal_eval(result) if "error" in dictionary.keys(): log.error('Error: ', dictionary.get('error')) elif "status" in dictionary.keys(): log.info('Status: ', dictionary.get('status')) except docker.errors.APIError: raise ReproduceError('Encountered a Docker API error while pushing a Docker image to Docker Registry.') except KeyboardInterrupt: log.error('Caught a KeyboardInterrupt while pushing a Docker image to Docker Registry.')
def gen_script(utils, job, dependence_solver): """ Invoke travis-build to generate the build script. """ build_sh = os.path.join('reproduce_tmp', job.job_id + '.sh') reproducing_dir = utils.get_reproducing_repo_dir(job) if dependence_solver: from bugswarm.dependency_solver.dependency_solver import fix_dict pip_patch_result = os.path.join(utils.get_jobpair_dir(job), '{}-pip-patch.json'.format(job.job_id)) commit_time = job.build.commit_time if not commit_time: github_wrapper = GitHubWrapper(GITHUB_TOKENS) _, commit_json = github_wrapper.get( 'https://api.github.com/repos/{}/git/commits/{}'.format( job.repo, job.travis_merge_sha)) commit_time = commit_json['committer']['date'] yaml_path = os.path.join(reproducing_dir, '.travis.yml') yaml_dict = job.config fixed_yaml_dict, pip_patch, apt_patch = fix_dict( reproducing_dir, yaml_dict, commit_time) with open(yaml_path, "w+") as f: yaml.dump(fixed_yaml_dict, f) if pip_patch: write_json(pip_patch_result, pip_patch) # update travis compile path based on https://github.com/travis-ci/travis-build/pull/1137 travis_command = '~/.travis/travis-build/bin/travis compile > {}'.format( build_sh) else: # default travis compile should include build number and job number to resolve the matrix travis_command = '~/.travis/travis-build/bin/travis compile {} > {}'.format( job.build_job, build_sh) cd_command = 'cd {}'.format(reproducing_dir) _, stderr, returncode = ShellWrapper.run_commands(cd_command, travis_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if returncode != 0: raise ReproduceError( 'Encountered an error while generating the build script with travis-build: {}.' .format(stderr))
def push_image(self, image_tag): try: result = self.client.images.push(self.utils.config.dockerhub_repo, tag=image_tag, stream=False, auth_config=self.auth_config) result = result.splitlines() result = result[-1] dictionary = ast.literal_eval(result) if "error" in dictionary.keys(): log.error('Error: ', dictionary.get('error')) elif "status" in dictionary.keys(): log.info('Status: ', dictionary.get('status')) except docker.errors.APIError: raise ReproduceError( 'Encountered a Docker API error while pushing a Docker image.') except KeyboardInterrupt: log.error( 'Caught a KeyboardInterrupt while pushing a Docker image.')
def _get_original_result(analyzer, utils, job_id, trigger_sha, repo): original_log_path = utils.get_orig_log_path(job_id) # If the original log does not exist in the expected location, try to download it to that location. If the log # cannot be downloaded, return error. if not os.path.isfile(original_log_path): log.debug('Original log not found at {}.'.format(original_log_path)) log.info('Download original log.') if not download_log(job_id, original_log_path): log.info('Could not download original log.') return None, original_log_path original_result = analyzer.analyze_single_log(original_log_path, job_id, trigger_sha=trigger_sha, repo=repo) if original_result.get('not_in_supported_language') is True: raise ReproduceError( 'Original log was not generated from a job in a supported programming language. ' 'The primary language was "{}."'.format( original_result['primary_language'])) return original_result, original_log_path
def copy_log(job_dispatcher, job): if job_dispatcher.utils.check_if_log_exist(job): if job_dispatcher.utils.check_is_bad_log(job): raise ReproduceError('Bad log.') job_dispatcher.utils.copy_logs_into_current_task_dir(job)
def setup_repo(job, utils, job_dispatcher): to_setup_repo = False clone_repo = False wait_for_repo_cloned = False wait_for_repo_setup = False job_id = job.job_id if job.repo in job_dispatcher.cloned_repos and job_dispatcher.cloned_repos[ job.repo] == -1: # Already tried cloning this repository and failed. So skip it. raise ReproduceError( 'Previously encountered an error while cloning a repository. Skipping.' ) if job_id in job_dispatcher.workspace_locks and job_dispatcher.workspace_locks[ job_id] == -1: # Already tried setting up this repository and failed. So skip it. raise ReproduceError( 'Previously encountered an error while setting up a repository. Skipping.' ) # ------------ Clone repository ----------- job_dispatcher.lock.acquire() if job.repo not in job_dispatcher.cloned_repos: job_dispatcher.cloned_repos[job.repo] = 0 clone_repo = True else: if job_dispatcher.cloned_repos[job.repo] == 0: wait_for_repo_cloned = True job_dispatcher.lock.release() if wait_for_repo_cloned: while job_dispatcher.cloned_repos[job.repo] == 0: time.sleep(3) if job_dispatcher.cloned_repos[job.repo] == -1: raise ReproduceError('already error in cloning repo') if clone_repo: try: clone_project_repo_if_not_exists(utils, job) except KeyboardInterrupt: log.error('Caught a KeyboardInterrupt while cloning a repository.') except Exception: job_dispatcher.cloned_repos[job.repo] = -1 job_dispatcher.job_center.repos[job.repo].clone_error = True job_dispatcher.job_center.repos[ job.repo].set_all_jobs_in_repo_to_skip() raise ReproduceError( 'Encountered an error while cloning a repository.') else: job_dispatcher.cloned_repos[job.repo] = 1 job_dispatcher.job_center.repos[job.repo].has_repo = True # ------- setup_repo: Copy, reset, and tar ------- job_dispatcher.lock.acquire() if job_id not in job_dispatcher.workspace_locks: job_dispatcher.workspace_locks[job_id] = 0 to_setup_repo = True else: if job_dispatcher.workspace_locks[job_id] == 0: wait_for_repo_setup = True job_dispatcher.lock.release() if wait_for_repo_setup: while job_dispatcher.workspace_locks[job_id] == 0: time.sleep(3) if job_dispatcher.workspace_locks[job_id] == -1: raise ReproduceError('already error in setup_repo') if to_setup_repo: try: if job.resettable is False and job.github_archived is True: download_repo(job, utils) elif job.resettable is True: copy_and_reset_repo(job, utils) else: raise ReproduceError( 'Job is neither resettable nor GitHub archived.') except KeyboardInterrupt: log.error( 'Caught a KeyboardInterrupt while setting up a repository.') raise except Exception as e: job_dispatcher.workspace_locks[job_id] = -1 raise ReproduceError( 'Encountered an error while setting up a repository: {}'. format(e)) else: job_dispatcher.workspace_locks[job_id] = 1 else: log.debug('Job', job_id, 'is already set up.') # Lastly, check if .travis.yml exists in the repository. If not, skip. if not os.path.isfile( os.path.join(job_dispatcher.utils.get_reproducing_repo_dir(job), '.travis.yml')): raise ReproduceError( 'Cannot find .travis.yml in repository. Skipping.')