def __init__( self, git_repo_url=None, git_repo_dir=None, tokenized_git_repo_url=None, tokenized_git_repo_dir=None, ): self.git_repo_url = git_repo_url self.git_repo_dir = git_repo_dir self.tokenized_git_repo_url = tokenized_git_repo_url self.tokenized_git_repo_dir = tokenized_git_repo_dir with ThreadPoolExecutorResult(max_workers=3) as executor: if self.git_repo_url is not None: logger.info( f"Cloning {self.git_repo_url} to {self.git_repo_dir}...") executor.submit(self.clone_git_repo, self.git_repo_url, self.git_repo_dir) if self.tokenized_git_repo_url is not None: logger.info( f"Cloning {self.tokenized_git_repo_url} to {self.tokenized_git_repo_dir}..." ) executor.submit( self.clone_git_repo, self.tokenized_git_repo_url, self.tokenized_git_repo_dir, ) logger.info( "Initializing mapping between git and mercurial commits...") self.init_mapping()
def generate(self): db_path = os.path.join("data", self.git_repo_path) db.register( db_path, "https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/", VERSION, ) is_old_version = db.is_old_schema(db_path) with ThreadPoolExecutorResult(max_workers=2) as executor: cloner = executor.submit(repository.clone, self.repo_dir) cloner.add_done_callback( lambda future: logger.info("mozilla-central cloned") ) git_user = get_secret("GIT_USER") git_password = get_secret("GIT_PASSWORD") repo_push_url = self.repo_url.replace( "https://", f"https://{git_user}:{git_password}@" ) if not is_old_version: executor.submit(self.clone_git_repo) else: executor.submit(self.init_git_repo) tenacity.retry( lambda: subprocess.run( ["git", "config", "--global", "http.postBuffer", "12M"], check=True ), wait=tenacity.wait_fixed(30), stop=tenacity.stop_after_attempt(5), )() push_args = ["git", "push", repo_push_url, "master"] if is_old_version: push_args.append("--force") done = False while not done: done = generator.generate( self.repo_dir, self.git_repo_path, limit=COMMITS_STEP, tokenize=self.tokenize, remove_comments=self.remove_comments, ) tenacity.retry( lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True), wait=tenacity.wait_fixed(30), stop=tenacity.stop_after_attempt(5), )() # We are not using db.upload as we don't need to upload the git repo. upload_s3([f"{db_path}.version"])
def generate(self): db_path = os.path.join("data", self.git_repo_path) db.register( db_path, f"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.microannotate_{self.git_repo_path}.latest/artifacts/public/", VERSION, ) # TODO: Check the version again once we can run tasks for longer (https://bugzilla.mozilla.org/show_bug.cgi?id=1604175). is_old_version = False # db.is_old_schema(db_path) with ThreadPoolExecutorResult(max_workers=2) as executor: cloner = executor.submit(repository.clone, self.repo_dir) cloner.add_done_callback( lambda future: logger.info("mozilla-central cloned")) git_user = get_secret("GIT_USER") git_password = get_secret("GIT_PASSWORD") repo_push_url = self.repo_url.replace( "https://", f"https://{git_user}:{git_password}@") if not is_old_version: executor.submit(self.clone_git_repo) else: executor.submit(self.init_git_repo) tenacity.retry( lambda: subprocess.run( ["git", "config", "--global", "http.postBuffer", "12M"], check=True), wait=tenacity.wait_fixed(30), stop=tenacity.stop_after_attempt(5), )() push_args = ["git", "push", repo_push_url, "master"] if is_old_version: push_args.append("--force") done = False while not done: done = generator.generate( self.repo_dir, self.git_repo_path, limit=COMMITS_STEP, tokenize=self.tokenize, remove_comments=self.remove_comments, ) tenacity.retry( lambda: subprocess.run( push_args, cwd=self.git_repo_path, check=True), wait=tenacity.wait_fixed(30), stop=tenacity.stop_after_attempt(5), )()