def clone_git_repo(self, repo_url, repo_dir): if not os.path.exists(repo_dir): retry(lambda: subprocess.run(["git", "clone", repo_url, repo_dir], check=True)) retry(lambda: subprocess.run( ["git", "pull", repo_url, "master"], cwd=repo_dir, capture_output=True, check=True, ))
def generate(self): db_path = os.path.join("data", self.git_repo_path) db.register( db_path, f"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.microannotate_{self.git_repo_path}.latest/artifacts/public/", VERSION, ) # TODO: Check the version again once we can run tasks for longer (https://bugzilla.mozilla.org/show_bug.cgi?id=1604175). is_old_version = False # db.is_old_schema(db_path) with ThreadPoolExecutorResult(max_workers=2) as executor: cloner = executor.submit(repository.clone, self.repo_dir) cloner.add_done_callback( lambda future: logger.info("mozilla-central cloned") ) git_user = get_secret("GIT_USER") git_password = get_secret("GIT_PASSWORD") repo_push_url = self.repo_url.replace( "https://", f"https://{git_user}:{git_password}@" ) if not is_old_version: executor.submit(self.clone_git_repo) else: executor.submit(self.init_git_repo) retry( lambda: subprocess.run( ["git", "config", "--global", "http.postBuffer", "12M"], check=True ) ) push_args = ["git", "push", repo_push_url, "master"] if is_old_version: push_args.append("--force") done = False while not done: done = generator.generate( self.repo_dir, self.git_repo_path, limit=COMMITS_STEP, tokenize=self.tokenize, remove_comments=self.remove_comments, ) retry(lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True))
def clone_gecko_dev(repo_dir): repo_url = "https://github.com/mozilla/gecko-dev" if not os.path.exists(repo_dir): retry(lambda: subprocess.run(["git", "clone", repo_url, repo_dir], check=True)) retry( lambda: subprocess.run( ["git", "pull", repo_url, "master"], cwd=repo_dir, capture_output=True, check=True, ) )
def clone_git_repo(self, git_repo_path): retry(lambda: subprocess.run( ["git", "clone", self.repo_url, git_repo_path], check=True)) try: retry(lambda: subprocess.run( ["git", "pull", self.repo_url, "master"], cwd=git_repo_path, capture_output=True, check=True, )) except subprocess.CalledProcessError as e: # When the repo is empty. if b"Couldn't find remote ref master" in e.stdout: pass
def generate(self): shared_dir = self.repo_dir + "-shared" cmd = hglib.util.cmdbuilder( "robustcheckout", "https://hg.mozilla.org/mozilla-central", self.repo_dir, purge=True, sharebase=shared_dir, networkattempts=7, branch=b"tip", ) cmd.insert(0, hglib.HGPATH) proc = hglib.util.popen(cmd) out, err = proc.communicate() if proc.returncode: raise hglib.error.CommandError(cmd, proc.returncode, out, err) logger.info("mozilla-central cloned") git_user = get_secret("GIT_USER") git_password = get_secret("GIT_PASSWORD") repo_url = "https://github.com/marco-c/gecko-dev-wordified" repo_push_url = ( f"https://{git_user}:{git_password}@github.com/marco-c/gecko-dev-wordified" ) git_repo_path = os.path.basename(repo_url) retry(lambda: subprocess.run(["git", "clone", repo_url, git_repo_path], check=True)) try: retry(lambda: subprocess.run( ["git", "pull", repo_url, "master"], cwd=git_repo_path, capture_output=True, check=True, )) except subprocess.CalledProcessError as e: # When the repo is empty. if b"Couldn't find remote ref master" in e.stdout: pass done = generator.generate(self.repo_dir, git_repo_path, limit=10000) with open("done", "w") as f: f.write(str(1 if done else 0)) retry(lambda: subprocess.run( ["git", "config", "--global", "http.postBuffer", "12M"], check=True)) retry(lambda: subprocess.run(["git", "push", repo_push_url, "master"], cwd=git_repo_path, check=True))
def generate(self): repository.clone(self.repo_dir) logger.info("mozilla-central cloned") git_user = get_secret("GIT_USER") git_password = get_secret("GIT_PASSWORD") repo_push_url = self.repo_url.replace( "https://", f"https://{git_user}:{git_password}@") git_repo_path = os.path.basename(self.repo_url) retry(lambda: subprocess.run( ["git", "clone", self.repo_url, git_repo_path], check=True)) try: retry(lambda: subprocess.run( ["git", "pull", self.repo_url, "master"], cwd=git_repo_path, capture_output=True, check=True, )) except subprocess.CalledProcessError as e: # When the repo is empty. if b"Couldn't find remote ref master" in e.stdout: pass retry(lambda: subprocess.run( ["git", "config", "--global", "http.postBuffer", "12M"], check=True)) for i in range(STEPS): logger.info(f"Step {i} out of {STEPS}") done = generator.generate( self.repo_dir, git_repo_path, limit=TOTAL_COMMITS // STEPS, tokenize=self.tokenize, remove_comments=self.remove_comments, ) with open("done", "w") as f: f.write(str(1 if done else 0)) retry(lambda: subprocess.run( ["git", "push", repo_push_url, "master"], cwd=git_repo_path, check=True, )) if done: break
def generate(self): with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor: cloner = executor.submit(repository.clone, self.repo_dir) cloner.add_done_callback( lambda future: logger.info("mozilla-central cloned")) git_user = get_secret("GIT_USER") git_password = get_secret("GIT_PASSWORD") repo_push_url = self.repo_url.replace( "https://", f"https://{git_user}:{git_password}@") git_repo_path = os.path.basename(self.repo_url) executor.submit(self.clone_git_repo, git_repo_path) retry(lambda: subprocess.run( ["git", "config", "--global", "http.postBuffer", "12M"], check=True)) for i in range(STEPS): logger.info(f"Step {i} out of {STEPS}") done = generator.generate( self.repo_dir, git_repo_path, limit=TOTAL_COMMITS // STEPS, tokenize=self.tokenize, remove_comments=self.remove_comments, ) with open("done", "w") as f: f.write(str(1 if done else 0)) retry(lambda: subprocess.run( ["git", "push", repo_push_url, "master"], cwd=git_repo_path, check=True, )) if done: break
def clone_git_repo(self, repo_url, repo_dir, rev="master"): logger.info(f"Cloning {repo_url}...") if not os.path.exists(repo_dir): retry(lambda: subprocess.run(["git", "clone", repo_url, repo_dir], check=True)) retry(lambda: subprocess.run( ["git", "pull", repo_url, "master"], cwd=repo_dir, capture_output=True, check=True, )) retry(lambda: subprocess.run(["git", "checkout", rev], cwd=repo_dir, capture_output=True, check=True))