def clone_git_repo(self, repo_url, repo_dir):
if not os.path.exists(repo_dir):
retry(lambda: subprocess.run(["git", "clone", repo_url, repo_dir],
check=True))
retry(lambda: subprocess.run(
["git", "pull", repo_url, "master"],
cwd=repo_dir,
capture_output=True,
check=True,
))
def generate(self):
db_path = os.path.join("data", self.git_repo_path)
db.register(
db_path,
f"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.microannotate_{self.git_repo_path}.latest/artifacts/public/",
VERSION,
)
# TODO: Check the version again once we can run tasks for longer (https://bugzilla.mozilla.org/show_bug.cgi?id=1604175).
is_old_version = False # db.is_old_schema(db_path)
with ThreadPoolExecutorResult(max_workers=2) as executor:
cloner = executor.submit(repository.clone, self.repo_dir)
cloner.add_done_callback(
lambda future: logger.info("mozilla-central cloned")
)
git_user = get_secret("GIT_USER")
git_password = get_secret("GIT_PASSWORD")
repo_push_url = self.repo_url.replace(
"https://", f"https://{git_user}:{git_password}@"
)
if not is_old_version:
executor.submit(self.clone_git_repo)
else:
executor.submit(self.init_git_repo)
retry(
lambda: subprocess.run(
["git", "config", "--global", "http.postBuffer", "12M"], check=True
)
)
push_args = ["git", "push", repo_push_url, "master"]
if is_old_version:
push_args.append("--force")
done = False
while not done:
done = generator.generate(
self.repo_dir,
self.git_repo_path,
limit=COMMITS_STEP,
tokenize=self.tokenize,
remove_comments=self.remove_comments,
)
retry(lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True))
def clone_gecko_dev(repo_dir):
repo_url = "https://github.com/mozilla/gecko-dev"
if not os.path.exists(repo_dir):
retry(lambda: subprocess.run(["git", "clone", repo_url, repo_dir], check=True))
retry(
lambda: subprocess.run(
["git", "pull", repo_url, "master"],
cwd=repo_dir,
capture_output=True,
check=True,
)
)
def clone_git_repo(self, git_repo_path):
retry(lambda: subprocess.run(
["git", "clone", self.repo_url, git_repo_path], check=True))
try:
retry(lambda: subprocess.run(
["git", "pull", self.repo_url, "master"],
cwd=git_repo_path,
capture_output=True,
check=True,
))
except subprocess.CalledProcessError as e:
# When the repo is empty.
if b"Couldn't find remote ref master" in e.stdout:
pass
def generate(self):
shared_dir = self.repo_dir + "-shared"
cmd = hglib.util.cmdbuilder(
"robustcheckout",
"https://hg.mozilla.org/mozilla-central",
self.repo_dir,
purge=True,
sharebase=shared_dir,
networkattempts=7,
branch=b"tip",
)
cmd.insert(0, hglib.HGPATH)
proc = hglib.util.popen(cmd)
out, err = proc.communicate()
if proc.returncode:
raise hglib.error.CommandError(cmd, proc.returncode, out, err)
logger.info("mozilla-central cloned")
git_user = get_secret("GIT_USER")
git_password = get_secret("GIT_PASSWORD")
repo_url = "https://github.com/marco-c/gecko-dev-wordified"
repo_push_url = (
f"https://{git_user}:{git_password}@github.com/marco-c/gecko-dev-wordified"
)
git_repo_path = os.path.basename(repo_url)
retry(lambda: subprocess.run(["git", "clone", repo_url, git_repo_path],
check=True))
try:
retry(lambda: subprocess.run(
["git", "pull", repo_url, "master"],
cwd=git_repo_path,
capture_output=True,
check=True,
))
except subprocess.CalledProcessError as e:
# When the repo is empty.
if b"Couldn't find remote ref master" in e.stdout:
pass
done = generator.generate(self.repo_dir, git_repo_path, limit=10000)
with open("done", "w") as f:
f.write(str(1 if done else 0))
retry(lambda: subprocess.run(
["git", "config", "--global", "http.postBuffer", "12M"],
check=True))
retry(lambda: subprocess.run(["git", "push", repo_push_url, "master"],
cwd=git_repo_path,
check=True))
def generate(self):
repository.clone(self.repo_dir)
logger.info("mozilla-central cloned")
git_user = get_secret("GIT_USER")
git_password = get_secret("GIT_PASSWORD")
repo_push_url = self.repo_url.replace(
"https://", f"https://{git_user}:{git_password}@")
git_repo_path = os.path.basename(self.repo_url)
retry(lambda: subprocess.run(
["git", "clone", self.repo_url, git_repo_path], check=True))
try:
retry(lambda: subprocess.run(
["git", "pull", self.repo_url, "master"],
cwd=git_repo_path,
capture_output=True,
check=True,
))
except subprocess.CalledProcessError as e:
# When the repo is empty.
if b"Couldn't find remote ref master" in e.stdout:
pass
retry(lambda: subprocess.run(
["git", "config", "--global", "http.postBuffer", "12M"],
check=True))
for i in range(STEPS):
logger.info(f"Step {i} out of {STEPS}")
done = generator.generate(
self.repo_dir,
git_repo_path,
limit=TOTAL_COMMITS // STEPS,
tokenize=self.tokenize,
remove_comments=self.remove_comments,
)
with open("done", "w") as f:
f.write(str(1 if done else 0))
retry(lambda: subprocess.run(
["git", "push", repo_push_url, "master"],
cwd=git_repo_path,
check=True,
))
if done:
break
def generate(self):
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
cloner = executor.submit(repository.clone, self.repo_dir)
cloner.add_done_callback(
lambda future: logger.info("mozilla-central cloned"))
git_user = get_secret("GIT_USER")
git_password = get_secret("GIT_PASSWORD")
repo_push_url = self.repo_url.replace(
"https://", f"https://{git_user}:{git_password}@")
git_repo_path = os.path.basename(self.repo_url)
executor.submit(self.clone_git_repo, git_repo_path)
retry(lambda: subprocess.run(
["git", "config", "--global", "http.postBuffer", "12M"],
check=True))
for i in range(STEPS):
logger.info(f"Step {i} out of {STEPS}")
done = generator.generate(
self.repo_dir,
git_repo_path,
limit=TOTAL_COMMITS // STEPS,
tokenize=self.tokenize,
remove_comments=self.remove_comments,
)
with open("done", "w") as f:
f.write(str(1 if done else 0))
retry(lambda: subprocess.run(
["git", "push", repo_push_url, "master"],
cwd=git_repo_path,
check=True,
))
if done:
break
def clone_git_repo(self, repo_url, repo_dir, rev="master"):
logger.info(f"Cloning {repo_url}...")
if not os.path.exists(repo_dir):
retry(lambda: subprocess.run(["git", "clone", repo_url, repo_dir],
check=True))
retry(lambda: subprocess.run(
["git", "pull", repo_url, "master"],
cwd=repo_dir,
capture_output=True,
check=True,
))
retry(lambda: subprocess.run(["git", "checkout", rev],
cwd=repo_dir,
capture_output=True,
check=True))
