def __init__(
self,
git_repo_url=None,
git_repo_dir=None,
tokenized_git_repo_url=None,
tokenized_git_repo_dir=None,
):
self.git_repo_url = git_repo_url
self.git_repo_dir = git_repo_dir
self.tokenized_git_repo_url = tokenized_git_repo_url
self.tokenized_git_repo_dir = tokenized_git_repo_dir
with ThreadPoolExecutorResult(max_workers=3) as executor:
if self.git_repo_url is not None:
logger.info(
f"Cloning {self.git_repo_url} to {self.git_repo_dir}...")
executor.submit(self.clone_git_repo, self.git_repo_url,
self.git_repo_dir)
if self.tokenized_git_repo_url is not None:
logger.info(
f"Cloning {self.tokenized_git_repo_url} to {self.tokenized_git_repo_dir}..."
)
executor.submit(
self.clone_git_repo,
self.tokenized_git_repo_url,
self.tokenized_git_repo_dir,
)
logger.info(
"Initializing mapping between git and mercurial commits...")
self.init_mapping()
def generate(self):
db_path = os.path.join("data", self.git_repo_path)
db.register(
db_path,
"https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/",
VERSION,
)
is_old_version = db.is_old_schema(db_path)
with ThreadPoolExecutorResult(max_workers=2) as executor:
cloner = executor.submit(repository.clone, self.repo_dir)
cloner.add_done_callback(
lambda future: logger.info("mozilla-central cloned")
)
git_user = get_secret("GIT_USER")
git_password = get_secret("GIT_PASSWORD")
repo_push_url = self.repo_url.replace(
"https://", f"https://{git_user}:{git_password}@"
)
if not is_old_version:
executor.submit(self.clone_git_repo)
else:
executor.submit(self.init_git_repo)
tenacity.retry(
lambda: subprocess.run(
["git", "config", "--global", "http.postBuffer", "12M"], check=True
),
wait=tenacity.wait_fixed(30),
stop=tenacity.stop_after_attempt(5),
)()
push_args = ["git", "push", repo_push_url, "master"]
if is_old_version:
push_args.append("--force")
done = False
while not done:
done = generator.generate(
self.repo_dir,
self.git_repo_path,
limit=COMMITS_STEP,
tokenize=self.tokenize,
remove_comments=self.remove_comments,
)
tenacity.retry(
lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True),
wait=tenacity.wait_fixed(30),
stop=tenacity.stop_after_attempt(5),
)()
# We are not using db.upload as we don't need to upload the git repo.
upload_s3([f"{db_path}.version"])
def generate(self):
db_path = os.path.join("data", self.git_repo_path)
db.register(
db_path,
f"https://community-tc.services.mozilla.com/api/index/v1/task/project.relman.bugbug.microannotate_{self.git_repo_path}.latest/artifacts/public/",
VERSION,
)
# TODO: Check the version again once we can run tasks for longer (https://bugzilla.mozilla.org/show_bug.cgi?id=1604175).
is_old_version = False # db.is_old_schema(db_path)
with ThreadPoolExecutorResult(max_workers=2) as executor:
cloner = executor.submit(repository.clone, self.repo_dir)
cloner.add_done_callback(
lambda future: logger.info("mozilla-central cloned"))
git_user = get_secret("GIT_USER")
git_password = get_secret("GIT_PASSWORD")
repo_push_url = self.repo_url.replace(
"https://", f"https://{git_user}:{git_password}@")
if not is_old_version:
executor.submit(self.clone_git_repo)
else:
executor.submit(self.init_git_repo)
tenacity.retry(
lambda: subprocess.run(
["git", "config", "--global", "http.postBuffer", "12M"],
check=True),
wait=tenacity.wait_fixed(30),
stop=tenacity.stop_after_attempt(5),
)()
push_args = ["git", "push", repo_push_url, "master"]
if is_old_version:
push_args.append("--force")
done = False
while not done:
done = generator.generate(
self.repo_dir,
self.git_repo_path,
limit=COMMITS_STEP,
tokenize=self.tokenize,
remove_comments=self.remove_comments,
)
tenacity.retry(
lambda: subprocess.run(
push_args, cwd=self.git_repo_path, check=True),
wait=tenacity.wait_fixed(30),
stop=tenacity.stop_after_attempt(5),
)()