def generate(self):
db_path = os.path.join("data", self.git_repo_path)
db.register(
db_path,
"https://s3-us-west-2.amazonaws.com/communitytc-bugbug/data/",
VERSION,
)
is_old_version = db.is_old_schema(db_path)
with ThreadPoolExecutorResult(max_workers=2) as executor:
cloner = executor.submit(repository.clone, self.repo_dir)
cloner.add_done_callback(
lambda future: logger.info("mozilla-central cloned")
)
git_user = get_secret("GIT_USER")
git_password = get_secret("GIT_PASSWORD")
repo_push_url = self.repo_url.replace(
"https://", f"https://{git_user}:{git_password}@"
)
if not is_old_version:
executor.submit(self.clone_git_repo)
else:
executor.submit(self.init_git_repo)
tenacity.retry(
lambda: subprocess.run(
["git", "config", "--global", "http.postBuffer", "12M"], check=True
),
wait=tenacity.wait_fixed(30),
stop=tenacity.stop_after_attempt(5),
)()
push_args = ["git", "push", repo_push_url, "master"]
if is_old_version:
push_args.append("--force")
done = False
while not done:
done = generator.generate(
self.repo_dir,
self.git_repo_path,
limit=COMMITS_STEP,
tokenize=self.tokenize,
remove_comments=self.remove_comments,
)
tenacity.retry(
lambda: subprocess.run(push_args, cwd=self.git_repo_path, check=True),
wait=tenacity.wait_fixed(30),
stop=tenacity.stop_after_attempt(5),
)()
# We are not using db.upload as we don't need to upload the git repo.
upload_s3([f"{db_path}.version"])
def upload(path):
support_files_paths = [
os.path.join(os.path.dirname(path), support_file_path)
for support_file_path in DATABASES[path]["support_files"]
]
utils.upload_s3([f"{path}.zst", f"{path}.version"] + support_files_paths)