def extract_commit_experiences():
try:
utils.extract_file(repository.COMMIT_EXPERIENCES_DB)
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info("Commit experiences DB extracted.")
def download(path, support_files_too=False, extract=True):
# If a DB with the current schema is not available yet, we can't download.
if is_different_schema(path):
return False
zst_path = f"{path}.zst"
url = DATABASES[path]["url"]
try:
logger.info(f"Downloading {url} to {zst_path}")
updated = utils.download_check_etag(url, zst_path)
if extract and updated:
utils.extract_file(zst_path)
os.remove(zst_path)
successful = True
if support_files_too:
for support_file in DATABASES[path]["support_files"]:
successful |= download_support_file(path, support_file,
extract)
return successful
except requests.exceptions.HTTPError:
logger.info(f"{url} is not yet available to download", exc_info=True)
return False
def extract_touched_together():
try:
utils.extract_file(test_scheduling.TOUCHED_TOGETHER_DB)
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info("Touched together DB extracted.")
def extract_past_failures_group():
try:
utils.extract_file(test_scheduling.PAST_FAILURES_GROUP_DB)
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info("Group-level past failures DB extracted.")
def extract_past_failures_label():
try:
utils.extract_file(test_scheduling.PAST_FAILURES_LABEL_DB)
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info("Label-level past failures DB extracted.")
def test_extract_db_bad_format(tmp_path):
path = tmp_path / "prova.pickle"
with open(path, "wb") as output_f:
pickle.dump({"Hello": "World"}, output_f)
with pytest.raises(AssertionError):
utils.extract_file(path)
def extract_commit_experiences() -> None:
try:
utils.extract_file(os.path.join("data", repository.COMMIT_EXPERIENCES_DB))
logger.info("Commit experiences DB extracted.")
except FileNotFoundError:
logger.info(
"Commit experiences DB not extracted, but missing models are allowed."
)
assert ALLOW_MISSING_MODELS
def extract_commits() -> bool:
try:
utils.extract_file(f"{repository.COMMITS_DB}.zst")
logger.info("Commits DB extracted.")
return True
except FileNotFoundError:
logger.info("Commits DB not extracted, but missing models are allowed.")
assert ALLOW_MISSING_MODELS
return False
def extract_commits():
try:
utils.extract_file(f"{repository.COMMITS_DB}.zst")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
return False
logger.info("Commits DB extracted.")
return True
def extract_touched_together():
try:
utils.extract_file(
os.path.join("data", test_scheduling.TOUCHED_TOGETHER_DB))
logger.info("Touched together DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Touched together DB not extracted, but missing models are allowed."
)
def extract_past_failures_group():
try:
utils.extract_file(
os.path.join("data", test_scheduling.PAST_FAILURES_GROUP_DB))
logger.info("Group-level past failures DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Group-level past failures DB not extracted, but missing models are allowed."
)
def extract_failing_together_config_group() -> None:
try:
utils.extract_file(
os.path.join("data", test_scheduling.FAILING_TOGETHER_CONFIG_GROUP_DB)
)
logger.info("Failing together config/group DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Failing together config/group DB not extracted, but missing models are allowed."
)
def extract_failing_together_label() -> None:
try:
utils.extract_file(
os.path.join("data", test_scheduling.FAILING_TOGETHER_LABEL_DB)
)
logger.info("Failing together label DB extracted.")
except FileNotFoundError:
assert ALLOW_MISSING_MODELS
logger.info(
"Failing together label DB not extracted, but missing models are allowed."
)
def test_extract_db_zst(tmp_path, mock_zst):
path = tmp_path / f"prova.zst"
mock_zst(path)
utils.extract_file(path)
with open(f"{os.path.splitext(path)[0]}", "rb") as f:
file_decomp = json.load(f)
assert file_decomp == {"Hello": "World"}
def download_support_file(path, file_name, extract=True):
# If a DB with the current schema is not available yet, we can't download.
if is_different_schema(path):
return False
try:
url = urljoin(DATABASES[path]["url"], file_name)
path = os.path.join(os.path.dirname(path), file_name)
logger.info(f"Downloading {url} to {path}")
updated = utils.download_check_etag(url, path)
if extract and updated and path.endswith(".zst"):
utils.extract_file(path)
os.remove(path)
return True
except requests.exceptions.HTTPError:
logger.info(f"{file_name} is not yet available to download for {path}",
exc_info=True)
return False