def test_serialize_git_storage():
storage = Git(
flow_path="flow.py",
repo="test/repo",
repo_host="bitbucket.org",
flow_name="my-flow",
git_token_secret_name="MY_TOKEN",
git_token_username="******",
branch_name="my_branch",
tag=None,
commit=None,
clone_depth=1,
use_ssh=False,
format_access_token=True,
)
serialized_storage = storage.serialize()
assert serialized_storage["type"] == "Git"
assert serialized_storage["repo"] == "test/repo"
assert serialized_storage["flow_path"] == "flow.py"
assert serialized_storage["repo_host"] == "bitbucket.org"
assert serialized_storage["flow_name"] == "my-flow"
assert serialized_storage["git_token_secret_name"] == "MY_TOKEN"
assert serialized_storage["git_token_username"] == "my_user"
assert serialized_storage["branch_name"] == "my_branch"
assert serialized_storage["tag"] == None
assert serialized_storage["commit"] == None
assert serialized_storage["clone_depth"] == 1
assert serialized_storage["use_ssh"] == False
assert serialized_storage["format_access_token"] == True
def test_add_flow_to_git_storage():
storage = Git(repo="test/repo", flow_path="flow.py")
f = Flow("test")
assert f.name not in storage
assert storage.add_flow(f) == "flow.py"
assert f.name in storage
def test_add_flow_to_git_already_added():
storage = Git(repo="test/repo", flow_path="flow.py")
f = Flow("test")
assert f.name not in storage
assert storage.add_flow(f) == "flow.py"
assert f.name in storage
with pytest.raises(ValueError):
storage.add_flow(f)
def test_get_flow_file_not_found(fake_temp_repo, caplog):
storage = Git(repo="test/repo", flow_path="flow.py", flow_name="my-flow")
storage.add_flow(Flow("my-flow"))
with pytest.raises(FileNotFoundError):
flow = storage.get_flow("my-flow")
fake_temp_repo.assert_called_with(
branch_name=storage.branch_name,
clone_depth=storage.clone_depth,
git_clone_url=storage.git_clone_url,
tag=storage.tag,
commit=storage.commit,
)
def test_get_flow(fake_temp_repo, fake_extract_flow_from_file):
storage = Git(repo="test/repo", flow_path="flow.py", flow_name="my-flow")
storage.add_flow(Flow("my-flow"))
flow = storage.get_flow("my-flow")
fake_temp_repo.assert_called_with(
branch_name=storage.branch_name,
clone_depth=storage.clone_depth,
git_clone_url=storage.git_clone_url,
tag=storage.tag,
commit=storage.commit,
)
fake_extract_flow_from_file.assert_called_with(
file_path=f"/tmp/test/{storage.flow_path}", flow_name=storage.flow_name
)
def test_create_git_storage_init_args():
storage = Git(
flow_path="flow.py",
repo="test/repo",
repo_host="bitbucket.org",
flow_name="my-flow",
git_token_secret_name="MY_TOKEN",
git_token_username="******",
branch_name="my_branch",
tag=None,
commit=None,
clone_depth=1,
use_ssh=False,
format_access_token=True,
)
assert storage
assert storage.flows == dict()
assert storage.flow_path == "flow.py"
assert storage.repo == "test/repo"
assert storage.repo_host == "bitbucket.org"
assert storage.flow_name == "my-flow"
assert storage.git_token_secret_name == "MY_TOKEN"
assert storage.git_token_username == "my_user"
assert storage.branch_name == "my_branch"
assert storage.tag is None
assert storage.commit is None
assert storage.clone_depth == 1
assert storage.use_ssh == False
assert storage.format_access_token == True
def test_git_access_token_errors_if_provided_and_not_found(caplog):
storage = Git(
repo="test/repo", flow_path="flow.py", git_token_secret_name="MISSING"
)
with context(secrets={}):
with pytest.raises(Exception, match="MISSING"):
storage.git_token_secret
def test_get_git_clone_url(
repo_host,
repo,
access_token_secret_name,
access_token_secret,
access_token_username,
use_ssh,
format_access_token,
git_clone_url_secret_name,
git_clone_url_secret,
expected_git_clone_url,
):
storage = Git(
repo=repo,
flow_path="foo.py",
repo_host=repo_host,
git_token_secret_name=access_token_secret_name,
git_token_username=access_token_username,
git_clone_url_secret_name=git_clone_url_secret_name,
use_ssh=use_ssh,
format_access_token=format_access_token,
)
with context(
secrets={
access_token_secret_name: access_token_secret,
git_clone_url_secret_name: git_clone_url_secret,
}):
assert storage.git_clone_url == expected_git_clone_url
def test_create_git_storage_with_git_clone_url_secret_name_and_other_repo_params_logs_warning(
caplog, ):
storage = Git(
flow_path="flow.py",
repo="test/repo",
git_clone_url_secret_name="MY_SECRET",
)
assert (
"Git storage initialized with a `git_clone_url_secret_name`. The value of this Secret will be used to clone the repository, ignoring"
in caplog.text)
def test_create_git_storage_with_use_ssh_and_git_token_secret_name_logs_warning(caplog):
storage = Git(
flow_path="flow.py",
repo="test/repo",
git_token_secret_name="MY_SECRET",
use_ssh=True,
)
assert (
"Git Storage initialized with `use_ssh = True` and `git_token_secret_name` provided. SSH will be used to clone the repository. `git_token_secret_name` will be ignored"
in caplog.text
)
def test_create_git_storage_with_tag_and_branch_name_errors():
with pytest.raises(ValueError):
storage = Git(
flow_path="flow.py", repo="test/repo", branch_name="my_branch", tag="v0.1"
)
def test_create_git_storage():
storage = Git(repo="test/repo", flow_path="flow.py")
assert storage
assert storage.logger
logger = prefect.context.get("logger")
logger.info(
f"The answer to the Ultimate Question of Life, the Universe, and Everything is: {answer}"
)
@task
def say_bye():
logger = prefect.context.get("logger")
logger.info("Bye!")
STORAGE = Git(
repo_host="github.com",
repo="dyvenia/viadot",
flow_path="viadot/examples/hello_world.py",
branch_name="0.2.3",
git_token_secret_name=
"github_token", # name of the Prefect secret with the GitHub token
)
RUN_CONFIG = DockerRun(
image="prefecthq/prefect",
env={"SOME_VAR": "value"},
labels=["dev"],
)
with Flow("Hello, world!", storage=STORAGE, run_config=RUN_CONFIG) as flow:
hello = say_hello()
print_answer = show_answer()
bye = say_bye()
print_answer.set_upstream(hello, flow=flow)
def test_create_git_storage_without_repo_or_git_clone_url_secret_name_errors():
with pytest.raises(
ValueError,
match="Either `repo` or `git_clone_url_secret_name` must be provided",
):
storage = Git(flow_path="flow.py")