def test_to_and_from_file(self):
env = LocalEnvironment().build(error_flow())
with tempfile.NamedTemporaryFile() as tmp:
env.to_file(tmp.name)
new_env = from_file(tmp.name)
assert isinstance(new_env, LocalEnvironment)
assert new_env.encryption_key == env.encryption_key
assert new_env.serialized_flow == env.serialized_flow
def create_dockerfile(self, flow: "prefect.Flow", directory: str = None) -> None:
"""Creates a dockerfile to use as the container.
In order for the docker python library to build a container it needs a
Dockerfile that it can use to define the container. This function takes the
image and python_dependencies then writes them to a file called Dockerfile.
*Note*: if `files` are added to this container, they will be copied to this directory as well.
Args:
- flow (Flow): the flow that the container will run
- directory (str, optional): A directory where the Dockerfile will be created,
if no directory is specified is will be created in the current working directory
"""
directory = directory or "./"
with open(os.path.join(directory, "Dockerfile"), "w+") as dockerfile:
# Generate RUN pip install commands for python dependencies
pip_installs = ""
if self.python_dependencies:
for dependency in self.python_dependencies:
pip_installs += "RUN pip install {}\n".format(dependency)
env_vars = ""
if self.env_vars:
white_space = " " * 20
env_vars = "ENV " + " \ \n{}".format(white_space).join(
"{k}={v}".format(k=k, v=v) for k, v in self.env_vars.items()
)
copy_files = ""
if self.files:
for src, dest in self.files.items():
fname = os.path.basename(src)
full_fname = os.path.join(directory, fname)
if (
os.path.exists(full_fname)
and filecmp.cmp(src, full_fname) is False
):
raise ValueError(
"File {fname} already exists in {directory}".format(
fname=full_fname, directory=directory
)
)
else:
shutil.copy2(src, full_fname)
copy_files += "COPY {fname} {dest}\n".format(fname=fname, dest=dest)
# Create a LocalEnvironment to run the flow
# the local environment will be placed in the container and run when the container
# runs
local_environment = LocalEnvironment().build(flow=flow)
flow_path = os.path.join(directory, "flow_env.prefect")
local_environment.to_file(flow_path)
# write a healthcheck script into the image
healthcheck = textwrap.dedent(
"""\
print('Beginning health check...')
from prefect.utilities.environments import from_file
local_env = from_file('/root/.prefect/flow_env.prefect')
flow = local_env.deserialize_flow_from_bytes(local_env.serialized_flow)
print('Healthcheck: OK')
"""
)
with open(os.path.join(directory, "healthcheck.py"), "w") as health_file:
health_file.write(healthcheck)
file_contents = textwrap.dedent(
"""\
FROM {base_image}
RUN apt-get -qq -y update && apt-get -qq -y install --no-install-recommends --no-install-suggests git
RUN pip install pip --upgrade
RUN pip install wheel
{pip_installs}
RUN mkdir /root/.prefect/
COPY flow_env.prefect /root/.prefect/flow_env.prefect
COPY healthcheck.py /root/.prefect/healthcheck.py
{copy_files}
ENV PREFECT_ENVIRONMENT_FILE="/root/.prefect/flow_env.prefect"
ENV PREFECT__USER_CONFIG_PATH="/root/.prefect/config.toml"
{env_vars}
RUN pip install prefect
RUN python /root/.prefect/healthcheck.py
""".format(
base_image=self.base_image,
pip_installs=pip_installs,
copy_files=copy_files,
env_vars=env_vars,
access_token=os.getenv("PERSONAL_ACCESS_TOKEN"),
)
)
dockerfile.write(file_contents)