def test_prepare_docker_with_py_and_dependency(self):
""" Test the whole prepare docker from python function and dependencies """
# prepare
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
python_filepath = os.path.join(test_data_dir, 'basic.py')
local_tarball_path = os.path.join(test_data_dir, 'test_docker.tar.gz')
# check
docker_helper = DockerfileHelper(arc_dockerfile_name='dockerfile')
dependencies = {
VersionedDependency(name='tensorflow',
min_version='0.10.0',
max_version='0.11.0'),
VersionedDependency(name='kubernetes', min_version='0.6.0'),
}
docker_helper.prepare_docker_tarball_with_py(
arc_python_filename='main.py',
python_filepath=python_filepath,
base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.8.0',
local_tarball_path=local_tarball_path,
dependency=dependencies)
temp_tarball_handle = tarfile.open(local_tarball_path)
temp_files = temp_tarball_handle.getmembers()
self.assertTrue(len(temp_files) == 3)
for temp_file in temp_files:
self.assertTrue(temp_file.name in
['dockerfile', 'main.py', 'requirements.txt'])
# clean up
os.remove(local_tarball_path)
def test_generate_dockerfile(self):
""" Test generate dockerfile """
# prepare
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
target_dockerfile = os.path.join(test_data_dir,
'component.temp.dockerfile')
golden_dockerfile_payload = '''\
FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0
RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools
RUN pip3 install fire
ADD main.py /ml/
ENTRYPOINT ["python3", "/ml/main.py"]'''
# check
docker_helper = DockerfileHelper(arc_dockerfile_name=target_dockerfile,
gcs_path='')
docker_helper._generate_dockerfile_with_py(
target_file=target_dockerfile,
base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0',
python_filepath='main.py')
with open(target_dockerfile, 'r') as f:
target_dockerfile_payload = f.read()
self.assertEqual(target_dockerfile_payload, golden_dockerfile_payload)
# clean up
os.remove(target_dockerfile)
def test_wrap_files_in_tarball(self):
""" Test wrap files in a tarball """
# prepare
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
temp_file_one = os.path.join(test_data_dir, 'test_data_one.tmp')
temp_file_two = os.path.join(test_data_dir, 'test_data_two.tmp')
temp_tarball = os.path.join(test_data_dir, 'test_data.tmp.tar.gz')
with open(temp_file_one, 'w') as f:
f.write('temporary file one content')
with open(temp_file_two, 'w') as f:
f.write('temporary file two content')
# check
docker_helper = DockerfileHelper(arc_dockerfile_name='')
docker_helper._wrap_files_in_tarball(temp_tarball, {
'dockerfile': temp_file_one,
'main.py': temp_file_two
})
self.assertTrue(os.path.exists(temp_tarball))
temp_tarball_handle = tarfile.open(temp_tarball)
temp_files = temp_tarball_handle.getmembers()
self.assertTrue(len(temp_files) == 2)
for temp_file in temp_files:
self.assertTrue(temp_file.name in ['dockerfile', 'main.py'])
# clean up
os.remove(temp_file_one)
os.remove(temp_file_two)
os.remove(temp_tarball)
def test_prepare_docker_tarball(self):
""" Test the whole prepare docker tarball """
# prepare
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
dockerfile_path = os.path.join(test_data_dir,
'component.target.dockerfile')
Path(dockerfile_path).touch()
downloaded_tarball = os.path.join(test_data_dir, 'test_docker.tar.gz')
gcs_tar_path = os.path.join(GCS_BASE, 'test_docker.tar.gz')
# check
docker_helper = DockerfileHelper(arc_dockerfile_name='dockerfile',
gcs_path=gcs_tar_path)
docker_helper.prepare_docker_tarball(dockerfile_path=dockerfile_path)
GCSHelper.download_gcs_blob(downloaded_tarball, gcs_tar_path)
temp_tarball_handler = tarfile.open(downloaded_tarball)
temp_files = temp_tarball_handler.getmembers()
self.assertTrue(len(temp_files) == 1)
for temp_file in temp_files:
self.assertTrue(temp_file.name in ['dockerfile'])
# clean up
os.remove(downloaded_tarball)
os.remove(dockerfile_path)
GCSHelper.remove_gcs_blob(gcs_tar_path)
def test_prepare_docker_with_py(self):
""" Test the whole prepare docker from python function """
# prepare
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
python_filepath = os.path.join(test_data_dir, 'basic.py')
downloaded_tarball = os.path.join(test_data_dir, 'test_docker.tar.gz')
gcs_tar_path = os.path.join(GCS_BASE, 'test_docker.tar.gz')
# check
docker_helper = DockerfileHelper(arc_dockerfile_name='dockerfile',
gcs_path=gcs_tar_path)
docker_helper.prepare_docker_tarball_with_py(
arc_python_filename='main.py',
python_filepath=python_filepath,
base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.8.0')
GCSHelper.download_gcs_blob(downloaded_tarball, gcs_tar_path)
temp_tarball_handler = tarfile.open(downloaded_tarball)
temp_files = temp_tarball_handler.getmembers()
self.assertTrue(len(temp_files) == 2)
for temp_file in temp_files:
self.assertTrue(temp_file.name in ['dockerfile', 'main.py'])
# clean up
os.remove(downloaded_tarball)
GCSHelper.remove_gcs_blob(gcs_tar_path)
def test_prepare_docker_tarball(self):
""" Test the whole prepare docker tarball """
# prepare
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
dockerfile_path = os.path.join(test_data_dir, 'component.target.dockerfile')
Path(dockerfile_path).touch()
generated_tarball = os.path.join(test_data_dir, 'test_docker.tar.gz')
# check
docker_helper = DockerfileHelper(arc_dockerfile_name='dockerfile')
docker_helper.prepare_docker_tarball(dockerfile_path=dockerfile_path, local_tarball_path=generated_tarball)
temp_tarball_handler = tarfile.open(generated_tarball)
temp_files = temp_tarball_handler.getmembers()
self.assertTrue(len(temp_files) == 1)
for temp_file in temp_files:
self.assertTrue(temp_file.name in ['dockerfile'])
# clean up
os.remove(generated_tarball)
os.remove(dockerfile_path)
def test_prepare_docker_with_py(self):
""" Test the whole prepare docker from python function """
# prepare
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
python_filepath = os.path.join(test_data_dir, 'basic.py')
local_tarball_path = os.path.join(test_data_dir, 'test_docker.tar.gz')
# check
docker_helper = DockerfileHelper(arc_dockerfile_name='dockerfile')
docker_helper.prepare_docker_tarball_with_py(
arc_python_filename='main.py',
python_filepath=python_filepath,
base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.8.0',
local_tarball_path=local_tarball_path,
python_version='python3')
with tarfile.open(local_tarball_path) as temp_tarball_handle:
temp_files = temp_tarball_handle.getmembers()
self.assertTrue(len(temp_files) == 2)
for temp_file in temp_files:
self.assertTrue(temp_file.name in ['dockerfile', 'main.py'])
# clean up
os.remove(local_tarball_path)
def test_generate_dockerfile(self):
""" Test generate dockerfile """
# prepare
test_data_dir = os.path.join(os.path.dirname(__file__), 'testdata')
target_dockerfile = os.path.join(test_data_dir,
'component.temp.dockerfile')
golden_dockerfile_payload_one = '''\
FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0
RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools
ADD main.py /ml/
ENTRYPOINT ["python3", "/ml/main.py"]'''
golden_dockerfile_payload_two = '''\
FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0
RUN apt-get update -y && apt-get install --no-install-recommends -y -q python3 python3-pip python3-setuptools
ADD requirements.txt /ml/
RUN pip3 install -r /ml/requirements.txt
ADD main.py /ml/
ENTRYPOINT ["python3", "/ml/main.py"]'''
golden_dockerfile_payload_three = '''\
FROM gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0
RUN apt-get update -y && apt-get install --no-install-recommends -y -q python python-pip python-setuptools
ADD requirements.txt /ml/
RUN pip install -r /ml/requirements.txt
ADD main.py /ml/
ENTRYPOINT ["python", "/ml/main.py"]'''
# check
docker_helper = DockerfileHelper(arc_dockerfile_name=target_dockerfile)
docker_helper._generate_dockerfile_with_py(
target_file=target_dockerfile,
base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0',
python_filepath='main.py',
has_requirement_file=False,
python_version='python3')
with open(target_dockerfile, 'r') as f:
target_dockerfile_payload = f.read()
self.assertEqual(target_dockerfile_payload,
golden_dockerfile_payload_one)
docker_helper._generate_dockerfile_with_py(
target_file=target_dockerfile,
base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0',
python_filepath='main.py',
has_requirement_file=True,
python_version='python3')
with open(target_dockerfile, 'r') as f:
target_dockerfile_payload = f.read()
self.assertEqual(target_dockerfile_payload,
golden_dockerfile_payload_two)
docker_helper._generate_dockerfile_with_py(
target_file=target_dockerfile,
base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0',
python_filepath='main.py',
has_requirement_file=True,
python_version='python2')
with open(target_dockerfile, 'r') as f:
target_dockerfile_payload = f.read()
self.assertEqual(target_dockerfile_payload,
golden_dockerfile_payload_three)
self.assertRaises(
ValueError,
docker_helper._generate_dockerfile_with_py,
target_file=target_dockerfile,
base_image='gcr.io/ngao-mlpipeline-testing/tensorflow:1.10.0',
python_filepath='main.py',
has_requirement_file=True,
python_version='python4')
# clean up
os.remove(target_dockerfile)