def test_makes_replica_name_safe():
shdocker = SnowShuDocker()
VALID_REP_NAMES = [
"Replica With Spaces", "replica-with-----dashes",
"--replica-lead-dashes", "replica.with.periods",
"1_Replica_____with___underscores"
]
INVALID_REP_NAMES = [
"invalid-©haracter", "$uper.i!!eg@l", "replica_i!!egal_char"
]
valid_result = [
shdocker.sanitize_replica_name(rep) for rep in VALID_REP_NAMES
]
assert valid_result == [
"snowshu_replica_replica-with-spaces",
"snowshu_replica_replica-with-dashes",
"snowshu_replica_replica-lead-dashes",
"snowshu_replica_replica-with-periods",
"snowshu_replica_1-replica-with-underscores"
]
for rep in INVALID_REP_NAMES:
with pytest.raises(ValueError):
shdocker.sanitize_replica_name(rep)
def finalize_replica(self) -> str:
"""returns the image name of the completed replica.
"""
shdocker = SnowShuDocker()
logger.info('Finalizing target container into replica...')
replica_image = shdocker.convert_container_to_replica(
self.replica_meta['name'], self.container, self)
logger.info(f'Finalized replica image {self.replica_meta["name"]}')
return replica_image.tags[0]
def test_remounts_data_in_replica():
container = mock.MagicMock()
container.exec_run.return_value = (
0,
'',
)
shdocker = SnowShuDocker()
shdocker._remount_replica_data(container, PostgresAdapter())
assert [arg for arg in container.exec_run.call_args_list
][0][0][0] == "/bin/bash -c 'mkdir /snowshu_replica_data'"
def _init_image(self, source_adapter_name: str) -> None:
shdocker = SnowShuDocker()
logger.info('Initializing target container...')
self.container = shdocker.startup(
self.DOCKER_IMAGE, self.DOCKER_START_COMMAND,
self.DOCKER_TARGET_PORT, self.CLASSNAME, source_adapter_name,
self._build_snowshu_envars(self.DOCKER_SNOWSHU_ENVARS))
logger.info('Container initialized.')
while not self.target_database_is_ready():
sleep(.5)
self._initialize_snowshu_meta_database()
def list():
shdocker = SnowShuDocker()
images = shdocker.find_snowshu_images()
if len(images) < 1:
return "\n\nNo SnowShu replicas found.\n\
You can create a new replica by running `snowshu create`.\n\n"
collection = [(
shdocker.replica_image_name_to_common_name(img.tags[0]),
datetime.strftime(parse(img.attrs['Metadata']['LastTagTime']),
"%Y-%m-%d %H:%M:%S"),
img.labels['source_adapter'],
img.labels['target_adapter'],
img.tags[0],
) for img in images]
return format_set_of_available_images(collection)
def launch_docker_command(replica: str) -> str:
"""Finds the replica and returns a docker run command to launch it.
Args:
replica: the common name of the replica, ie "integration-test" for image "snowshu_replica_integration-test".
Returns:
The docker command to run a detached replica mounted to port 9999.
"""
shdocker = SnowShuDocker()
images = shdocker.find_snowshu_images()
cmd_string = 'docker run -d -p 9999:9999 --rm --name {} {}'
for image in images:
image_name = image.tags[0]
image_without_registry_or_tag = (image_name.split(
os.path.sep)[-1]).split(':')[0]
if shdocker.sanitize_replica_name(
replica) == image_without_registry_or_tag:
return cmd_string.format(replica, image_name)
return f'No replica found for {replica}.'
def test_creates_replica(docker_flush):
# build image
# load it up with some data
# convert it to a replica
# spin it all down
# start the replica
# query it and confirm that the data is in there
shdocker = SnowShuDocker()
target_adapter = PostgresAdapter(replica_metadata={})
target_container = shdocker.startup(
target_adapter.DOCKER_IMAGE, target_adapter.DOCKER_START_COMMAND, 9999,
target_adapter, 'SnowflakeAdapter', [
'POSTGRES_USER=snowshu', 'POSTGRES_PASSWORD=snowshu',
'POSTGRES_DB=snowshu', f'PGDATA=/{DOCKER_REMOUNT_DIRECTORY}'
])
# load test data
time.sleep(
DOCKER_SPIN_UP_TIMEOUT) # give pg a moment to spin up all the way
engine = create_engine(
'postgresql://*****:*****@snowshu_target:9999/snowshu')
engine.execute(
f'CREATE TABLE {TEST_TABLE} (column_one VARCHAR, column_two INT)')
engine.execute(
f"INSERT INTO {TEST_TABLE} VALUES ('a',1), ('b',2), ('c',3)")
checkpoint = engine.execute(f"SELECT * FROM {TEST_TABLE}").fetchall()
assert ('a', 1) == checkpoint[0]
replica = shdocker.convert_container_to_replica(TEST_NAME,
target_container)
# get a new replica
client = docker.from_env()
client.containers.run(replica.id,
ports={'9999/tcp': 9999},
name=TEST_NAME,
network='snowshu',
detach=True)
time.sleep(
DOCKER_SPIN_UP_TIMEOUT) # give pg a moment to spin up all the way
engine = create_engine(
f'postgresql://*****:*****@{TEST_NAME}:9999/snowshu')
res = engine.execute(f'SELECT * FROM {TEST_TABLE}').fetchall()
assert (
'a',
1,
) in res
assert (
'b',
2,
) in res
assert (
'c',
3,
) in res
# verify that the extra OS packages are installed
res = engine.execute("create extension plpython3u;")
shdocker.remove_container(TEST_NAME)
def test_using_different_image(end_to_end):
shdocker = SnowShuDocker()
target_adapter = PostgresAdapter(replica_metadata={})
envars = [
'POSTGRES_USER=snowshu', 'POSTGRES_PASSWORD=snowshu',
'POSTGRES_DB=snowshu', f'PGDATA=/{DOCKER_REMOUNT_DIRECTORY}'
]
target_container = shdocker.get_stopped_container(
'snowshu_replica_integration-test',
target_adapter.DOCKER_START_COMMAND,
envars,
9900,
name=DOCKER_TARGET_CONTAINER,
labels=dict(snowshu_replica='true',
target_adapter=target_adapter.CLASSNAME,
source_adapter='SnowflakeAdapter'))
assert target_container.status == 'created'
assert target_container.image.tags[
0] == 'snowshu_replica_integration-test:latest'
target_container.start()
target_container.reload()
assert target_container.status == 'running'
target_container.remove(force=True)
def initialize_replica(
self,
source_adapter_name: str,
override_image: str = None
) -> None: # noqa pylint:disable=too-many-branches
"""shimming but will want to move _init_image public with this
interface.
Args:
source_adapter_name: the classname of the source adapter
override_image: the name of incremental image to initialize,
if specified will override default image
"""
if override_image:
try:
shdocker = SnowShuDocker()
images = shdocker.client.images.list(name=override_image)
logger.debug(
f"List of images found with name {override_image}: {images}"
)
image_commands = []
for item in images[0].history():
if ("postgres"
in item["CreatedBy"]) or ("PGDATA"
in item["CreatedBy"]):
image_commands.append(item["CreatedBy"])
if len(image_commands) > 0:
self.__class__.DOCKER_IMAGE = override_image
else:
logger.error(
f"The override image is not a Postgres image: {override_image}"
)
raise Exception(
f"The override image is not a Postgres image: {override_image}"
)
except Exception as error:
logger.error(
"Looks like provided DOCKER_IMAGE does not exists, error:\n%s",
error)
raise error
self._init_image(source_adapter_name)