def add_engine_c(image, port, username, no_init, no_sgconfig, inject_source,
no_pull, name, password, set_default):
"""
Create and start a Splitgraph engine.
This will pull the Splitgraph engine image, start it, create a Postgres user and initialize
the engine.
This also creates Docker volumes required to persist data/metadata.
The engine Docker container by default will be named `splitgraph_engine_default` and
its data and metadata volumes will have names `splitgraph_engine_default_data` and
`splitgraph_engine_default_metadata`.
"""
from splitgraph.engine.postgres.engine import PostgresEngine
from splitgraph.config import CONFIG
from docker.types import Mount
client = get_docker_client()
if not no_pull:
click.echo("Pulling image %s..." % image)
_pretty_pull(client, image)
container_name = _get_container_name(name)
data_name = _get_data_volume_name(name)
metadata_name = _get_metadata_volume_name(name)
# Setup required mounts for data/metadata
data_volume = Mount(target="/var/lib/splitgraph/objects",
source=data_name,
type="volume")
metadata_volume = Mount(target="/var/lib/postgresql/data",
source=metadata_name,
type="volume")
mounts = [data_volume, metadata_volume]
click.echo("Creating container %s." % container_name)
click.echo("Data volume: %s." % data_name)
click.echo("Metadata volume: %s." % metadata_name)
if inject_source:
source_path = _convert_source_path(
os.getenv(
"SG_SOURCE_ROOT",
os.path.abspath(os.path.join(os.path.dirname(__file__),
"../"))))
source_volume = Mount(target="/splitgraph/splitgraph",
source=source_path,
type="bind")
mounts.append(source_volume)
click.echo("Source path: %s" % source_path)
container = client.containers.run(
image=image,
detach=True,
name=container_name,
ports={"5432/tcp": port},
mounts=mounts,
environment={
"POSTGRES_USER": username,
"POSTGRES_PASSWORD": password,
"POSTGRES_DB": "splitgraph",
# Actual config to be injected later
"SG_CONFIG_FILE": "/.sgconfig",
},
)
click.echo("Container created, ID %s" % container.short_id)
# Extract the host that we can reach the container on
# (might be different from localhost if docker-machine is used)
hostname = urlparse(client.api.base_url).hostname
conn_params: Dict[str, str] = {
"SG_ENGINE_HOST": hostname,
"SG_ENGINE_PORT": str(port),
# Even if the engine is exposed on a different port on the host,
# need to make sure that it uses the default 5432 port to connect
# to itself.
"SG_ENGINE_FDW_HOST": "localhost",
"SG_ENGINE_FDW_PORT": "5432",
"SG_ENGINE_USER": username,
"SG_ENGINE_PWD": password,
"SG_ENGINE_DB_NAME": "splitgraph",
"SG_ENGINE_POSTGRES_DB_NAME": "postgres",
"SG_ENGINE_ADMIN_USER": username,
"SG_ENGINE_ADMIN_PWD": password,
}
if not no_sgconfig:
if name != DEFAULT_ENGINE and not set_default:
config_patch = {"remotes": {name: conn_params}}
else:
config_patch = conn_params
config_path = patch_and_save_config(CONFIG, config_patch)
else:
config_path = CONFIG["SG_CONFIG_FILE"]
if not no_init:
engine = PostgresEngine(name=name, conn_params=conn_params)
engine.initialize()
engine.commit()
click.echo("Engine initialized successfully.")
inject_config_into_engines(CONFIG["SG_ENGINE_PREFIX"], config_path)
click.echo("Done.")
def write(self, value_: Any, **kwargs: Any) -> Result:
"""
Writes the result to a repository on Splitgraph
Args:
- value_ (Any): the value to write; will then be stored as the `value` attribute
of the returned `Result` instance
- **kwargs (optional): if provided, will be used to format the `table`, `comment`, and `tag`
Returns:
- Result: returns a new `Result` with both `value`, `comment`, `table`, and `tag` attributes
"""
cfg = patch_config(create_config_dict(), self.env or dict())
engine = PostgresEngine(name='SplitgraphResult', conn_params=cfg)
engine.initialize()
repo = Repository(namespace=self.namespace, repository=self.repo_name, engine=engine)
assert isinstance(value_, pd.DataFrame)
assert engine.connected
if not repository_exists(repo) and self.auto_init_repo:
self.logger.info("Creating repo {}/{}...".format(repo.namespace, repo.repository))
repo.init()
# TODO: Retrieve the repo from bedrock first
new = self.format(**kwargs)
new.value = value_
self.logger.info("Starting to upload result to {}...".format(new.table))
with self.atomic(engine):
self.logger.info("checkout")
img = repo.head
img.checkout(force=True)
self.logger.info("df to table")
df_to_table(new.value, repository=repo, table=new.table, if_exists='replace')
self.logger.info("commit")
new_img = repo.commit(comment=new.comment, chunk_size=10000)
new_img.tag(new.tag)
# if (repo.diff(new.table, img, new_img)):
if self.auto_push:
self.logger.info("push")
repo.push(
self.get_upstream(repo),
handler="S3",
overwrite_objects=True,
overwrite_tags=True,
reupload_objects=True,
)
engine.close()
self.logger.info("Finished uploading result to {}...".format(new.table))
return new