def project_init(ctx, overwrite, dbnd_home, dbnd_system):
"""Initialize the project structure"""
from dbnd._core.errors import DatabandSystemError
from dbnd import databand_lib_path
os.environ["SKIP_DAGS_PARSING"] = "True" # Exclude airflow dag examples
conf_folder = databand_lib_path("conf/project_init")
if os.path.exists(os.path.join(dbnd_home, "project.cfg")):
if not overwrite:
raise DatabandSystemError(
"You are trying to re-initialize your project. You already have dbnd configuration at %s. "
"You can force project-init by providing --overwrite flag. "
"If you need to create/update database use `dbnd db init` instead"
% dbnd_system
)
logger.warning(
"You are re-initializing your project, all files at %s are going to be over written!"
% dbnd_home
)
copy_tree(conf_folder, dbnd_home)
click.echo("Databand project has been initialized at %s" % dbnd_home)
return
def project_init(ctx, no_init_db, overwrite, dbnd_home, dbnd_system):
"""Initialize the project structure and local db"""
from dbnd._core.errors import DatabandSystemError
from dbnd import databand_lib_path
os.environ["SKIP_DAGS_PARSING"] = "True" # Exclude airflow dag examples
conf_folder = databand_lib_path("conf/project_init")
project_name = os.path.basename(dbnd_home)
output_dir = os.path.dirname(dbnd_home)
if os.path.exists(os.path.join(dbnd_home, "project.cfg")):
if not overwrite:
raise DatabandSystemError(
"You are trying to re-initialize your project. You already have dbnd configuration at %s. "
"You can force project-init by providing --overwrite flag. "
"If you need to create/update database use `dbnd db init` instead"
% dbnd_system)
logger.warning(
"You are re-initializing your project, all files at %s are going to be over written!"
% dbnd_home)
copy_tree(conf_folder, dbnd_home)
click.echo("Databand project has been initialized at %s" % dbnd_home)
config.load_system_configs(force=True)
if no_init_db:
click.echo("Don't forget to run `dbnd db init` ")
return
if is_web_enabled():
from dbnd_web.cmd_db import init as db_init
ctx.invoke(db_init)
import datetime
import logging
from time import sleep
from dbnd import config, databand_lib_path, override, pipeline, task
from dbnd.tasks.basics import dbnd_sanity_check
from dbnd_docker.docker.docker_task import DockerRunTask
logger = logging.getLogger(__name__)
@task(task_config=dict(kubernetes=dict(
pod_yaml=override(
databand_lib_path("conf", "kubernetes-pod-tensorflow.yaml")),
trap_exit_file_flag=override("/output/training_logs/main-terminated"),
limits=override({"nvidia.com/gpu": 1}),
)))
def task_with_custom_k8s_yml_gpu(
check_time=datetime.datetime.now(), sleep_time_sec=120):
# type: ( datetime.datetime, int)-> str
config.log_current_config(as_table=True)
logger.info("Running Kube Sanity Check!")
if sleep_time_sec:
logger.info("sleeping for %s", sleep_time_sec)
sleep(sleep_time_sec)
return "Databand checked at %s" % check_time
class ExampleDockerNativeTask(DockerRunTask):
command = "echo hi"