def get_project_git():
global _project_git_version
if is_defined(_project_git_version):
return _project_git_version
_project_git_version = get_git_commit(project_path(), verbose=is_verbose())
return _project_git_version
class TPySpark(SparkTask):
t_param = data
t_param2 = parameter[int]
python_script = project_path("foo.py")
t_output = output
def run_using_kaniko(self):
if self.tag:
self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag)
else:
self.image_name_with_tag = self.full_image_name
command = "{} -c {} -f {}".format(
self.kaniko_command, self.context, self.docker_file
)
if not self.destinations:
command = command + " --no-push"
else:
destination_list = [
" -d {}".format(destination) for destination in self.destinations
]
command = command + "".join(destination_list)
if self.build_args:
build_args_list = [" --build-arg {}".format(arg) for arg in self.build_args]
command = command + "".join(build_args_list)
if self.label:
command = command + " --label " + self.label
if self.target:
command = command + " --target " + self.target
try:
logger.info("Running build using Kaniko: %s", command)
run_cmd(command, shell=True, cwd=project_path())
except Exception as e:
raise DatabandRuntimeError(
"failed building docker image {}".format("?"), nested_exceptions=[e]
)
def build_code_detector(cls, system_code_dirs=None):
# should be called withing databand environment
if system_code_dirs is None:
system_code_dirs = [
databand_lib_path(),
databand_lib_path("../targets")
]
code_dir = project_path()
return cls(code_dir=code_dir, system_code_dirs=system_code_dirs)
class DockerBuild(Task):
docker_file = parameter(default=project_path("Dockerfile"))[str]
image_name = parameter()[str]
tag = parameter(default="latest")[str]
label = parameter(default="")[str]
push = parameter(default=True)[bool]
target = parameter(default=None)[str]
working_dir = parameter(default=None)[str]
full_image_name = None
computed_tag = None
image_name_with_tag = None
def run(self):
if self.tag:
self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag)
else:
self.image_name_with_tag = self.full_image_name
try:
cmd = "docker build -t {} -f {} .".format(
self.image_name_with_tag, self.docker_file
)
if self.label:
cmd = cmd + " --label " + self.label
if self.target:
cmd = cmd + " --target " + self.target
logger.info("Running docker build: %s", cmd)
cwd = self.working_dir or project_path()
run_cmd(cmd, shell=True, cwd=cwd)
except Exception as e:
raise DatabandRuntimeError(
"failed building docker image {}".format(self.image_name_with_tag),
nested_exceptions=[e],
)
if self.push:
try:
cmd = "docker push {}".format(self.image_name_with_tag)
logger.info("Running docker push: '%s'", cmd)
run_cmd(cmd, shell=True)
except Exception as e:
raise DatabandRuntimeError(
"failed to push docker image {}".format(self.image_name_with_tag),
nested_exceptions=[e],
)
else:
logger.info("skipping docker push")
return self.image_name_with_tag
def run_using_docker_build(self):
if self.tag:
self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag)
else:
self.image_name_with_tag = self.full_image_name
try:
cmd = "docker build -t {} -f {} .".format(
self.image_name_with_tag, self.docker_file
)
if self.label:
cmd = cmd + " --label " + self.label
if self.target:
cmd = cmd + " --target " + self.target
if self.build_args:
build_args_list = [
" --build-arg {}".format(arg) for arg in self.build_args
]
cmd = cmd + "".join(build_args_list)
logger.info("Running docker build command: `%s`\n\n", cmd)
cwd = self.working_dir or project_path()
run_cmd(cmd, shell=True, cwd=cwd)
except Exception as e:
logger.error(
"^^^^^^^^^^^^^^^ SEE DOCKER ERROR MESSAGE ABOVE THIS LINE ^^^^^^^^^^^^^^^\n\n"
)
raise DatabandRuntimeError(
"failed building docker image {}".format(self.image_name_with_tag),
nested_exceptions=[e],
)
if self.push:
try:
cmd = "docker push {}".format(self.image_name_with_tag)
logger.info("Running docker push command: '%s'", cmd)
run_cmd(cmd, shell=True)
except Exception as e:
raise DatabandRuntimeError(
"failed to push docker image {}".format(self.image_name_with_tag),
nested_exceptions=[e],
)
else:
logger.info("skipping docker push")
return self.image_name_with_tag
def run_dbnd_subprocess_test(*args, **kwargs):
logger.error(
" cwd='%s'," " sys.path=\n\t%s" "\nPYTHONPATH=%s",
os.getcwd(),
"\n\t".join(sys.path),
os.environ.get("PYTHONPATH"),
)
env = os.environ.copy()
env["PYTHONPATH"] = ":".join(
[
env.get("PYTHONPATH", ""),
project_path("modules", "dbnd"),
] # we add current project so we can import test_dbnd
)
return run_dbnd_subprocess__dbnd_run(*args, module=factories, env=env, **kwargs)
def build_task_run_info(self):
task_run_env_uid = get_uuid()
import dbnd
logging.debug("Created new task run env with uid '%s'", task_run_env_uid)
machine = environ.get(ENV_DBND__ENV_MACHINE, "")
if environ.get(ENV_DBND__ENV_IMAGE, None):
machine += " image=%s" % environ.get(ENV_DBND__ENV_IMAGE)
return TaskRunEnvInfo(
uid=task_run_env_uid,
databand_version=dbnd.__version__,
user_code_version=self.source_version,
user_code_committed=True,
cmd_line=subprocess.list2cmdline(sys.argv),
user=self.user or dbnd_getuser(),
machine=machine,
project_root=project_path(),
user_data=safe_string(self.user_data, max_value_len=500),
heartbeat=utcnow(),
)
def run(self):
if self.tag:
self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag)
else:
self.image_name_with_tag = self.full_image_name
try:
cmd = "docker build -t {} -f {} .".format(
self.image_name_with_tag, self.docker_file
)
if self.label:
cmd = cmd + " --label " + self.label
if self.target:
cmd = cmd + " --target " + self.target
logger.info("Running docker build: %s", cmd)
cwd = self.working_dir or project_path()
run_cmd(cmd, shell=True, cwd=cwd)
except Exception as e:
raise DatabandRuntimeError(
"failed building docker image {}".format(self.image_name_with_tag),
nested_exceptions=[e],
)
if self.push:
try:
cmd = "docker push {}".format(self.image_name_with_tag)
logger.info("Running docker push: '%s'", cmd)
run_cmd(cmd, shell=True)
except Exception as e:
raise DatabandRuntimeError(
"failed to push docker image {}".format(self.image_name_with_tag),
nested_exceptions=[e],
)
else:
logger.info("skipping docker push")
return self.image_name_with_tag
class DockerBuild(Task):
use_kaniko = parameter(default=False)[bool]
docker_file = parameter(default=project_path("Dockerfile"))[str]
image_name = parameter()[str]
tag = parameter(default="latest")[str]
label = parameter(default="")[str]
push = parameter(default=True)[bool]
target = parameter(default=None)[str]
working_dir = parameter(default=None)[str]
kaniko_command = parameter(default=None)[str]
context = parameter(default=None)[str]
destinations = parameter(default=None)[list]
build_args = parameter(default=None)[list]
full_image_name = None
computed_tag = None
image_name_with_tag = None
def run(self):
if self.use_kaniko:
return self.run_using_kaniko()
else:
return self.run_using_docker_build()
def run_using_docker_build(self):
if self.tag:
self.image_name_with_tag = "{}:{}".format(self.image_name,
self.tag)
else:
self.image_name_with_tag = self.full_image_name
try:
cmd = "docker build -t {} -f {} .".format(self.image_name_with_tag,
self.docker_file)
if self.label:
cmd = cmd + " --label " + self.label
if self.target:
cmd = cmd + " --target " + self.target
if self.build_args:
build_args_list = [
" --build-arg {}".format(arg) for arg in self.build_args
]
cmd = cmd + "".join(build_args_list)
logger.info("Running docker build: %s", cmd)
cwd = self.working_dir or project_path()
run_cmd(cmd, shell=True, cwd=cwd)
except Exception as e:
raise DatabandRuntimeError(
"failed building docker image {}".format(
self.image_name_with_tag),
nested_exceptions=[e],
)
if self.push:
try:
cmd = "docker push {}".format(self.image_name_with_tag)
logger.info("Running docker push: '%s'", cmd)
run_cmd(cmd, shell=True)
except Exception as e:
raise DatabandRuntimeError(
"failed to push docker image {}".format(
self.image_name_with_tag),
nested_exceptions=[e],
)
else:
logger.info("skipping docker push")
return self.image_name_with_tag
def run_using_kaniko(self):
if self.tag:
self.image_name_with_tag = "{}:{}".format(self.image_name,
self.tag)
else:
self.image_name_with_tag = self.full_image_name
command = "{} -c {} -f {}".format(self.kaniko_command, self.context,
self.docker_file)
if not self.destinations:
command = command + " --no-push"
else:
destination_list = [
" -d {}".format(destination)
for destination in self.destinations
]
command = command + "".join(destination_list)
if self.build_args:
build_args_list = [
" --build-arg {}".format(arg) for arg in self.build_args
]
command = command + "".join(build_args_list)
if self.label:
command = command + " --label " + self.label
if self.target:
command = command + " --target " + self.target
try:
logger.info("Running build using Kaniko: %s", command)
run_cmd(command, shell=True, cwd=project_path())
except Exception as e:
raise DatabandRuntimeError(
"failed building docker image {}".format("?"),
nested_exceptions=[e])
def sample_path(*path):
return project_path("tests/dbnd_airflow/targets_tests/performance/data",
*path)
