def get_project_git(): global _project_git_version if is_defined(_project_git_version): return _project_git_version _project_git_version = get_git_commit(project_path(), verbose=is_verbose()) return _project_git_version
class TPySpark(SparkTask): t_param = data t_param2 = parameter[int] python_script = project_path("foo.py") t_output = output
def run_using_kaniko(self): if self.tag: self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag) else: self.image_name_with_tag = self.full_image_name command = "{} -c {} -f {}".format( self.kaniko_command, self.context, self.docker_file ) if not self.destinations: command = command + " --no-push" else: destination_list = [ " -d {}".format(destination) for destination in self.destinations ] command = command + "".join(destination_list) if self.build_args: build_args_list = [" --build-arg {}".format(arg) for arg in self.build_args] command = command + "".join(build_args_list) if self.label: command = command + " --label " + self.label if self.target: command = command + " --target " + self.target try: logger.info("Running build using Kaniko: %s", command) run_cmd(command, shell=True, cwd=project_path()) except Exception as e: raise DatabandRuntimeError( "failed building docker image {}".format("?"), nested_exceptions=[e] )
def build_code_detector(cls, system_code_dirs=None): # should be called withing databand environment if system_code_dirs is None: system_code_dirs = [ databand_lib_path(), databand_lib_path("../targets") ] code_dir = project_path() return cls(code_dir=code_dir, system_code_dirs=system_code_dirs)
class DockerBuild(Task): docker_file = parameter(default=project_path("Dockerfile"))[str] image_name = parameter()[str] tag = parameter(default="latest")[str] label = parameter(default="")[str] push = parameter(default=True)[bool] target = parameter(default=None)[str] working_dir = parameter(default=None)[str] full_image_name = None computed_tag = None image_name_with_tag = None def run(self): if self.tag: self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag) else: self.image_name_with_tag = self.full_image_name try: cmd = "docker build -t {} -f {} .".format( self.image_name_with_tag, self.docker_file ) if self.label: cmd = cmd + " --label " + self.label if self.target: cmd = cmd + " --target " + self.target logger.info("Running docker build: %s", cmd) cwd = self.working_dir or project_path() run_cmd(cmd, shell=True, cwd=cwd) except Exception as e: raise DatabandRuntimeError( "failed building docker image {}".format(self.image_name_with_tag), nested_exceptions=[e], ) if self.push: try: cmd = "docker push {}".format(self.image_name_with_tag) logger.info("Running docker push: '%s'", cmd) run_cmd(cmd, shell=True) except Exception as e: raise DatabandRuntimeError( "failed to push docker image {}".format(self.image_name_with_tag), nested_exceptions=[e], ) else: logger.info("skipping docker push") return self.image_name_with_tag
def run_using_docker_build(self): if self.tag: self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag) else: self.image_name_with_tag = self.full_image_name try: cmd = "docker build -t {} -f {} .".format( self.image_name_with_tag, self.docker_file ) if self.label: cmd = cmd + " --label " + self.label if self.target: cmd = cmd + " --target " + self.target if self.build_args: build_args_list = [ " --build-arg {}".format(arg) for arg in self.build_args ] cmd = cmd + "".join(build_args_list) logger.info("Running docker build command: `%s`\n\n", cmd) cwd = self.working_dir or project_path() run_cmd(cmd, shell=True, cwd=cwd) except Exception as e: logger.error( "^^^^^^^^^^^^^^^ SEE DOCKER ERROR MESSAGE ABOVE THIS LINE ^^^^^^^^^^^^^^^\n\n" ) raise DatabandRuntimeError( "failed building docker image {}".format(self.image_name_with_tag), nested_exceptions=[e], ) if self.push: try: cmd = "docker push {}".format(self.image_name_with_tag) logger.info("Running docker push command: '%s'", cmd) run_cmd(cmd, shell=True) except Exception as e: raise DatabandRuntimeError( "failed to push docker image {}".format(self.image_name_with_tag), nested_exceptions=[e], ) else: logger.info("skipping docker push") return self.image_name_with_tag
def run_dbnd_subprocess_test(*args, **kwargs): logger.error( " cwd='%s'," " sys.path=\n\t%s" "\nPYTHONPATH=%s", os.getcwd(), "\n\t".join(sys.path), os.environ.get("PYTHONPATH"), ) env = os.environ.copy() env["PYTHONPATH"] = ":".join( [ env.get("PYTHONPATH", ""), project_path("modules", "dbnd"), ] # we add current project so we can import test_dbnd ) return run_dbnd_subprocess__dbnd_run(*args, module=factories, env=env, **kwargs)
def build_task_run_info(self): task_run_env_uid = get_uuid() import dbnd logging.debug("Created new task run env with uid '%s'", task_run_env_uid) machine = environ.get(ENV_DBND__ENV_MACHINE, "") if environ.get(ENV_DBND__ENV_IMAGE, None): machine += " image=%s" % environ.get(ENV_DBND__ENV_IMAGE) return TaskRunEnvInfo( uid=task_run_env_uid, databand_version=dbnd.__version__, user_code_version=self.source_version, user_code_committed=True, cmd_line=subprocess.list2cmdline(sys.argv), user=self.user or dbnd_getuser(), machine=machine, project_root=project_path(), user_data=safe_string(self.user_data, max_value_len=500), heartbeat=utcnow(), )
def run(self): if self.tag: self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag) else: self.image_name_with_tag = self.full_image_name try: cmd = "docker build -t {} -f {} .".format( self.image_name_with_tag, self.docker_file ) if self.label: cmd = cmd + " --label " + self.label if self.target: cmd = cmd + " --target " + self.target logger.info("Running docker build: %s", cmd) cwd = self.working_dir or project_path() run_cmd(cmd, shell=True, cwd=cwd) except Exception as e: raise DatabandRuntimeError( "failed building docker image {}".format(self.image_name_with_tag), nested_exceptions=[e], ) if self.push: try: cmd = "docker push {}".format(self.image_name_with_tag) logger.info("Running docker push: '%s'", cmd) run_cmd(cmd, shell=True) except Exception as e: raise DatabandRuntimeError( "failed to push docker image {}".format(self.image_name_with_tag), nested_exceptions=[e], ) else: logger.info("skipping docker push") return self.image_name_with_tag
class DockerBuild(Task): use_kaniko = parameter(default=False)[bool] docker_file = parameter(default=project_path("Dockerfile"))[str] image_name = parameter()[str] tag = parameter(default="latest")[str] label = parameter(default="")[str] push = parameter(default=True)[bool] target = parameter(default=None)[str] working_dir = parameter(default=None)[str] kaniko_command = parameter(default=None)[str] context = parameter(default=None)[str] destinations = parameter(default=None)[list] build_args = parameter(default=None)[list] full_image_name = None computed_tag = None image_name_with_tag = None def run(self): if self.use_kaniko: return self.run_using_kaniko() else: return self.run_using_docker_build() def run_using_docker_build(self): if self.tag: self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag) else: self.image_name_with_tag = self.full_image_name try: cmd = "docker build -t {} -f {} .".format(self.image_name_with_tag, self.docker_file) if self.label: cmd = cmd + " --label " + self.label if self.target: cmd = cmd + " --target " + self.target if self.build_args: build_args_list = [ " --build-arg {}".format(arg) for arg in self.build_args ] cmd = cmd + "".join(build_args_list) logger.info("Running docker build: %s", cmd) cwd = self.working_dir or project_path() run_cmd(cmd, shell=True, cwd=cwd) except Exception as e: raise DatabandRuntimeError( "failed building docker image {}".format( self.image_name_with_tag), nested_exceptions=[e], ) if self.push: try: cmd = "docker push {}".format(self.image_name_with_tag) logger.info("Running docker push: '%s'", cmd) run_cmd(cmd, shell=True) except Exception as e: raise DatabandRuntimeError( "failed to push docker image {}".format( self.image_name_with_tag), nested_exceptions=[e], ) else: logger.info("skipping docker push") return self.image_name_with_tag def run_using_kaniko(self): if self.tag: self.image_name_with_tag = "{}:{}".format(self.image_name, self.tag) else: self.image_name_with_tag = self.full_image_name command = "{} -c {} -f {}".format(self.kaniko_command, self.context, self.docker_file) if not self.destinations: command = command + " --no-push" else: destination_list = [ " -d {}".format(destination) for destination in self.destinations ] command = command + "".join(destination_list) if self.build_args: build_args_list = [ " --build-arg {}".format(arg) for arg in self.build_args ] command = command + "".join(build_args_list) if self.label: command = command + " --label " + self.label if self.target: command = command + " --target " + self.target try: logger.info("Running build using Kaniko: %s", command) run_cmd(command, shell=True, cwd=project_path()) except Exception as e: raise DatabandRuntimeError( "failed building docker image {}".format("?"), nested_exceptions=[e])
def sample_path(*path): return project_path("tests/dbnd_airflow/targets_tests/performance/data", *path)