def log_file_ref(self, path: str, name: str = None, hash: str = None, content=None): """Logs file reference. Args: path: str, filepath, the name is extracted from the filepath. name: str, if the name is passed it will be used instead of the filename from the path. hash: str, optional, default = None, the hash version of the file, if not provided it will be calculated based on the file content. content: the file content. """ summary = {"path": path} if hash: summary["hash"] = hash elif content is not None: summary["hash"] = hash_value(content) name = name or os.path.basename(path) if name: artifact_run = V1RunArtifact( name=name, kind=V1ArtifactKind.FILE, summary=summary, is_input=True, ) self.log_artifact_lineage(body=artifact_run)
def log_data_ref( self, name: str, hash: str = None, path: str = None, content=None, is_input: bool = True, ): """Logs data reference. Args: name: str, name of the data. hash: str, optional, default = None, the hash version of the data, if not provided it will be calculated based on the data in the content. path: str, optional, path of where the data is coming from. is_input: bool, if the data reference is an input or outputs. content: the data content. """ summary = {} if hash: summary["hash"] = hash elif content is not None: summary["hash"] = hash_value(content) if path is not None: summary["path"] = path if name: artifact_run = V1RunArtifact( name=name, kind=V1ArtifactKind.DATA, path=path, summary=summary, is_input=is_input, ) self.log_artifact_lineage(body=artifact_run)
def file(file_context, filepath, copy_path, track): """Create auth context.""" from polyaxon.init.file import create_file_lineage from polyaxon.utils.hashing import hash_value try: file_context = V1FileType.from_dict(ConfigSpec.read_from(file_context)) except (PolyaxonSchemaError, ValidationError) as e: Printer.print_error("received a non valid file context.") Printer.print_error("Error message: {}.".format(e)) sys.exit(1) filepath = os.path.join(filepath, file_context.filename) check_or_create_path(filepath, is_dir=False) # Clean any previous file on that path if os.path.exists(filepath): os.remove(filepath) with open(filepath, "w") as generated_file: generated_file.write(file_context.content) if file_context.chmod: subprocess.check_call(["chmod", file_context.chmod, filepath]) if copy_path: filepath = copy_file(filepath, copy_path) if track: create_file_lineage( filepath=filepath, summary={"hash": hash_value(file_context.content)}, kind=file_context.kind, ) Printer.print_success("File is initialized, path: `{}`".format(filepath))
def log_data_ref(self, name: str, data): if name: artifact_run = V1RunArtifact( name=name, kind=V1ArtifactKind.DATA, summary={"hash": hash_value(data)}, is_input=True, ) self.log_artifact_lineage(body=artifact_run)
def generate(polyaxonfile, python_module, build_context, destination, copy_path, params, track): """Generate a dockerfile given the polyaxonfile.""" from polyaxon.init.dockerfile import create_dockerfile_lineage from polyaxon.utils.hashing import hash_value if all([polyaxonfile, build_context]): Printer.print_error( "Only a polyaxonfile or a build context option is required.") sys.exit(1) if build_context: try: build_context = [ V1DockerfileType.from_dict(ConfigSpec.read_from(build_context)) ] except (PolyaxonSchemaError, ValidationError) as e: Printer.print_error("received a non valid build context.") Printer.print_error("Error message: {}.".format(e)) sys.exit(1) else: specification = check_polyaxonfile( polyaxonfile=polyaxonfile, python_module=python_module, params=params, verbose=False, ) try: compiled_operation = OperationSpecification.compile_operation( specification) compiled_operation.apply_params(params=specification.config.params) compiled_operation = CompiledOperationSpecification.apply_operation_contexts( compiled_operation) except PolyaxonSchemaError: Printer.print_error( "Could not run this polyaxonfile locally, " "a context is required to resolve it dependencies.") sys.exit(1) build_context = compiled_operation.init_dockerfiles for init_dockerfile in build_context: generator = DockerFileGenerator(build_context=init_dockerfile, destination=destination or ".") generator.create() Printer.print_success("Dockerfile was generated, path: `{}`".format( generator.dockerfile_path)) dockerfile_path = generator.dockerfile_path if copy_path: dockerfile_path = copy_file(dockerfile_path, copy_path) if track: hash_content = hash_value(init_dockerfile.to_dict()) create_dockerfile_lineage(dockerfile_path, summary={"hash": hash_content})
def log_file_ref(self, path: str, hash: str = None, content=None): summary = {"path": path} if hash: summary["hash"] = hash elif content is not None: summary["hash"] = hash_value(content) name = os.path.basename(path) if name: artifact_run = V1RunArtifact( name=name, kind=V1ArtifactKind.FILE, summary=summary, is_input=True, ) self.log_artifact_lineage(body=artifact_run)
def log_data_ref(self, name: str, hash: str = None, path: str = None, content=None): summary = {} if hash: summary["hash"] = hash elif content is not None: summary["hash"] = hash_value(content) if path is not None: summary["path"] = path if name: artifact_run = V1RunArtifact( name=name, kind=V1ArtifactKind.DATA, summary=summary, is_input=True, ) self.log_artifact_lineage(body=artifact_run)
def log_file_ref( self, path: str, name: str = None, hash: str = None, content=None, is_input: bool = False, rel_path: str = None, ): """Logs file reference. Args: path: str, filepath, the name is extracted from the filepath. name: str, if the name is passed it will be used instead of the filename from the path. hash: str, optional, default = None, the hash version of the file, if not provided it will be calculated based on the file content. content: the file content. is_input: bool, if the file reference is an input or outputs. rel_path: str, optional relative path to the run artifacts path. """ summary = {"path": path} if hash: summary["hash"] = hash elif content is not None: summary["hash"] = hash_value(content) name = name or os.path.basename(path) rel_path = self._get_rel_asset_path(path=path, rel_path=rel_path) if name: artifact_run = V1RunArtifact( name=name, kind=V1ArtifactKind.FILE, path=rel_path, summary=summary, is_input=is_input, ) self.log_artifact_lineage(body=artifact_run)
def _get_conda_env_name(conda_env): conda_env_contents = open(conda_env).read() if conda_env else "" conda_tag = hash_value(conda_env_contents) return "polyaxon-{}".format(conda_tag)