def log_file_ref(self,
path: str,
name: str = None,
hash: str = None,
content=None):
"""Logs file reference.
Args:
path: str, filepath, the name is extracted from the filepath.
name: str, if the name is passed it will be used instead of the filename from the path.
hash: str, optional, default = None, the hash version of the file,
if not provided it will be calculated based on the file content.
content: the file content.
"""
summary = {"path": path}
if hash:
summary["hash"] = hash
elif content is not None:
summary["hash"] = hash_value(content)
name = name or os.path.basename(path)
if name:
artifact_run = V1RunArtifact(
name=name,
kind=V1ArtifactKind.FILE,
summary=summary,
is_input=True,
)
self.log_artifact_lineage(body=artifact_run)
def log_data_ref(
self,
name: str,
hash: str = None,
path: str = None,
content=None,
is_input: bool = True,
):
"""Logs data reference.
Args:
name: str, name of the data.
hash: str, optional, default = None, the hash version of the data,
if not provided it will be calculated based on the data in the content.
path: str, optional, path of where the data is coming from.
is_input: bool, if the data reference is an input or outputs.
content: the data content.
"""
summary = {}
if hash:
summary["hash"] = hash
elif content is not None:
summary["hash"] = hash_value(content)
if path is not None:
summary["path"] = path
if name:
artifact_run = V1RunArtifact(
name=name,
kind=V1ArtifactKind.DATA,
path=path,
summary=summary,
is_input=is_input,
)
self.log_artifact_lineage(body=artifact_run)
def file(file_context, filepath, copy_path, track):
"""Create auth context."""
from polyaxon.init.file import create_file_lineage
from polyaxon.utils.hashing import hash_value
try:
file_context = V1FileType.from_dict(ConfigSpec.read_from(file_context))
except (PolyaxonSchemaError, ValidationError) as e:
Printer.print_error("received a non valid file context.")
Printer.print_error("Error message: {}.".format(e))
sys.exit(1)
filepath = os.path.join(filepath, file_context.filename)
check_or_create_path(filepath, is_dir=False)
# Clean any previous file on that path
if os.path.exists(filepath):
os.remove(filepath)
with open(filepath, "w") as generated_file:
generated_file.write(file_context.content)
if file_context.chmod:
subprocess.check_call(["chmod", file_context.chmod, filepath])
if copy_path:
filepath = copy_file(filepath, copy_path)
if track:
create_file_lineage(
filepath=filepath,
summary={"hash": hash_value(file_context.content)},
kind=file_context.kind,
)
Printer.print_success("File is initialized, path: `{}`".format(filepath))
def log_data_ref(self, name: str, data):
if name:
artifact_run = V1RunArtifact(
name=name,
kind=V1ArtifactKind.DATA,
summary={"hash": hash_value(data)},
is_input=True,
)
self.log_artifact_lineage(body=artifact_run)
def generate(polyaxonfile, python_module, build_context, destination,
copy_path, params, track):
"""Generate a dockerfile given the polyaxonfile."""
from polyaxon.init.dockerfile import create_dockerfile_lineage
from polyaxon.utils.hashing import hash_value
if all([polyaxonfile, build_context]):
Printer.print_error(
"Only a polyaxonfile or a build context option is required.")
sys.exit(1)
if build_context:
try:
build_context = [
V1DockerfileType.from_dict(ConfigSpec.read_from(build_context))
]
except (PolyaxonSchemaError, ValidationError) as e:
Printer.print_error("received a non valid build context.")
Printer.print_error("Error message: {}.".format(e))
sys.exit(1)
else:
specification = check_polyaxonfile(
polyaxonfile=polyaxonfile,
python_module=python_module,
params=params,
verbose=False,
)
try:
compiled_operation = OperationSpecification.compile_operation(
specification)
compiled_operation.apply_params(params=specification.config.params)
compiled_operation = CompiledOperationSpecification.apply_operation_contexts(
compiled_operation)
except PolyaxonSchemaError:
Printer.print_error(
"Could not run this polyaxonfile locally, "
"a context is required to resolve it dependencies.")
sys.exit(1)
build_context = compiled_operation.init_dockerfiles
for init_dockerfile in build_context:
generator = DockerFileGenerator(build_context=init_dockerfile,
destination=destination or ".")
generator.create()
Printer.print_success("Dockerfile was generated, path: `{}`".format(
generator.dockerfile_path))
dockerfile_path = generator.dockerfile_path
if copy_path:
dockerfile_path = copy_file(dockerfile_path, copy_path)
if track:
hash_content = hash_value(init_dockerfile.to_dict())
create_dockerfile_lineage(dockerfile_path,
summary={"hash": hash_content})
def log_file_ref(self, path: str, hash: str = None, content=None):
summary = {"path": path}
if hash:
summary["hash"] = hash
elif content is not None:
summary["hash"] = hash_value(content)
name = os.path.basename(path)
if name:
artifact_run = V1RunArtifact(
name=name,
kind=V1ArtifactKind.FILE,
summary=summary,
is_input=True,
)
self.log_artifact_lineage(body=artifact_run)
def log_data_ref(self,
name: str,
hash: str = None,
path: str = None,
content=None):
summary = {}
if hash:
summary["hash"] = hash
elif content is not None:
summary["hash"] = hash_value(content)
if path is not None:
summary["path"] = path
if name:
artifact_run = V1RunArtifact(
name=name,
kind=V1ArtifactKind.DATA,
summary=summary,
is_input=True,
)
self.log_artifact_lineage(body=artifact_run)
def log_file_ref(
self,
path: str,
name: str = None,
hash: str = None,
content=None,
is_input: bool = False,
rel_path: str = None,
):
"""Logs file reference.
Args:
path: str, filepath, the name is extracted from the filepath.
name: str, if the name is passed it will be used instead of the filename from the path.
hash: str, optional, default = None, the hash version of the file,
if not provided it will be calculated based on the file content.
content: the file content.
is_input: bool, if the file reference is an input or outputs.
rel_path: str, optional relative path to the run artifacts path.
"""
summary = {"path": path}
if hash:
summary["hash"] = hash
elif content is not None:
summary["hash"] = hash_value(content)
name = name or os.path.basename(path)
rel_path = self._get_rel_asset_path(path=path, rel_path=rel_path)
if name:
artifact_run = V1RunArtifact(
name=name,
kind=V1ArtifactKind.FILE,
path=rel_path,
summary=summary,
is_input=is_input,
)
self.log_artifact_lineage(body=artifact_run)
def _get_conda_env_name(conda_env):
conda_env_contents = open(conda_env).read() if conda_env else ""
conda_tag = hash_value(conda_env_contents)
return "polyaxon-{}".format(conda_tag)
